From 33c801c5b320974a94a9a22c7d0257f22e913abb Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 18 Aug 2016 11:34:25 +0300 Subject: [PATCH 001/569] Implemented exremely simple JIT using DynASM (it may crash on x86-64 yet, because of incorrect stack alignment) --- ext/opcache/Makefile.frag | 9 + ext/opcache/ZendAccelerator.c | 13 + ext/opcache/ZendAccelerator.h | 4 + ext/opcache/config.m4 | 13 +- ext/opcache/dynasm/dasm_arm.h | 456 ++ ext/opcache/dynasm/dasm_arm.lua | 1125 ++++ ext/opcache/dynasm/dasm_arm64.h | 518 ++ ext/opcache/dynasm/dasm_arm64.lua | 1166 ++++ ext/opcache/dynasm/dasm_mips.h | 419 ++ ext/opcache/dynasm/dasm_mips.lua | 1008 ++++ ext/opcache/dynasm/dasm_mips64.lua | 12 + ext/opcache/dynasm/dasm_ppc.h | 419 ++ ext/opcache/dynasm/dasm_ppc.lua | 1919 ++++++ ext/opcache/dynasm/dasm_proto.h | 83 + ext/opcache/dynasm/dasm_x64.lua | 12 + ext/opcache/dynasm/dasm_x86.h | 498 ++ ext/opcache/dynasm/dasm_x86.lua | 2274 ++++++++ ext/opcache/dynasm/dynasm.lua | 1094 ++++ ext/opcache/dynasm/minilua.c | 7770 +++++++++++++++++++++++++ ext/opcache/jit/zend_jit.c | 412 ++ ext/opcache/jit/zend_jit.h | 36 + ext/opcache/jit/zend_jit_x86.dasc | 204 + ext/opcache/zend_accelerator_module.c | 4 + ext/opcache/zend_persist.c | 16 + 24 files changed, 19483 insertions(+), 1 deletion(-) create mode 100644 ext/opcache/Makefile.frag create mode 100644 ext/opcache/dynasm/dasm_arm.h create mode 100644 ext/opcache/dynasm/dasm_arm.lua create mode 100644 ext/opcache/dynasm/dasm_arm64.h create mode 100644 ext/opcache/dynasm/dasm_arm64.lua create mode 100644 ext/opcache/dynasm/dasm_mips.h create mode 100644 ext/opcache/dynasm/dasm_mips.lua create mode 100644 ext/opcache/dynasm/dasm_mips64.lua create mode 100644 ext/opcache/dynasm/dasm_ppc.h create mode 100644 ext/opcache/dynasm/dasm_ppc.lua create mode 100644 ext/opcache/dynasm/dasm_proto.h create mode 100644 ext/opcache/dynasm/dasm_x64.lua create mode 100644 ext/opcache/dynasm/dasm_x86.h create mode 100644 ext/opcache/dynasm/dasm_x86.lua create mode 100644 ext/opcache/dynasm/dynasm.lua create mode 100644 ext/opcache/dynasm/minilua.c create mode 100644 ext/opcache/jit/zend_jit.c create mode 100644 ext/opcache/jit/zend_jit.h create mode 100644 ext/opcache/jit/zend_jit_x86.dasc diff --git a/ext/opcache/Makefile.frag b/ext/opcache/Makefile.frag new file mode 100644 index 0000000000000..9273e8595aa0a --- /dev/null +++ b/ext/opcache/Makefile.frag @@ -0,0 +1,9 @@ + +$(builddir)/minilua: $(srcdir)/dynasm/minilua.c + $(CC) $(srcdir)/dynasm/minilua.c -lm -o $@ + +$(builddir)/jit/zend_jit_x86.c: $(srcdir)/jit/zend_jit_x86.dasc $(srcdir)/dynasm/*.lua $(builddir)/minilua + $(builddir)/minilua $(srcdir)/dynasm/dynasm.lua -o $@ $(srcdir)/jit/zend_jit_x86.dasc + +$(builddir)/jit/zend_jit.lo: $(builddir)/jit/zend_jit_x86.c + diff --git a/ext/opcache/ZendAccelerator.c b/ext/opcache/ZendAccelerator.c index 769ce92049629..52834edaedfe3 100644 --- a/ext/opcache/ZendAccelerator.c +++ b/ext/opcache/ZendAccelerator.c @@ -46,6 +46,10 @@ # include "zend_file_cache.h" #endif +#ifdef HAVE_JIT +# include "jit/zend_jit.h" +#endif + #ifndef ZEND_WIN32 #include #endif @@ -2764,6 +2768,11 @@ static int accel_startup(zend_extension *extension) zend_accel_init_auto_globals(); zend_shared_alloc_lock(); +#ifdef HAVE_JIT + if (ZCG(accel_directives).jit_buffer_size) { + zend_jit_startup(ZCG(accel_directives).jit_buffer_size); + } +#endif zend_shared_alloc_save_state(); zend_shared_alloc_unlock(); @@ -2847,6 +2856,10 @@ void accel_shutdown(void) zend_ini_entry *ini_entry; zend_bool file_cache_only = 0; +#ifdef HAVE_JIT + zend_jit_shutdown(); +#endif + zend_optimizer_shutdown(); zend_accel_blacklist_shutdown(&accel_blacklist); diff --git a/ext/opcache/ZendAccelerator.h b/ext/opcache/ZendAccelerator.h index d45be562799eb..f69af7dd68707 100644 --- a/ext/opcache/ZendAccelerator.h +++ b/ext/opcache/ZendAccelerator.h @@ -208,6 +208,10 @@ typedef struct _zend_accel_directives { #ifdef HAVE_HUGE_CODE_PAGES zend_bool huge_code_pages; #endif +#ifdef HAVE_JIT + zend_long jit_buffer_size; + zend_long jit_debug; +#endif } zend_accel_directives; typedef struct _zend_accel_globals { diff --git a/ext/opcache/config.m4 b/ext/opcache/config.m4 index ded7f3dab21da..45a44f544a10e 100644 --- a/ext/opcache/config.m4 +++ b/ext/opcache/config.m4 @@ -12,6 +12,9 @@ PHP_ARG_ENABLE(huge-code-pages, whether to enable copying PHP CODE pages into HU [ --disable-huge-code-pages Disable copying PHP CODE pages into HUGE PAGES], yes, no) +PHP_ARG_ENABLE(opcache-jit, whether to enable JIT, +[ --disable-opcache-jit Disable JIT], yes) + if test "$PHP_OPCACHE" != "no"; then if test "$PHP_OPCACHE_FILE" = "yes"; then @@ -22,6 +25,10 @@ if test "$PHP_OPCACHE" != "no"; then AC_DEFINE(HAVE_HUGE_CODE_PAGES, 1, [Define to enable copying PHP CODE pages into HUGE PAGES (experimental)]) fi + if test "$PHP_OPCACHE_JIT" = "yes"; then + AC_DEFINE(HAVE_JIT, 1, [Define to enable JIT]) + fi + AC_CHECK_FUNC(mprotect,[ AC_DEFINE(HAVE_MPROTECT, 1, [Define if you have mprotect() function]) ]) @@ -410,9 +417,13 @@ fi Optimizer/zend_inference.c \ Optimizer/zend_func_info.c \ Optimizer/zend_call_graph.c \ - Optimizer/zend_dump.c, + Optimizer/zend_dump.c \ + jit/zend_jit.c, shared,,-DZEND_ENABLE_STATIC_TSRMLS_CACHE=1,,yes) PHP_ADD_BUILD_DIR([$ext_builddir/Optimizer], 1) + PHP_ADD_BUILD_DIR([$ext_builddir/jit], 1) PHP_ADD_EXTENSION_DEP(opcache, pcre) + + PHP_ADD_MAKEFILE_FRAGMENT fi diff --git a/ext/opcache/dynasm/dasm_arm.h b/ext/opcache/dynasm/dasm_arm.h new file mode 100644 index 0000000000000..0fa69ac06f96b --- /dev/null +++ b/ext/opcache/dynasm/dasm_arm.h @@ -0,0 +1,456 @@ +/* +** DynASM ARM encoding engine. +** Copyright (C) 2005-2016 Mike Pall. All rights reserved. +** Released under the MIT license. See dynasm.lua for full copyright notice. +*/ + +#include +#include +#include +#include + +#define DASM_ARCH "arm" + +#ifndef DASM_EXTERN +#define DASM_EXTERN(a,b,c,d) 0 +#endif + +/* Action definitions. */ +enum { + DASM_STOP, DASM_SECTION, DASM_ESC, DASM_REL_EXT, + /* The following actions need a buffer position. */ + DASM_ALIGN, DASM_REL_LG, DASM_LABEL_LG, + /* The following actions also have an argument. */ + DASM_REL_PC, DASM_LABEL_PC, + DASM_IMM, DASM_IMM12, DASM_IMM16, DASM_IMML8, DASM_IMML12, DASM_IMMV8, + DASM__MAX +}; + +/* Maximum number of section buffer positions for a single dasm_put() call. */ +#define DASM_MAXSECPOS 25 + +/* DynASM encoder status codes. Action list offset or number are or'ed in. */ +#define DASM_S_OK 0x00000000 +#define DASM_S_NOMEM 0x01000000 +#define DASM_S_PHASE 0x02000000 +#define DASM_S_MATCH_SEC 0x03000000 +#define DASM_S_RANGE_I 0x11000000 +#define DASM_S_RANGE_SEC 0x12000000 +#define DASM_S_RANGE_LG 0x13000000 +#define DASM_S_RANGE_PC 0x14000000 +#define DASM_S_RANGE_REL 0x15000000 +#define DASM_S_UNDEF_LG 0x21000000 +#define DASM_S_UNDEF_PC 0x22000000 + +/* Macros to convert positions (8 bit section + 24 bit index). */ +#define DASM_POS2IDX(pos) ((pos)&0x00ffffff) +#define DASM_POS2BIAS(pos) ((pos)&0xff000000) +#define DASM_SEC2POS(sec) ((sec)<<24) +#define DASM_POS2SEC(pos) ((pos)>>24) +#define DASM_POS2PTR(D, pos) (D->sections[DASM_POS2SEC(pos)].rbuf + (pos)) + +/* Action list type. */ +typedef const unsigned int *dasm_ActList; + +/* Per-section structure. */ +typedef struct dasm_Section { + int *rbuf; /* Biased buffer pointer (negative section bias). */ + int *buf; /* True buffer pointer. */ + size_t bsize; /* Buffer size in bytes. */ + int pos; /* Biased buffer position. */ + int epos; /* End of biased buffer position - max single put. */ + int ofs; /* Byte offset into section. */ +} dasm_Section; + +/* Core structure holding the DynASM encoding state. */ +struct dasm_State { + size_t psize; /* Allocated size of this structure. */ + dasm_ActList actionlist; /* Current actionlist pointer. */ + int *lglabels; /* Local/global chain/pos ptrs. */ + size_t lgsize; + int *pclabels; /* PC label chains/pos ptrs. */ + size_t pcsize; + void **globals; /* Array of globals (bias -10). */ + dasm_Section *section; /* Pointer to active section. */ + size_t codesize; /* Total size of all code sections. */ + int maxsection; /* 0 <= sectionidx < maxsection. */ + int status; /* Status code. */ + dasm_Section sections[1]; /* All sections. Alloc-extended. */ +}; + +/* The size of the core structure depends on the max. number of sections. */ +#define DASM_PSZ(ms) (sizeof(dasm_State)+(ms-1)*sizeof(dasm_Section)) + + +/* Initialize DynASM state. */ +void dasm_init(Dst_DECL, int maxsection) +{ + dasm_State *D; + size_t psz = 0; + int i; + Dst_REF = NULL; + DASM_M_GROW(Dst, struct dasm_State, Dst_REF, psz, DASM_PSZ(maxsection)); + D = Dst_REF; + D->psize = psz; + D->lglabels = NULL; + D->lgsize = 0; + D->pclabels = NULL; + D->pcsize = 0; + D->globals = NULL; + D->maxsection = maxsection; + for (i = 0; i < maxsection; i++) { + D->sections[i].buf = NULL; /* Need this for pass3. */ + D->sections[i].rbuf = D->sections[i].buf - DASM_SEC2POS(i); + D->sections[i].bsize = 0; + D->sections[i].epos = 0; /* Wrong, but is recalculated after resize. */ + } +} + +/* Free DynASM state. */ +void dasm_free(Dst_DECL) +{ + dasm_State *D = Dst_REF; + int i; + for (i = 0; i < D->maxsection; i++) + if (D->sections[i].buf) + DASM_M_FREE(Dst, D->sections[i].buf, D->sections[i].bsize); + if (D->pclabels) DASM_M_FREE(Dst, D->pclabels, D->pcsize); + if (D->lglabels) DASM_M_FREE(Dst, D->lglabels, D->lgsize); + DASM_M_FREE(Dst, D, D->psize); +} + +/* Setup global label array. Must be called before dasm_setup(). */ +void dasm_setupglobal(Dst_DECL, void **gl, unsigned int maxgl) +{ + dasm_State *D = Dst_REF; + D->globals = gl - 10; /* Negative bias to compensate for locals. */ + DASM_M_GROW(Dst, int, D->lglabels, D->lgsize, (10+maxgl)*sizeof(int)); +} + +/* Grow PC label array. Can be called after dasm_setup(), too. */ +void dasm_growpc(Dst_DECL, unsigned int maxpc) +{ + dasm_State *D = Dst_REF; + size_t osz = D->pcsize; + DASM_M_GROW(Dst, int, D->pclabels, D->pcsize, maxpc*sizeof(int)); + memset((void *)(((unsigned char *)D->pclabels)+osz), 0, D->pcsize-osz); +} + +/* Setup encoder. */ +void dasm_setup(Dst_DECL, const void *actionlist) +{ + dasm_State *D = Dst_REF; + int i; + D->actionlist = (dasm_ActList)actionlist; + D->status = DASM_S_OK; + D->section = &D->sections[0]; + memset((void *)D->lglabels, 0, D->lgsize); + if (D->pclabels) memset((void *)D->pclabels, 0, D->pcsize); + for (i = 0; i < D->maxsection; i++) { + D->sections[i].pos = DASM_SEC2POS(i); + D->sections[i].ofs = 0; + } +} + + +#ifdef DASM_CHECKS +#define CK(x, st) \ + do { if (!(x)) { \ + D->status = DASM_S_##st|(p-D->actionlist-1); return; } } while (0) +#define CKPL(kind, st) \ + do { if ((size_t)((char *)pl-(char *)D->kind##labels) >= D->kind##size) { \ + D->status = DASM_S_RANGE_##st|(p-D->actionlist-1); return; } } while (0) +#else +#define CK(x, st) ((void)0) +#define CKPL(kind, st) ((void)0) +#endif + +static int dasm_imm12(unsigned int n) +{ + int i; + for (i = 0; i < 16; i++, n = (n << 2) | (n >> 30)) + if (n <= 255) return (int)(n + (i << 8)); + return -1; +} + +/* Pass 1: Store actions and args, link branches/labels, estimate offsets. */ +void dasm_put(Dst_DECL, int start, ...) +{ + va_list ap; + dasm_State *D = Dst_REF; + dasm_ActList p = D->actionlist + start; + dasm_Section *sec = D->section; + int pos = sec->pos, ofs = sec->ofs; + int *b; + + if (pos >= sec->epos) { + DASM_M_GROW(Dst, int, sec->buf, sec->bsize, + sec->bsize + 2*DASM_MAXSECPOS*sizeof(int)); + sec->rbuf = sec->buf - DASM_POS2BIAS(pos); + sec->epos = (int)sec->bsize/sizeof(int) - DASM_MAXSECPOS+DASM_POS2BIAS(pos); + } + + b = sec->rbuf; + b[pos++] = start; + + va_start(ap, start); + while (1) { + unsigned int ins = *p++; + unsigned int action = (ins >> 16); + if (action >= DASM__MAX) { + ofs += 4; + } else { + int *pl, n = action >= DASM_REL_PC ? va_arg(ap, int) : 0; + switch (action) { + case DASM_STOP: goto stop; + case DASM_SECTION: + n = (ins & 255); CK(n < D->maxsection, RANGE_SEC); + D->section = &D->sections[n]; goto stop; + case DASM_ESC: p++; ofs += 4; break; + case DASM_REL_EXT: break; + case DASM_ALIGN: ofs += (ins & 255); b[pos++] = ofs; break; + case DASM_REL_LG: + n = (ins & 2047) - 10; pl = D->lglabels + n; + /* Bkwd rel or global. */ + if (n >= 0) { CK(n>=10||*pl<0, RANGE_LG); CKPL(lg, LG); goto putrel; } + pl += 10; n = *pl; + if (n < 0) n = 0; /* Start new chain for fwd rel if label exists. */ + goto linkrel; + case DASM_REL_PC: + pl = D->pclabels + n; CKPL(pc, PC); + putrel: + n = *pl; + if (n < 0) { /* Label exists. Get label pos and store it. */ + b[pos] = -n; + } else { + linkrel: + b[pos] = n; /* Else link to rel chain, anchored at label. */ + *pl = pos; + } + pos++; + break; + case DASM_LABEL_LG: + pl = D->lglabels + (ins & 2047) - 10; CKPL(lg, LG); goto putlabel; + case DASM_LABEL_PC: + pl = D->pclabels + n; CKPL(pc, PC); + putlabel: + n = *pl; /* n > 0: Collapse rel chain and replace with label pos. */ + while (n > 0) { int *pb = DASM_POS2PTR(D, n); n = *pb; *pb = pos; + } + *pl = -pos; /* Label exists now. */ + b[pos++] = ofs; /* Store pass1 offset estimate. */ + break; + case DASM_IMM: + case DASM_IMM16: +#ifdef DASM_CHECKS + CK((n & ((1<<((ins>>10)&31))-1)) == 0, RANGE_I); + if ((ins & 0x8000)) + CK(((n + (1<<(((ins>>5)&31)-1)))>>((ins>>5)&31)) == 0, RANGE_I); + else + CK((n>>((ins>>5)&31)) == 0, RANGE_I); +#endif + b[pos++] = n; + break; + case DASM_IMMV8: + CK((n & 3) == 0, RANGE_I); + n >>= 2; + case DASM_IMML8: + case DASM_IMML12: + CK(n >= 0 ? ((n>>((ins>>5)&31)) == 0) : + (((-n)>>((ins>>5)&31)) == 0), RANGE_I); + b[pos++] = n; + break; + case DASM_IMM12: + CK(dasm_imm12((unsigned int)n) != -1, RANGE_I); + b[pos++] = n; + break; + } + } + } +stop: + va_end(ap); + sec->pos = pos; + sec->ofs = ofs; +} +#undef CK + +/* Pass 2: Link sections, shrink aligns, fix label offsets. */ +int dasm_link(Dst_DECL, size_t *szp) +{ + dasm_State *D = Dst_REF; + int secnum; + int ofs = 0; + +#ifdef DASM_CHECKS + *szp = 0; + if (D->status != DASM_S_OK) return D->status; + { + int pc; + for (pc = 0; pc*sizeof(int) < D->pcsize; pc++) + if (D->pclabels[pc] > 0) return DASM_S_UNDEF_PC|pc; + } +#endif + + { /* Handle globals not defined in this translation unit. */ + int idx; + for (idx = 20; idx*sizeof(int) < D->lgsize; idx++) { + int n = D->lglabels[idx]; + /* Undefined label: Collapse rel chain and replace with marker (< 0). */ + while (n > 0) { int *pb = DASM_POS2PTR(D, n); n = *pb; *pb = -idx; } + } + } + + /* Combine all code sections. No support for data sections (yet). */ + for (secnum = 0; secnum < D->maxsection; secnum++) { + dasm_Section *sec = D->sections + secnum; + int *b = sec->rbuf; + int pos = DASM_SEC2POS(secnum); + int lastpos = sec->pos; + + while (pos != lastpos) { + dasm_ActList p = D->actionlist + b[pos++]; + while (1) { + unsigned int ins = *p++; + unsigned int action = (ins >> 16); + switch (action) { + case DASM_STOP: case DASM_SECTION: goto stop; + case DASM_ESC: p++; break; + case DASM_REL_EXT: break; + case DASM_ALIGN: ofs -= (b[pos++] + ofs) & (ins & 255); break; + case DASM_REL_LG: case DASM_REL_PC: pos++; break; + case DASM_LABEL_LG: case DASM_LABEL_PC: b[pos++] += ofs; break; + case DASM_IMM: case DASM_IMM12: case DASM_IMM16: + case DASM_IMML8: case DASM_IMML12: case DASM_IMMV8: pos++; break; + } + } + stop: (void)0; + } + ofs += sec->ofs; /* Next section starts right after current section. */ + } + + D->codesize = ofs; /* Total size of all code sections */ + *szp = ofs; + return DASM_S_OK; +} + +#ifdef DASM_CHECKS +#define CK(x, st) \ + do { if (!(x)) return DASM_S_##st|(p-D->actionlist-1); } while (0) +#else +#define CK(x, st) ((void)0) +#endif + +/* Pass 3: Encode sections. */ +int dasm_encode(Dst_DECL, void *buffer) +{ + dasm_State *D = Dst_REF; + char *base = (char *)buffer; + unsigned int *cp = (unsigned int *)buffer; + int secnum; + + /* Encode all code sections. No support for data sections (yet). */ + for (secnum = 0; secnum < D->maxsection; secnum++) { + dasm_Section *sec = D->sections + secnum; + int *b = sec->buf; + int *endb = sec->rbuf + sec->pos; + + while (b != endb) { + dasm_ActList p = D->actionlist + *b++; + while (1) { + unsigned int ins = *p++; + unsigned int action = (ins >> 16); + int n = (action >= DASM_ALIGN && action < DASM__MAX) ? *b++ : 0; + switch (action) { + case DASM_STOP: case DASM_SECTION: goto stop; + case DASM_ESC: *cp++ = *p++; break; + case DASM_REL_EXT: + n = DASM_EXTERN(Dst, (unsigned char *)cp, (ins&2047), !(ins&2048)); + goto patchrel; + case DASM_ALIGN: + ins &= 255; while ((((char *)cp - base) & ins)) *cp++ = 0xe1a00000; + break; + case DASM_REL_LG: + CK(n >= 0, UNDEF_LG); + case DASM_REL_PC: + CK(n >= 0, UNDEF_PC); + n = *DASM_POS2PTR(D, n) - (int)((char *)cp - base) - 4; + patchrel: + if ((ins & 0x800) == 0) { + CK((n & 3) == 0 && ((n+0x02000000) >> 26) == 0, RANGE_REL); + cp[-1] |= ((n >> 2) & 0x00ffffff); + } else if ((ins & 0x1000)) { + CK((n & 3) == 0 && -256 <= n && n <= 256, RANGE_REL); + goto patchimml8; + } else if ((ins & 0x2000) == 0) { + CK((n & 3) == 0 && -4096 <= n && n <= 4096, RANGE_REL); + goto patchimml; + } else { + CK((n & 3) == 0 && -1020 <= n && n <= 1020, RANGE_REL); + n >>= 2; + goto patchimml; + } + break; + case DASM_LABEL_LG: + ins &= 2047; if (ins >= 20) D->globals[ins-10] = (void *)(base + n); + break; + case DASM_LABEL_PC: break; + case DASM_IMM: + cp[-1] |= ((n>>((ins>>10)&31)) & ((1<<((ins>>5)&31))-1)) << (ins&31); + break; + case DASM_IMM12: + cp[-1] |= dasm_imm12((unsigned int)n); + break; + case DASM_IMM16: + cp[-1] |= ((n & 0xf000) << 4) | (n & 0x0fff); + break; + case DASM_IMML8: patchimml8: + cp[-1] |= n >= 0 ? (0x00800000 | (n & 0x0f) | ((n & 0xf0) << 4)) : + ((-n & 0x0f) | ((-n & 0xf0) << 4)); + break; + case DASM_IMML12: case DASM_IMMV8: patchimml: + cp[-1] |= n >= 0 ? (0x00800000 | n) : (-n); + break; + default: *cp++ = ins; break; + } + } + stop: (void)0; + } + } + + if (base + D->codesize != (char *)cp) /* Check for phase errors. */ + return DASM_S_PHASE; + return DASM_S_OK; +} +#undef CK + +/* Get PC label offset. */ +int dasm_getpclabel(Dst_DECL, unsigned int pc) +{ + dasm_State *D = Dst_REF; + if (pc*sizeof(int) < D->pcsize) { + int pos = D->pclabels[pc]; + if (pos < 0) return *DASM_POS2PTR(D, -pos); + if (pos > 0) return -1; /* Undefined. */ + } + return -2; /* Unused or out of range. */ +} + +#ifdef DASM_CHECKS +/* Optional sanity checker to call between isolated encoding steps. */ +int dasm_checkstep(Dst_DECL, int secmatch) +{ + dasm_State *D = Dst_REF; + if (D->status == DASM_S_OK) { + int i; + for (i = 1; i <= 9; i++) { + if (D->lglabels[i] > 0) { D->status = DASM_S_UNDEF_LG|i; break; } + D->lglabels[i] = 0; + } + } + if (D->status == DASM_S_OK && secmatch >= 0 && + D->section != &D->sections[secmatch]) + D->status = DASM_S_MATCH_SEC|(D->section-D->sections); + return D->status; +} +#endif + diff --git a/ext/opcache/dynasm/dasm_arm.lua b/ext/opcache/dynasm/dasm_arm.lua new file mode 100644 index 0000000000000..e2ed922a3c68d --- /dev/null +++ b/ext/opcache/dynasm/dasm_arm.lua @@ -0,0 +1,1125 @@ +------------------------------------------------------------------------------ +-- DynASM ARM module. +-- +-- Copyright (C) 2005-2016 Mike Pall. All rights reserved. +-- See dynasm.lua for full copyright notice. +------------------------------------------------------------------------------ + +-- Module information: +local _info = { + arch = "arm", + description = "DynASM ARM module", + version = "1.4.0", + vernum = 10400, + release = "2015-10-18", + author = "Mike Pall", + license = "MIT", +} + +-- Exported glue functions for the arch-specific module. +local _M = { _info = _info } + +-- Cache library functions. +local type, tonumber, pairs, ipairs = type, tonumber, pairs, ipairs +local assert, setmetatable, rawget = assert, setmetatable, rawget +local _s = string +local sub, format, byte, char = _s.sub, _s.format, _s.byte, _s.char +local match, gmatch, gsub = _s.match, _s.gmatch, _s.gsub +local concat, sort, insert = table.concat, table.sort, table.insert +local bit = bit or require("bit") +local band, shl, shr, sar = bit.band, bit.lshift, bit.rshift, bit.arshift +local ror, tohex = bit.ror, bit.tohex + +-- Inherited tables and callbacks. +local g_opt, g_arch +local wline, werror, wfatal, wwarn + +-- Action name list. +-- CHECK: Keep this in sync with the C code! +local action_names = { + "STOP", "SECTION", "ESC", "REL_EXT", + "ALIGN", "REL_LG", "LABEL_LG", + "REL_PC", "LABEL_PC", "IMM", "IMM12", "IMM16", "IMML8", "IMML12", "IMMV8", +} + +-- Maximum number of section buffer positions for dasm_put(). +-- CHECK: Keep this in sync with the C code! +local maxsecpos = 25 -- Keep this low, to avoid excessively long C lines. + +-- Action name -> action number. +local map_action = {} +for n,name in ipairs(action_names) do + map_action[name] = n-1 +end + +-- Action list buffer. +local actlist = {} + +-- Argument list for next dasm_put(). Start with offset 0 into action list. +local actargs = { 0 } + +-- Current number of section buffer positions for dasm_put(). +local secpos = 1 + +------------------------------------------------------------------------------ + +-- Dump action names and numbers. +local function dumpactions(out) + out:write("DynASM encoding engine action codes:\n") + for n,name in ipairs(action_names) do + local num = map_action[name] + out:write(format(" %-10s %02X %d\n", name, num, num)) + end + out:write("\n") +end + +-- Write action list buffer as a huge static C array. +local function writeactions(out, name) + local nn = #actlist + if nn == 0 then nn = 1; actlist[0] = map_action.STOP end + out:write("static const unsigned int ", name, "[", nn, "] = {\n") + for i = 1,nn-1 do + assert(out:write("0x", tohex(actlist[i]), ",\n")) + end + assert(out:write("0x", tohex(actlist[nn]), "\n};\n\n")) +end + +------------------------------------------------------------------------------ + +-- Add word to action list. +local function wputxw(n) + assert(n >= 0 and n <= 0xffffffff and n % 1 == 0, "word out of range") + actlist[#actlist+1] = n +end + +-- Add action to list with optional arg. Advance buffer pos, too. +local function waction(action, val, a, num) + local w = assert(map_action[action], "bad action name `"..action.."'") + wputxw(w * 0x10000 + (val or 0)) + if a then actargs[#actargs+1] = a end + if a or num then secpos = secpos + (num or 1) end +end + +-- Flush action list (intervening C code or buffer pos overflow). +local function wflush(term) + if #actlist == actargs[1] then return end -- Nothing to flush. + if not term then waction("STOP") end -- Terminate action list. + wline(format("dasm_put(Dst, %s);", concat(actargs, ", ")), true) + actargs = { #actlist } -- Actionlist offset is 1st arg to next dasm_put(). + secpos = 1 -- The actionlist offset occupies a buffer position, too. +end + +-- Put escaped word. +local function wputw(n) + if n <= 0x000fffff then waction("ESC") end + wputxw(n) +end + +-- Reserve position for word. +local function wpos() + local pos = #actlist+1 + actlist[pos] = "" + return pos +end + +-- Store word to reserved position. +local function wputpos(pos, n) + assert(n >= 0 and n <= 0xffffffff and n % 1 == 0, "word out of range") + if n <= 0x000fffff then + insert(actlist, pos+1, n) + n = map_action.ESC * 0x10000 + end + actlist[pos] = n +end + +------------------------------------------------------------------------------ + +-- Global label name -> global label number. With auto assignment on 1st use. +local next_global = 20 +local map_global = setmetatable({}, { __index = function(t, name) + if not match(name, "^[%a_][%w_]*$") then werror("bad global label") end + local n = next_global + if n > 2047 then werror("too many global labels") end + next_global = n + 1 + t[name] = n + return n +end}) + +-- Dump global labels. +local function dumpglobals(out, lvl) + local t = {} + for name, n in pairs(map_global) do t[n] = name end + out:write("Global labels:\n") + for i=20,next_global-1 do + out:write(format(" %s\n", t[i])) + end + out:write("\n") +end + +-- Write global label enum. +local function writeglobals(out, prefix) + local t = {} + for name, n in pairs(map_global) do t[n] = name end + out:write("enum {\n") + for i=20,next_global-1 do + out:write(" ", prefix, t[i], ",\n") + end + out:write(" ", prefix, "_MAX\n};\n") +end + +-- Write global label names. +local function writeglobalnames(out, name) + local t = {} + for name, n in pairs(map_global) do t[n] = name end + out:write("static const char *const ", name, "[] = {\n") + for i=20,next_global-1 do + out:write(" \"", t[i], "\",\n") + end + out:write(" (const char *)0\n};\n") +end + +------------------------------------------------------------------------------ + +-- Extern label name -> extern label number. With auto assignment on 1st use. +local next_extern = 0 +local map_extern_ = {} +local map_extern = setmetatable({}, { __index = function(t, name) + -- No restrictions on the name for now. + local n = next_extern + if n > 2047 then werror("too many extern labels") end + next_extern = n + 1 + t[name] = n + map_extern_[n] = name + return n +end}) + +-- Dump extern labels. +local function dumpexterns(out, lvl) + out:write("Extern labels:\n") + for i=0,next_extern-1 do + out:write(format(" %s\n", map_extern_[i])) + end + out:write("\n") +end + +-- Write extern label names. +local function writeexternnames(out, name) + out:write("static const char *const ", name, "[] = {\n") + for i=0,next_extern-1 do + out:write(" \"", map_extern_[i], "\",\n") + end + out:write(" (const char *)0\n};\n") +end + +------------------------------------------------------------------------------ + +-- Arch-specific maps. + +-- Ext. register name -> int. name. +local map_archdef = { sp = "r13", lr = "r14", pc = "r15", } + +-- Int. register name -> ext. name. +local map_reg_rev = { r13 = "sp", r14 = "lr", r15 = "pc", } + +local map_type = {} -- Type name -> { ctype, reg } +local ctypenum = 0 -- Type number (for Dt... macros). + +-- Reverse defines for registers. +function _M.revdef(s) + return map_reg_rev[s] or s +end + +local map_shift = { lsl = 0, lsr = 1, asr = 2, ror = 3, } + +local map_cond = { + eq = 0, ne = 1, cs = 2, cc = 3, mi = 4, pl = 5, vs = 6, vc = 7, + hi = 8, ls = 9, ge = 10, lt = 11, gt = 12, le = 13, al = 14, + hs = 2, lo = 3, +} + +------------------------------------------------------------------------------ + +-- Template strings for ARM instructions. +local map_op = { + -- Basic data processing instructions. + and_3 = "e0000000DNPs", + eor_3 = "e0200000DNPs", + sub_3 = "e0400000DNPs", + rsb_3 = "e0600000DNPs", + add_3 = "e0800000DNPs", + adc_3 = "e0a00000DNPs", + sbc_3 = "e0c00000DNPs", + rsc_3 = "e0e00000DNPs", + tst_2 = "e1100000NP", + teq_2 = "e1300000NP", + cmp_2 = "e1500000NP", + cmn_2 = "e1700000NP", + orr_3 = "e1800000DNPs", + mov_2 = "e1a00000DPs", + bic_3 = "e1c00000DNPs", + mvn_2 = "e1e00000DPs", + + and_4 = "e0000000DNMps", + eor_4 = "e0200000DNMps", + sub_4 = "e0400000DNMps", + rsb_4 = "e0600000DNMps", + add_4 = "e0800000DNMps", + adc_4 = "e0a00000DNMps", + sbc_4 = "e0c00000DNMps", + rsc_4 = "e0e00000DNMps", + tst_3 = "e1100000NMp", + teq_3 = "e1300000NMp", + cmp_3 = "e1500000NMp", + cmn_3 = "e1700000NMp", + orr_4 = "e1800000DNMps", + mov_3 = "e1a00000DMps", + bic_4 = "e1c00000DNMps", + mvn_3 = "e1e00000DMps", + + lsl_3 = "e1a00000DMws", + lsr_3 = "e1a00020DMws", + asr_3 = "e1a00040DMws", + ror_3 = "e1a00060DMws", + rrx_2 = "e1a00060DMs", + + -- Multiply and multiply-accumulate. + mul_3 = "e0000090NMSs", + mla_4 = "e0200090NMSDs", + umaal_4 = "e0400090DNMSs", -- v6 + mls_4 = "e0600090DNMSs", -- v6T2 + umull_4 = "e0800090DNMSs", + umlal_4 = "e0a00090DNMSs", + smull_4 = "e0c00090DNMSs", + smlal_4 = "e0e00090DNMSs", + + -- Halfword multiply and multiply-accumulate. + smlabb_4 = "e1000080NMSD", -- v5TE + smlatb_4 = "e10000a0NMSD", -- v5TE + smlabt_4 = "e10000c0NMSD", -- v5TE + smlatt_4 = "e10000e0NMSD", -- v5TE + smlawb_4 = "e1200080NMSD", -- v5TE + smulwb_3 = "e12000a0NMS", -- v5TE + smlawt_4 = "e12000c0NMSD", -- v5TE + smulwt_3 = "e12000e0NMS", -- v5TE + smlalbb_4 = "e1400080NMSD", -- v5TE + smlaltb_4 = "e14000a0NMSD", -- v5TE + smlalbt_4 = "e14000c0NMSD", -- v5TE + smlaltt_4 = "e14000e0NMSD", -- v5TE + smulbb_3 = "e1600080NMS", -- v5TE + smultb_3 = "e16000a0NMS", -- v5TE + smulbt_3 = "e16000c0NMS", -- v5TE + smultt_3 = "e16000e0NMS", -- v5TE + + -- Miscellaneous data processing instructions. + clz_2 = "e16f0f10DM", -- v5T + rev_2 = "e6bf0f30DM", -- v6 + rev16_2 = "e6bf0fb0DM", -- v6 + revsh_2 = "e6ff0fb0DM", -- v6 + sel_3 = "e6800fb0DNM", -- v6 + usad8_3 = "e780f010NMS", -- v6 + usada8_4 = "e7800010NMSD", -- v6 + rbit_2 = "e6ff0f30DM", -- v6T2 + movw_2 = "e3000000DW", -- v6T2 + movt_2 = "e3400000DW", -- v6T2 + -- Note: the X encodes width-1, not width. + sbfx_4 = "e7a00050DMvX", -- v6T2 + ubfx_4 = "e7e00050DMvX", -- v6T2 + -- Note: the X encodes the msb field, not the width. + bfc_3 = "e7c0001fDvX", -- v6T2 + bfi_4 = "e7c00010DMvX", -- v6T2 + + -- Packing and unpacking instructions. + pkhbt_3 = "e6800010DNM", pkhbt_4 = "e6800010DNMv", -- v6 + pkhtb_3 = "e6800050DNM", pkhtb_4 = "e6800050DNMv", -- v6 + sxtab_3 = "e6a00070DNM", sxtab_4 = "e6a00070DNMv", -- v6 + sxtab16_3 = "e6800070DNM", sxtab16_4 = "e6800070DNMv", -- v6 + sxtah_3 = "e6b00070DNM", sxtah_4 = "e6b00070DNMv", -- v6 + sxtb_2 = "e6af0070DM", sxtb_3 = "e6af0070DMv", -- v6 + sxtb16_2 = "e68f0070DM", sxtb16_3 = "e68f0070DMv", -- v6 + sxth_2 = "e6bf0070DM", sxth_3 = "e6bf0070DMv", -- v6 + uxtab_3 = "e6e00070DNM", uxtab_4 = "e6e00070DNMv", -- v6 + uxtab16_3 = "e6c00070DNM", uxtab16_4 = "e6c00070DNMv", -- v6 + uxtah_3 = "e6f00070DNM", uxtah_4 = "e6f00070DNMv", -- v6 + uxtb_2 = "e6ef0070DM", uxtb_3 = "e6ef0070DMv", -- v6 + uxtb16_2 = "e6cf0070DM", uxtb16_3 = "e6cf0070DMv", -- v6 + uxth_2 = "e6ff0070DM", uxth_3 = "e6ff0070DMv", -- v6 + + -- Saturating instructions. + qadd_3 = "e1000050DMN", -- v5TE + qsub_3 = "e1200050DMN", -- v5TE + qdadd_3 = "e1400050DMN", -- v5TE + qdsub_3 = "e1600050DMN", -- v5TE + -- Note: the X for ssat* encodes sat_imm-1, not sat_imm. + ssat_3 = "e6a00010DXM", ssat_4 = "e6a00010DXMp", -- v6 + usat_3 = "e6e00010DXM", usat_4 = "e6e00010DXMp", -- v6 + ssat16_3 = "e6a00f30DXM", -- v6 + usat16_3 = "e6e00f30DXM", -- v6 + + -- Parallel addition and subtraction. + sadd16_3 = "e6100f10DNM", -- v6 + sasx_3 = "e6100f30DNM", -- v6 + ssax_3 = "e6100f50DNM", -- v6 + ssub16_3 = "e6100f70DNM", -- v6 + sadd8_3 = "e6100f90DNM", -- v6 + ssub8_3 = "e6100ff0DNM", -- v6 + qadd16_3 = "e6200f10DNM", -- v6 + qasx_3 = "e6200f30DNM", -- v6 + qsax_3 = "e6200f50DNM", -- v6 + qsub16_3 = "e6200f70DNM", -- v6 + qadd8_3 = "e6200f90DNM", -- v6 + qsub8_3 = "e6200ff0DNM", -- v6 + shadd16_3 = "e6300f10DNM", -- v6 + shasx_3 = "e6300f30DNM", -- v6 + shsax_3 = "e6300f50DNM", -- v6 + shsub16_3 = "e6300f70DNM", -- v6 + shadd8_3 = "e6300f90DNM", -- v6 + shsub8_3 = "e6300ff0DNM", -- v6 + uadd16_3 = "e6500f10DNM", -- v6 + uasx_3 = "e6500f30DNM", -- v6 + usax_3 = "e6500f50DNM", -- v6 + usub16_3 = "e6500f70DNM", -- v6 + uadd8_3 = "e6500f90DNM", -- v6 + usub8_3 = "e6500ff0DNM", -- v6 + uqadd16_3 = "e6600f10DNM", -- v6 + uqasx_3 = "e6600f30DNM", -- v6 + uqsax_3 = "e6600f50DNM", -- v6 + uqsub16_3 = "e6600f70DNM", -- v6 + uqadd8_3 = "e6600f90DNM", -- v6 + uqsub8_3 = "e6600ff0DNM", -- v6 + uhadd16_3 = "e6700f10DNM", -- v6 + uhasx_3 = "e6700f30DNM", -- v6 + uhsax_3 = "e6700f50DNM", -- v6 + uhsub16_3 = "e6700f70DNM", -- v6 + uhadd8_3 = "e6700f90DNM", -- v6 + uhsub8_3 = "e6700ff0DNM", -- v6 + + -- Load/store instructions. + str_2 = "e4000000DL", str_3 = "e4000000DL", str_4 = "e4000000DL", + strb_2 = "e4400000DL", strb_3 = "e4400000DL", strb_4 = "e4400000DL", + ldr_2 = "e4100000DL", ldr_3 = "e4100000DL", ldr_4 = "e4100000DL", + ldrb_2 = "e4500000DL", ldrb_3 = "e4500000DL", ldrb_4 = "e4500000DL", + strh_2 = "e00000b0DL", strh_3 = "e00000b0DL", + ldrh_2 = "e01000b0DL", ldrh_3 = "e01000b0DL", + ldrd_2 = "e00000d0DL", ldrd_3 = "e00000d0DL", -- v5TE + ldrsb_2 = "e01000d0DL", ldrsb_3 = "e01000d0DL", + strd_2 = "e00000f0DL", strd_3 = "e00000f0DL", -- v5TE + ldrsh_2 = "e01000f0DL", ldrsh_3 = "e01000f0DL", + + ldm_2 = "e8900000oR", ldmia_2 = "e8900000oR", ldmfd_2 = "e8900000oR", + ldmda_2 = "e8100000oR", ldmfa_2 = "e8100000oR", + ldmdb_2 = "e9100000oR", ldmea_2 = "e9100000oR", + ldmib_2 = "e9900000oR", ldmed_2 = "e9900000oR", + stm_2 = "e8800000oR", stmia_2 = "e8800000oR", stmfd_2 = "e8800000oR", + stmda_2 = "e8000000oR", stmfa_2 = "e8000000oR", + stmdb_2 = "e9000000oR", stmea_2 = "e9000000oR", + stmib_2 = "e9800000oR", stmed_2 = "e9800000oR", + pop_1 = "e8bd0000R", push_1 = "e92d0000R", + + -- Branch instructions. + b_1 = "ea000000B", + bl_1 = "eb000000B", + blx_1 = "e12fff30C", + bx_1 = "e12fff10M", + + -- Miscellaneous instructions. + nop_0 = "e1a00000", + mrs_1 = "e10f0000D", + bkpt_1 = "e1200070K", -- v5T + svc_1 = "ef000000T", swi_1 = "ef000000T", + ud_0 = "e7f001f0", + + -- VFP instructions. + ["vadd.f32_3"] = "ee300a00dnm", + ["vadd.f64_3"] = "ee300b00Gdnm", + ["vsub.f32_3"] = "ee300a40dnm", + ["vsub.f64_3"] = "ee300b40Gdnm", + ["vmul.f32_3"] = "ee200a00dnm", + ["vmul.f64_3"] = "ee200b00Gdnm", + ["vnmul.f32_3"] = "ee200a40dnm", + ["vnmul.f64_3"] = "ee200b40Gdnm", + ["vmla.f32_3"] = "ee000a00dnm", + ["vmla.f64_3"] = "ee000b00Gdnm", + ["vmls.f32_3"] = "ee000a40dnm", + ["vmls.f64_3"] = "ee000b40Gdnm", + ["vnmla.f32_3"] = "ee100a40dnm", + ["vnmla.f64_3"] = "ee100b40Gdnm", + ["vnmls.f32_3"] = "ee100a00dnm", + ["vnmls.f64_3"] = "ee100b00Gdnm", + ["vdiv.f32_3"] = "ee800a00dnm", + ["vdiv.f64_3"] = "ee800b00Gdnm", + + ["vabs.f32_2"] = "eeb00ac0dm", + ["vabs.f64_2"] = "eeb00bc0Gdm", + ["vneg.f32_2"] = "eeb10a40dm", + ["vneg.f64_2"] = "eeb10b40Gdm", + ["vsqrt.f32_2"] = "eeb10ac0dm", + ["vsqrt.f64_2"] = "eeb10bc0Gdm", + ["vcmp.f32_2"] = "eeb40a40dm", + ["vcmp.f64_2"] = "eeb40b40Gdm", + ["vcmpe.f32_2"] = "eeb40ac0dm", + ["vcmpe.f64_2"] = "eeb40bc0Gdm", + ["vcmpz.f32_1"] = "eeb50a40d", + ["vcmpz.f64_1"] = "eeb50b40Gd", + ["vcmpze.f32_1"] = "eeb50ac0d", + ["vcmpze.f64_1"] = "eeb50bc0Gd", + + vldr_2 = "ed100a00dl|ed100b00Gdl", + vstr_2 = "ed000a00dl|ed000b00Gdl", + vldm_2 = "ec900a00or", + vldmia_2 = "ec900a00or", + vldmdb_2 = "ed100a00or", + vpop_1 = "ecbd0a00r", + vstm_2 = "ec800a00or", + vstmia_2 = "ec800a00or", + vstmdb_2 = "ed000a00or", + vpush_1 = "ed2d0a00r", + + ["vmov.f32_2"] = "eeb00a40dm|eeb00a00dY", -- #imm is VFPv3 only + ["vmov.f64_2"] = "eeb00b40Gdm|eeb00b00GdY", -- #imm is VFPv3 only + vmov_2 = "ee100a10Dn|ee000a10nD", + vmov_3 = "ec500a10DNm|ec400a10mDN|ec500b10GDNm|ec400b10GmDN", + + vmrs_0 = "eef1fa10", + vmrs_1 = "eef10a10D", + vmsr_1 = "eee10a10D", + + ["vcvt.s32.f32_2"] = "eebd0ac0dm", + ["vcvt.s32.f64_2"] = "eebd0bc0dGm", + ["vcvt.u32.f32_2"] = "eebc0ac0dm", + ["vcvt.u32.f64_2"] = "eebc0bc0dGm", + ["vcvtr.s32.f32_2"] = "eebd0a40dm", + ["vcvtr.s32.f64_2"] = "eebd0b40dGm", + ["vcvtr.u32.f32_2"] = "eebc0a40dm", + ["vcvtr.u32.f64_2"] = "eebc0b40dGm", + ["vcvt.f32.s32_2"] = "eeb80ac0dm", + ["vcvt.f64.s32_2"] = "eeb80bc0GdFm", + ["vcvt.f32.u32_2"] = "eeb80a40dm", + ["vcvt.f64.u32_2"] = "eeb80b40GdFm", + ["vcvt.f32.f64_2"] = "eeb70bc0dGm", + ["vcvt.f64.f32_2"] = "eeb70ac0GdFm", + + -- VFPv4 only: + ["vfma.f32_3"] = "eea00a00dnm", + ["vfma.f64_3"] = "eea00b00Gdnm", + ["vfms.f32_3"] = "eea00a40dnm", + ["vfms.f64_3"] = "eea00b40Gdnm", + ["vfnma.f32_3"] = "ee900a40dnm", + ["vfnma.f64_3"] = "ee900b40Gdnm", + ["vfnms.f32_3"] = "ee900a00dnm", + ["vfnms.f64_3"] = "ee900b00Gdnm", + + -- NYI: Advanced SIMD instructions. + + -- NYI: I have no need for these instructions right now: + -- swp, swpb, strex, ldrex, strexd, ldrexd, strexb, ldrexb, strexh, ldrexh + -- msr, nopv6, yield, wfe, wfi, sev, dbg, bxj, smc, srs, rfe + -- cps, setend, pli, pld, pldw, clrex, dsb, dmb, isb + -- stc, ldc, mcr, mcr2, mrc, mrc2, mcrr, mcrr2, mrrc, mrrc2, cdp, cdp2 +} + +-- Add mnemonics for "s" variants. +do + local t = {} + for k,v in pairs(map_op) do + if sub(v, -1) == "s" then + local v2 = sub(v, 1, 2)..char(byte(v, 3)+1)..sub(v, 4, -2) + t[sub(k, 1, -3).."s"..sub(k, -2)] = v2 + end + end + for k,v in pairs(t) do + map_op[k] = v + end +end + +------------------------------------------------------------------------------ + +local function parse_gpr(expr) + local tname, ovreg = match(expr, "^([%w_]+):(r1?[0-9])$") + local tp = map_type[tname or expr] + if tp then + local reg = ovreg or tp.reg + if not reg then + werror("type `"..(tname or expr).."' needs a register override") + end + expr = reg + end + local r = match(expr, "^r(1?[0-9])$") + if r then + r = tonumber(r) + if r <= 15 then return r, tp end + end + werror("bad register name `"..expr.."'") +end + +local function parse_gpr_pm(expr) + local pm, expr2 = match(expr, "^([+-]?)(.*)$") + return parse_gpr(expr2), (pm == "-") +end + +local function parse_vr(expr, tp) + local t, r = match(expr, "^([sd])([0-9]+)$") + if t == tp then + r = tonumber(r) + if r <= 31 then + if t == "s" then return shr(r, 1), band(r, 1) end + return band(r, 15), shr(r, 4) + end + end + werror("bad register name `"..expr.."'") +end + +local function parse_reglist(reglist) + reglist = match(reglist, "^{%s*([^}]*)}$") + if not reglist then werror("register list expected") end + local rr = 0 + for p in gmatch(reglist..",", "%s*([^,]*),") do + local rbit = shl(1, parse_gpr(gsub(p, "%s+$", ""))) + if band(rr, rbit) ~= 0 then + werror("duplicate register `"..p.."'") + end + rr = rr + rbit + end + return rr +end + +local function parse_vrlist(reglist) + local ta, ra, tb, rb = match(reglist, + "^{%s*([sd])([0-9]+)%s*%-%s*([sd])([0-9]+)%s*}$") + ra, rb = tonumber(ra), tonumber(rb) + if ta and ta == tb and ra and rb and ra <= 31 and rb <= 31 and ra <= rb then + local nr = rb+1 - ra + if ta == "s" then + return shl(shr(ra,1),12)+shl(band(ra,1),22) + nr + else + return shl(band(ra,15),12)+shl(shr(ra,4),22) + nr*2 + 0x100 + end + end + werror("register list expected") +end + +local function parse_imm(imm, bits, shift, scale, signed) + imm = match(imm, "^#(.*)$") + if not imm then werror("expected immediate operand") end + local n = tonumber(imm) + if n then + local m = sar(n, scale) + if shl(m, scale) == n then + if signed then + local s = sar(m, bits-1) + if s == 0 then return shl(m, shift) + elseif s == -1 then return shl(m + shl(1, bits), shift) end + else + if sar(m, bits) == 0 then return shl(m, shift) end + end + end + werror("out of range immediate `"..imm.."'") + else + waction("IMM", (signed and 32768 or 0)+scale*1024+bits*32+shift, imm) + return 0 + end +end + +local function parse_imm12(imm) + local n = tonumber(imm) + if n then + local m = band(n) + for i=0,-15,-1 do + if shr(m, 8) == 0 then return m + shl(band(i, 15), 8) end + m = ror(m, 2) + end + werror("out of range immediate `"..imm.."'") + else + waction("IMM12", 0, imm) + return 0 + end +end + +local function parse_imm16(imm) + imm = match(imm, "^#(.*)$") + if not imm then werror("expected immediate operand") end + local n = tonumber(imm) + if n then + if shr(n, 16) == 0 then return band(n, 0x0fff) + shl(band(n, 0xf000), 4) end + werror("out of range immediate `"..imm.."'") + else + waction("IMM16", 32*16, imm) + return 0 + end +end + +local function parse_imm_load(imm, ext) + local n = tonumber(imm) + if n then + if ext then + if n >= -255 and n <= 255 then + local up = 0x00800000 + if n < 0 then n = -n; up = 0 end + return shl(band(n, 0xf0), 4) + band(n, 0x0f) + up + end + else + if n >= -4095 and n <= 4095 then + if n >= 0 then return n+0x00800000 end + return -n + end + end + werror("out of range immediate `"..imm.."'") + else + waction(ext and "IMML8" or "IMML12", 32768 + shl(ext and 8 or 12, 5), imm) + return 0 + end +end + +local function parse_shift(shift, gprok) + if shift == "rrx" then + return 3 * 32 + else + local s, s2 = match(shift, "^(%S+)%s*(.*)$") + s = map_shift[s] + if not s then werror("expected shift operand") end + if sub(s2, 1, 1) == "#" then + return parse_imm(s2, 5, 7, 0, false) + shl(s, 5) + else + if not gprok then werror("expected immediate shift operand") end + return shl(parse_gpr(s2), 8) + shl(s, 5) + 16 + end + end +end + +local function parse_label(label, def) + local prefix = sub(label, 1, 2) + -- =>label (pc label reference) + if prefix == "=>" then + return "PC", 0, sub(label, 3) + end + -- ->name (global label reference) + if prefix == "->" then + return "LG", map_global[sub(label, 3)] + end + if def then + -- [1-9] (local label definition) + if match(label, "^[1-9]$") then + return "LG", 10+tonumber(label) + end + else + -- [<>][1-9] (local label reference) + local dir, lnum = match(label, "^([<>])([1-9])$") + if dir then -- Fwd: 1-9, Bkwd: 11-19. + return "LG", lnum + (dir == ">" and 0 or 10) + end + -- extern label (extern label reference) + local extname = match(label, "^extern%s+(%S+)$") + if extname then + return "EXT", map_extern[extname] + end + end + werror("bad label `"..label.."'") +end + +local function parse_load(params, nparams, n, op) + local oplo = band(op, 255) + local ext, ldrd = (oplo ~= 0), (oplo == 208) + local d + if (ldrd or oplo == 240) then + d = band(shr(op, 12), 15) + if band(d, 1) ~= 0 then werror("odd destination register") end + end + local pn = params[n] + local p1, wb = match(pn, "^%[%s*(.-)%s*%](!?)$") + local p2 = params[n+1] + if not p1 then + if not p2 then + if match(pn, "^[<>=%-]") or match(pn, "^extern%s+") then + local mode, n, s = parse_label(pn, false) + waction("REL_"..mode, n + (ext and 0x1800 or 0x0800), s, 1) + return op + 15 * 65536 + 0x01000000 + (ext and 0x00400000 or 0) + end + local reg, tailr = match(pn, "^([%w_:]+)%s*(.*)$") + if reg and tailr ~= "" then + local d, tp = parse_gpr(reg) + if tp then + waction(ext and "IMML8" or "IMML12", 32768 + 32*(ext and 8 or 12), + format(tp.ctypefmt, tailr)) + return op + shl(d, 16) + 0x01000000 + (ext and 0x00400000 or 0) + end + end + end + werror("expected address operand") + end + if wb == "!" then op = op + 0x00200000 end + if p2 then + if wb == "!" then werror("bad use of '!'") end + local p3 = params[n+2] + op = op + shl(parse_gpr(p1), 16) + local imm = match(p2, "^#(.*)$") + if imm then + local m = parse_imm_load(imm, ext) + if p3 then werror("too many parameters") end + op = op + m + (ext and 0x00400000 or 0) + else + local m, neg = parse_gpr_pm(p2) + if ldrd and (m == d or m-1 == d) then werror("register conflict") end + op = op + m + (neg and 0 or 0x00800000) + (ext and 0 or 0x02000000) + if p3 then op = op + parse_shift(p3) end + end + else + local p1a, p2 = match(p1, "^([^,%s]*)%s*(.*)$") + op = op + shl(parse_gpr(p1a), 16) + 0x01000000 + if p2 ~= "" then + local imm = match(p2, "^,%s*#(.*)$") + if imm then + local m = parse_imm_load(imm, ext) + op = op + m + (ext and 0x00400000 or 0) + else + local p2a, p3 = match(p2, "^,%s*([^,%s]*)%s*,?%s*(.*)$") + local m, neg = parse_gpr_pm(p2a) + if ldrd and (m == d or m-1 == d) then werror("register conflict") end + op = op + m + (neg and 0 or 0x00800000) + (ext and 0 or 0x02000000) + if p3 ~= "" then + if ext then werror("too many parameters") end + op = op + parse_shift(p3) + end + end + else + if wb == "!" then werror("bad use of '!'") end + op = op + (ext and 0x00c00000 or 0x00800000) + end + end + return op +end + +local function parse_vload(q) + local reg, imm = match(q, "^%[%s*([^,%s]*)%s*(.*)%]$") + if reg then + local d = shl(parse_gpr(reg), 16) + if imm == "" then return d end + imm = match(imm, "^,%s*#(.*)$") + if imm then + local n = tonumber(imm) + if n then + if n >= -1020 and n <= 1020 and n%4 == 0 then + return d + (n >= 0 and n/4+0x00800000 or -n/4) + end + werror("out of range immediate `"..imm.."'") + else + waction("IMMV8", 32768 + 32*8, imm) + return d + end + end + else + if match(q, "^[<>=%-]") or match(q, "^extern%s+") then + local mode, n, s = parse_label(q, false) + waction("REL_"..mode, n + 0x2800, s, 1) + return 15 * 65536 + end + local reg, tailr = match(q, "^([%w_:]+)%s*(.*)$") + if reg and tailr ~= "" then + local d, tp = parse_gpr(reg) + if tp then + waction("IMMV8", 32768 + 32*8, format(tp.ctypefmt, tailr)) + return shl(d, 16) + end + end + end + werror("expected address operand") +end + +------------------------------------------------------------------------------ + +-- Handle opcodes defined with template strings. +local function parse_template(params, template, nparams, pos) + local op = tonumber(sub(template, 1, 8), 16) + local n = 1 + local vr = "s" + + -- Process each character. + for p in gmatch(sub(template, 9), ".") do + local q = params[n] + if p == "D" then + op = op + shl(parse_gpr(q), 12); n = n + 1 + elseif p == "N" then + op = op + shl(parse_gpr(q), 16); n = n + 1 + elseif p == "S" then + op = op + shl(parse_gpr(q), 8); n = n + 1 + elseif p == "M" then + op = op + parse_gpr(q); n = n + 1 + elseif p == "d" then + local r,h = parse_vr(q, vr); op = op+shl(r,12)+shl(h,22); n = n + 1 + elseif p == "n" then + local r,h = parse_vr(q, vr); op = op+shl(r,16)+shl(h,7); n = n + 1 + elseif p == "m" then + local r,h = parse_vr(q, vr); op = op+r+shl(h,5); n = n + 1 + elseif p == "P" then + local imm = match(q, "^#(.*)$") + if imm then + op = op + parse_imm12(imm) + 0x02000000 + else + op = op + parse_gpr(q) + end + n = n + 1 + elseif p == "p" then + op = op + parse_shift(q, true); n = n + 1 + elseif p == "L" then + op = parse_load(params, nparams, n, op) + elseif p == "l" then + op = op + parse_vload(q) + elseif p == "B" then + local mode, n, s = parse_label(q, false) + waction("REL_"..mode, n, s, 1) + elseif p == "C" then -- blx gpr vs. blx label. + if match(q, "^([%w_]+):(r1?[0-9])$") or match(q, "^r(1?[0-9])$") then + op = op + parse_gpr(q) + else + if op < 0xe0000000 then werror("unconditional instruction") end + local mode, n, s = parse_label(q, false) + waction("REL_"..mode, n, s, 1) + op = 0xfa000000 + end + elseif p == "F" then + vr = "s" + elseif p == "G" then + vr = "d" + elseif p == "o" then + local r, wb = match(q, "^([^!]*)(!?)$") + op = op + shl(parse_gpr(r), 16) + (wb == "!" and 0x00200000 or 0) + n = n + 1 + elseif p == "R" then + op = op + parse_reglist(q); n = n + 1 + elseif p == "r" then + op = op + parse_vrlist(q); n = n + 1 + elseif p == "W" then + op = op + parse_imm16(q); n = n + 1 + elseif p == "v" then + op = op + parse_imm(q, 5, 7, 0, false); n = n + 1 + elseif p == "w" then + local imm = match(q, "^#(.*)$") + if imm then + op = op + parse_imm(q, 5, 7, 0, false); n = n + 1 + else + op = op + shl(parse_gpr(q), 8) + 16 + end + elseif p == "X" then + op = op + parse_imm(q, 5, 16, 0, false); n = n + 1 + elseif p == "Y" then + local imm = tonumber(match(q, "^#(.*)$")); n = n + 1 + if not imm or shr(imm, 8) ~= 0 then + werror("bad immediate operand") + end + op = op + shl(band(imm, 0xf0), 12) + band(imm, 0x0f) + elseif p == "K" then + local imm = tonumber(match(q, "^#(.*)$")); n = n + 1 + if not imm or shr(imm, 16) ~= 0 then + werror("bad immediate operand") + end + op = op + shl(band(imm, 0xfff0), 4) + band(imm, 0x000f) + elseif p == "T" then + op = op + parse_imm(q, 24, 0, 0, false); n = n + 1 + elseif p == "s" then + -- Ignored. + else + assert(false) + end + end + wputpos(pos, op) +end + +map_op[".template__"] = function(params, template, nparams) + if not params then return template:gsub("%x%x%x%x%x%x%x%x", "") end + + -- Limit number of section buffer positions used by a single dasm_put(). + -- A single opcode needs a maximum of 3 positions. + if secpos+3 > maxsecpos then wflush() end + local pos = wpos() + local lpos, apos, spos = #actlist, #actargs, secpos + + local ok, err + for t in gmatch(template, "[^|]+") do + ok, err = pcall(parse_template, params, t, nparams, pos) + if ok then return end + secpos = spos + actlist[lpos+1] = nil + actlist[lpos+2] = nil + actlist[lpos+3] = nil + actargs[apos+1] = nil + actargs[apos+2] = nil + actargs[apos+3] = nil + end + error(err, 0) +end + +------------------------------------------------------------------------------ + +-- Pseudo-opcode to mark the position where the action list is to be emitted. +map_op[".actionlist_1"] = function(params) + if not params then return "cvar" end + local name = params[1] -- No syntax check. You get to keep the pieces. + wline(function(out) writeactions(out, name) end) +end + +-- Pseudo-opcode to mark the position where the global enum is to be emitted. +map_op[".globals_1"] = function(params) + if not params then return "prefix" end + local prefix = params[1] -- No syntax check. You get to keep the pieces. + wline(function(out) writeglobals(out, prefix) end) +end + +-- Pseudo-opcode to mark the position where the global names are to be emitted. +map_op[".globalnames_1"] = function(params) + if not params then return "cvar" end + local name = params[1] -- No syntax check. You get to keep the pieces. + wline(function(out) writeglobalnames(out, name) end) +end + +-- Pseudo-opcode to mark the position where the extern names are to be emitted. +map_op[".externnames_1"] = function(params) + if not params then return "cvar" end + local name = params[1] -- No syntax check. You get to keep the pieces. + wline(function(out) writeexternnames(out, name) end) +end + +------------------------------------------------------------------------------ + +-- Label pseudo-opcode (converted from trailing colon form). +map_op[".label_1"] = function(params) + if not params then return "[1-9] | ->global | =>pcexpr" end + if secpos+1 > maxsecpos then wflush() end + local mode, n, s = parse_label(params[1], true) + if mode == "EXT" then werror("bad label definition") end + waction("LABEL_"..mode, n, s, 1) +end + +------------------------------------------------------------------------------ + +-- Pseudo-opcodes for data storage. +map_op[".long_*"] = function(params) + if not params then return "imm..." end + for _,p in ipairs(params) do + local n = tonumber(p) + if not n then werror("bad immediate `"..p.."'") end + if n < 0 then n = n + 2^32 end + wputw(n) + if secpos+2 > maxsecpos then wflush() end + end +end + +-- Alignment pseudo-opcode. +map_op[".align_1"] = function(params) + if not params then return "numpow2" end + if secpos+1 > maxsecpos then wflush() end + local align = tonumber(params[1]) + if align then + local x = align + -- Must be a power of 2 in the range (2 ... 256). + for i=1,8 do + x = x / 2 + if x == 1 then + waction("ALIGN", align-1, nil, 1) -- Action byte is 2**n-1. + return + end + end + end + werror("bad alignment") +end + +------------------------------------------------------------------------------ + +-- Pseudo-opcode for (primitive) type definitions (map to C types). +map_op[".type_3"] = function(params, nparams) + if not params then + return nparams == 2 and "name, ctype" or "name, ctype, reg" + end + local name, ctype, reg = params[1], params[2], params[3] + if not match(name, "^[%a_][%w_]*$") then + werror("bad type name `"..name.."'") + end + local tp = map_type[name] + if tp then + werror("duplicate type `"..name.."'") + end + -- Add #type to defines. A bit unclean to put it in map_archdef. + map_archdef["#"..name] = "sizeof("..ctype..")" + -- Add new type and emit shortcut define. + local num = ctypenum + 1 + map_type[name] = { + ctype = ctype, + ctypefmt = format("Dt%X(%%s)", num), + reg = reg, + } + wline(format("#define Dt%X(_V) (int)(ptrdiff_t)&(((%s *)0)_V)", num, ctype)) + ctypenum = num +end +map_op[".type_2"] = map_op[".type_3"] + +-- Dump type definitions. +local function dumptypes(out, lvl) + local t = {} + for name in pairs(map_type) do t[#t+1] = name end + sort(t) + out:write("Type definitions:\n") + for _,name in ipairs(t) do + local tp = map_type[name] + local reg = tp.reg or "" + out:write(format(" %-20s %-20s %s\n", name, tp.ctype, reg)) + end + out:write("\n") +end + +------------------------------------------------------------------------------ + +-- Set the current section. +function _M.section(num) + waction("SECTION", num) + wflush(true) -- SECTION is a terminal action. +end + +------------------------------------------------------------------------------ + +-- Dump architecture description. +function _M.dumparch(out) + out:write(format("DynASM %s version %s, released %s\n\n", + _info.arch, _info.version, _info.release)) + dumpactions(out) +end + +-- Dump all user defined elements. +function _M.dumpdef(out, lvl) + dumptypes(out, lvl) + dumpglobals(out, lvl) + dumpexterns(out, lvl) +end + +------------------------------------------------------------------------------ + +-- Pass callbacks from/to the DynASM core. +function _M.passcb(wl, we, wf, ww) + wline, werror, wfatal, wwarn = wl, we, wf, ww + return wflush +end + +-- Setup the arch-specific module. +function _M.setup(arch, opt) + g_arch, g_opt = arch, opt +end + +-- Merge the core maps and the arch-specific maps. +function _M.mergemaps(map_coreop, map_def) + setmetatable(map_op, { __index = function(t, k) + local v = map_coreop[k] + if v then return v end + local k1, cc, k2 = match(k, "^(.-)(..)([._].*)$") + local cv = map_cond[cc] + if cv then + local v = rawget(t, k1..k2) + if type(v) == "string" then + local scv = format("%x", cv) + return gsub(scv..sub(v, 2), "|e", "|"..scv) + end + end + end }) + setmetatable(map_def, { __index = map_archdef }) + return map_op, map_def +end + +return _M + +------------------------------------------------------------------------------ + diff --git a/ext/opcache/dynasm/dasm_arm64.h b/ext/opcache/dynasm/dasm_arm64.h new file mode 100644 index 0000000000000..d64e60a3e6e69 --- /dev/null +++ b/ext/opcache/dynasm/dasm_arm64.h @@ -0,0 +1,518 @@ +/* +** DynASM ARM64 encoding engine. +** Copyright (C) 2005-2016 Mike Pall. All rights reserved. +** Released under the MIT license. See dynasm.lua for full copyright notice. +*/ + +#include +#include +#include +#include + +#define DASM_ARCH "arm64" + +#ifndef DASM_EXTERN +#define DASM_EXTERN(a,b,c,d) 0 +#endif + +/* Action definitions. */ +enum { + DASM_STOP, DASM_SECTION, DASM_ESC, DASM_REL_EXT, + /* The following actions need a buffer position. */ + DASM_ALIGN, DASM_REL_LG, DASM_LABEL_LG, + /* The following actions also have an argument. */ + DASM_REL_PC, DASM_LABEL_PC, + DASM_IMM, DASM_IMM6, DASM_IMM12, DASM_IMM13W, DASM_IMM13X, DASM_IMML, + DASM__MAX +}; + +/* Maximum number of section buffer positions for a single dasm_put() call. */ +#define DASM_MAXSECPOS 25 + +/* DynASM encoder status codes. Action list offset or number are or'ed in. */ +#define DASM_S_OK 0x00000000 +#define DASM_S_NOMEM 0x01000000 +#define DASM_S_PHASE 0x02000000 +#define DASM_S_MATCH_SEC 0x03000000 +#define DASM_S_RANGE_I 0x11000000 +#define DASM_S_RANGE_SEC 0x12000000 +#define DASM_S_RANGE_LG 0x13000000 +#define DASM_S_RANGE_PC 0x14000000 +#define DASM_S_RANGE_REL 0x15000000 +#define DASM_S_UNDEF_LG 0x21000000 +#define DASM_S_UNDEF_PC 0x22000000 + +/* Macros to convert positions (8 bit section + 24 bit index). */ +#define DASM_POS2IDX(pos) ((pos)&0x00ffffff) +#define DASM_POS2BIAS(pos) ((pos)&0xff000000) +#define DASM_SEC2POS(sec) ((sec)<<24) +#define DASM_POS2SEC(pos) ((pos)>>24) +#define DASM_POS2PTR(D, pos) (D->sections[DASM_POS2SEC(pos)].rbuf + (pos)) + +/* Action list type. */ +typedef const unsigned int *dasm_ActList; + +/* Per-section structure. */ +typedef struct dasm_Section { + int *rbuf; /* Biased buffer pointer (negative section bias). */ + int *buf; /* True buffer pointer. */ + size_t bsize; /* Buffer size in bytes. */ + int pos; /* Biased buffer position. */ + int epos; /* End of biased buffer position - max single put. */ + int ofs; /* Byte offset into section. */ +} dasm_Section; + +/* Core structure holding the DynASM encoding state. */ +struct dasm_State { + size_t psize; /* Allocated size of this structure. */ + dasm_ActList actionlist; /* Current actionlist pointer. */ + int *lglabels; /* Local/global chain/pos ptrs. */ + size_t lgsize; + int *pclabels; /* PC label chains/pos ptrs. */ + size_t pcsize; + void **globals; /* Array of globals (bias -10). */ + dasm_Section *section; /* Pointer to active section. */ + size_t codesize; /* Total size of all code sections. */ + int maxsection; /* 0 <= sectionidx < maxsection. */ + int status; /* Status code. */ + dasm_Section sections[1]; /* All sections. Alloc-extended. */ +}; + +/* The size of the core structure depends on the max. number of sections. */ +#define DASM_PSZ(ms) (sizeof(dasm_State)+(ms-1)*sizeof(dasm_Section)) + + +/* Initialize DynASM state. */ +void dasm_init(Dst_DECL, int maxsection) +{ + dasm_State *D; + size_t psz = 0; + int i; + Dst_REF = NULL; + DASM_M_GROW(Dst, struct dasm_State, Dst_REF, psz, DASM_PSZ(maxsection)); + D = Dst_REF; + D->psize = psz; + D->lglabels = NULL; + D->lgsize = 0; + D->pclabels = NULL; + D->pcsize = 0; + D->globals = NULL; + D->maxsection = maxsection; + for (i = 0; i < maxsection; i++) { + D->sections[i].buf = NULL; /* Need this for pass3. */ + D->sections[i].rbuf = D->sections[i].buf - DASM_SEC2POS(i); + D->sections[i].bsize = 0; + D->sections[i].epos = 0; /* Wrong, but is recalculated after resize. */ + } +} + +/* Free DynASM state. */ +void dasm_free(Dst_DECL) +{ + dasm_State *D = Dst_REF; + int i; + for (i = 0; i < D->maxsection; i++) + if (D->sections[i].buf) + DASM_M_FREE(Dst, D->sections[i].buf, D->sections[i].bsize); + if (D->pclabels) DASM_M_FREE(Dst, D->pclabels, D->pcsize); + if (D->lglabels) DASM_M_FREE(Dst, D->lglabels, D->lgsize); + DASM_M_FREE(Dst, D, D->psize); +} + +/* Setup global label array. Must be called before dasm_setup(). */ +void dasm_setupglobal(Dst_DECL, void **gl, unsigned int maxgl) +{ + dasm_State *D = Dst_REF; + D->globals = gl - 10; /* Negative bias to compensate for locals. */ + DASM_M_GROW(Dst, int, D->lglabels, D->lgsize, (10+maxgl)*sizeof(int)); +} + +/* Grow PC label array. Can be called after dasm_setup(), too. */ +void dasm_growpc(Dst_DECL, unsigned int maxpc) +{ + dasm_State *D = Dst_REF; + size_t osz = D->pcsize; + DASM_M_GROW(Dst, int, D->pclabels, D->pcsize, maxpc*sizeof(int)); + memset((void *)(((unsigned char *)D->pclabels)+osz), 0, D->pcsize-osz); +} + +/* Setup encoder. */ +void dasm_setup(Dst_DECL, const void *actionlist) +{ + dasm_State *D = Dst_REF; + int i; + D->actionlist = (dasm_ActList)actionlist; + D->status = DASM_S_OK; + D->section = &D->sections[0]; + memset((void *)D->lglabels, 0, D->lgsize); + if (D->pclabels) memset((void *)D->pclabels, 0, D->pcsize); + for (i = 0; i < D->maxsection; i++) { + D->sections[i].pos = DASM_SEC2POS(i); + D->sections[i].ofs = 0; + } +} + + +#ifdef DASM_CHECKS +#define CK(x, st) \ + do { if (!(x)) { \ + D->status = DASM_S_##st|(p-D->actionlist-1); return; } } while (0) +#define CKPL(kind, st) \ + do { if ((size_t)((char *)pl-(char *)D->kind##labels) >= D->kind##size) { \ + D->status = DASM_S_RANGE_##st|(p-D->actionlist-1); return; } } while (0) +#else +#define CK(x, st) ((void)0) +#define CKPL(kind, st) ((void)0) +#endif + +static int dasm_imm12(unsigned int n) +{ + if ((n >> 12) == 0) + return n; + else if ((n & 0xff000fff) == 0) + return (n >> 12) | 0x1000; + else + return -1; +} + +static int dasm_ffs(unsigned long long x) +{ + int n = -1; + while (x) { x >>= 1; n++; } + return n; +} + +static int dasm_imm13(int lo, int hi) +{ + int inv = 0, w = 64, s = 0xfff, xa, xb; + unsigned long long n = (((unsigned long long)hi) << 32) | (unsigned int)lo; + unsigned long long m = 1ULL, a, b, c; + if (n & 1) { n = ~n; inv = 1; } + a = n & -n; b = (n+a)&-(n+a); c = (n+a-b)&-(n+a-b); + xa = dasm_ffs(a); xb = dasm_ffs(b); + if (c) { + w = dasm_ffs(c) - xa; + if (w == 32) m = 0x0000000100000001UL; + else if (w == 16) m = 0x0001000100010001UL; + else if (w == 8) m = 0x0101010101010101UL; + else if (w == 4) m = 0x1111111111111111UL; + else if (w == 2) m = 0x5555555555555555UL; + else return -1; + s = (-2*w & 0x3f) - 1; + } else if (!a) { + return -1; + } else if (xb == -1) { + xb = 64; + } + if ((b-a) * m != n) return -1; + if (inv) { + return ((w - xb) << 6) | (s+w+xa-xb); + } else { + return ((w - xa) << 6) | (s+xb-xa); + } + return -1; +} + +/* Pass 1: Store actions and args, link branches/labels, estimate offsets. */ +void dasm_put(Dst_DECL, int start, ...) +{ + va_list ap; + dasm_State *D = Dst_REF; + dasm_ActList p = D->actionlist + start; + dasm_Section *sec = D->section; + int pos = sec->pos, ofs = sec->ofs; + int *b; + + if (pos >= sec->epos) { + DASM_M_GROW(Dst, int, sec->buf, sec->bsize, + sec->bsize + 2*DASM_MAXSECPOS*sizeof(int)); + sec->rbuf = sec->buf - DASM_POS2BIAS(pos); + sec->epos = (int)sec->bsize/sizeof(int) - DASM_MAXSECPOS+DASM_POS2BIAS(pos); + } + + b = sec->rbuf; + b[pos++] = start; + + va_start(ap, start); + while (1) { + unsigned int ins = *p++; + unsigned int action = (ins >> 16); + if (action >= DASM__MAX) { + ofs += 4; + } else { + int *pl, n = action >= DASM_REL_PC ? va_arg(ap, int) : 0; + switch (action) { + case DASM_STOP: goto stop; + case DASM_SECTION: + n = (ins & 255); CK(n < D->maxsection, RANGE_SEC); + D->section = &D->sections[n]; goto stop; + case DASM_ESC: p++; ofs += 4; break; + case DASM_REL_EXT: break; + case DASM_ALIGN: ofs += (ins & 255); b[pos++] = ofs; break; + case DASM_REL_LG: + n = (ins & 2047) - 10; pl = D->lglabels + n; + /* Bkwd rel or global. */ + if (n >= 0) { CK(n>=10||*pl<0, RANGE_LG); CKPL(lg, LG); goto putrel; } + pl += 10; n = *pl; + if (n < 0) n = 0; /* Start new chain for fwd rel if label exists. */ + goto linkrel; + case DASM_REL_PC: + pl = D->pclabels + n; CKPL(pc, PC); + putrel: + n = *pl; + if (n < 0) { /* Label exists. Get label pos and store it. */ + b[pos] = -n; + } else { + linkrel: + b[pos] = n; /* Else link to rel chain, anchored at label. */ + *pl = pos; + } + pos++; + break; + case DASM_LABEL_LG: + pl = D->lglabels + (ins & 2047) - 10; CKPL(lg, LG); goto putlabel; + case DASM_LABEL_PC: + pl = D->pclabels + n; CKPL(pc, PC); + putlabel: + n = *pl; /* n > 0: Collapse rel chain and replace with label pos. */ + while (n > 0) { int *pb = DASM_POS2PTR(D, n); n = *pb; *pb = pos; + } + *pl = -pos; /* Label exists now. */ + b[pos++] = ofs; /* Store pass1 offset estimate. */ + break; + case DASM_IMM: + CK((n & ((1<<((ins>>10)&31))-1)) == 0, RANGE_I); + n >>= ((ins>>10)&31); +#ifdef DASM_CHECKS + if ((ins & 0x8000)) + CK(((n + (1<<(((ins>>5)&31)-1)))>>((ins>>5)&31)) == 0, RANGE_I); + else + CK((n>>((ins>>5)&31)) == 0, RANGE_I); +#endif + b[pos++] = n; + break; + case DASM_IMM6: + CK((n >> 6) == 0, RANGE_I); + b[pos++] = n; + break; + case DASM_IMM12: + CK(dasm_imm12((unsigned int)n) != -1, RANGE_I); + b[pos++] = n; + break; + case DASM_IMM13W: + CK(dasm_imm13(n, n) != -1, RANGE_I); + b[pos++] = n; + break; + case DASM_IMM13X: { + int m = va_arg(ap, int); + CK(dasm_imm13(n, m) != -1, RANGE_I); + b[pos++] = n; + b[pos++] = m; + break; + } + case DASM_IMML: { +#ifdef DASM_CHECKS + int scale = (p[-2] >> 30); + CK((!(n & ((1<>scale) < 4096) || + (unsigned int)(n+256) < 512, RANGE_I); +#endif + b[pos++] = n; + break; + } + } + } + } +stop: + va_end(ap); + sec->pos = pos; + sec->ofs = ofs; +} +#undef CK + +/* Pass 2: Link sections, shrink aligns, fix label offsets. */ +int dasm_link(Dst_DECL, size_t *szp) +{ + dasm_State *D = Dst_REF; + int secnum; + int ofs = 0; + +#ifdef DASM_CHECKS + *szp = 0; + if (D->status != DASM_S_OK) return D->status; + { + int pc; + for (pc = 0; pc*sizeof(int) < D->pcsize; pc++) + if (D->pclabels[pc] > 0) return DASM_S_UNDEF_PC|pc; + } +#endif + + { /* Handle globals not defined in this translation unit. */ + int idx; + for (idx = 20; idx*sizeof(int) < D->lgsize; idx++) { + int n = D->lglabels[idx]; + /* Undefined label: Collapse rel chain and replace with marker (< 0). */ + while (n > 0) { int *pb = DASM_POS2PTR(D, n); n = *pb; *pb = -idx; } + } + } + + /* Combine all code sections. No support for data sections (yet). */ + for (secnum = 0; secnum < D->maxsection; secnum++) { + dasm_Section *sec = D->sections + secnum; + int *b = sec->rbuf; + int pos = DASM_SEC2POS(secnum); + int lastpos = sec->pos; + + while (pos != lastpos) { + dasm_ActList p = D->actionlist + b[pos++]; + while (1) { + unsigned int ins = *p++; + unsigned int action = (ins >> 16); + switch (action) { + case DASM_STOP: case DASM_SECTION: goto stop; + case DASM_ESC: p++; break; + case DASM_REL_EXT: break; + case DASM_ALIGN: ofs -= (b[pos++] + ofs) & (ins & 255); break; + case DASM_REL_LG: case DASM_REL_PC: pos++; break; + case DASM_LABEL_LG: case DASM_LABEL_PC: b[pos++] += ofs; break; + case DASM_IMM: case DASM_IMM6: case DASM_IMM12: case DASM_IMM13W: + case DASM_IMML: pos++; break; + case DASM_IMM13X: pos += 2; break; + } + } + stop: (void)0; + } + ofs += sec->ofs; /* Next section starts right after current section. */ + } + + D->codesize = ofs; /* Total size of all code sections */ + *szp = ofs; + return DASM_S_OK; +} + +#ifdef DASM_CHECKS +#define CK(x, st) \ + do { if (!(x)) return DASM_S_##st|(p-D->actionlist-1); } while (0) +#else +#define CK(x, st) ((void)0) +#endif + +/* Pass 3: Encode sections. */ +int dasm_encode(Dst_DECL, void *buffer) +{ + dasm_State *D = Dst_REF; + char *base = (char *)buffer; + unsigned int *cp = (unsigned int *)buffer; + int secnum; + + /* Encode all code sections. No support for data sections (yet). */ + for (secnum = 0; secnum < D->maxsection; secnum++) { + dasm_Section *sec = D->sections + secnum; + int *b = sec->buf; + int *endb = sec->rbuf + sec->pos; + + while (b != endb) { + dasm_ActList p = D->actionlist + *b++; + while (1) { + unsigned int ins = *p++; + unsigned int action = (ins >> 16); + int n = (action >= DASM_ALIGN && action < DASM__MAX) ? *b++ : 0; + switch (action) { + case DASM_STOP: case DASM_SECTION: goto stop; + case DASM_ESC: *cp++ = *p++; break; + case DASM_REL_EXT: + n = DASM_EXTERN(Dst, (unsigned char *)cp, (ins&2047), !(ins&2048)); + goto patchrel; + case DASM_ALIGN: + ins &= 255; while ((((char *)cp - base) & ins)) *cp++ = 0xe1a00000; + break; + case DASM_REL_LG: + CK(n >= 0, UNDEF_LG); + case DASM_REL_PC: + CK(n >= 0, UNDEF_PC); + n = *DASM_POS2PTR(D, n) - (int)((char *)cp - base) + 4; + patchrel: + if (!(ins & 0xf800)) { /* B, BL */ + CK((n & 3) == 0 && ((n+0x08000000) >> 28) == 0, RANGE_REL); + cp[-1] |= ((n >> 2) & 0x03ffffff); + } else if ((ins & 0x800)) { /* B.cond, CBZ, CBNZ, LDR* literal */ + CK((n & 3) == 0 && ((n+0x00100000) >> 21) == 0, RANGE_REL); + cp[-1] |= ((n << 3) & 0x00ffffe0); + } else if ((ins & 0x3000) == 0x2000) { /* ADR */ + CK(((n+0x00100000) >> 21) == 0, RANGE_REL); + cp[-1] |= ((n << 3) & 0x00ffffe0) | ((n & 3) << 29); + } else if ((ins & 0x3000) == 0x3000) { /* ADRP */ + cp[-1] |= ((n >> 9) & 0x00ffffe0) | (((n >> 12) & 3) << 29); + } else if ((ins & 0x1000)) { /* TBZ, TBNZ */ + CK((n & 3) == 0 && ((n+0x00008000) >> 16) == 0, RANGE_REL); + cp[-1] |= ((n << 3) & 0x0007ffe0); + } + break; + case DASM_LABEL_LG: + ins &= 2047; if (ins >= 20) D->globals[ins-10] = (void *)(base + n); + break; + case DASM_LABEL_PC: break; + case DASM_IMM: + cp[-1] |= (n & ((1<<((ins>>5)&31))-1)) << (ins&31); + break; + case DASM_IMM6: + cp[-1] |= ((n&31) << 19) | ((n&32) << 26); + break; + case DASM_IMM12: + cp[-1] |= (dasm_imm12((unsigned int)n) << 10); + break; + case DASM_IMM13W: + cp[-1] |= (dasm_imm13(n, n) << 10); + break; + case DASM_IMM13X: + cp[-1] |= (dasm_imm13(n, *b++) << 10); + break; + case DASM_IMML: { + int scale = (p[-2] >> 30); + cp[-1] |= (!(n & ((1<>scale) < 4096) ? + ((n << (10-scale)) | 0x01000000) : ((n & 511) << 12); + break; + } + default: *cp++ = ins; break; + } + } + stop: (void)0; + } + } + + if (base + D->codesize != (char *)cp) /* Check for phase errors. */ + return DASM_S_PHASE; + return DASM_S_OK; +} +#undef CK + +/* Get PC label offset. */ +int dasm_getpclabel(Dst_DECL, unsigned int pc) +{ + dasm_State *D = Dst_REF; + if (pc*sizeof(int) < D->pcsize) { + int pos = D->pclabels[pc]; + if (pos < 0) return *DASM_POS2PTR(D, -pos); + if (pos > 0) return -1; /* Undefined. */ + } + return -2; /* Unused or out of range. */ +} + +#ifdef DASM_CHECKS +/* Optional sanity checker to call between isolated encoding steps. */ +int dasm_checkstep(Dst_DECL, int secmatch) +{ + dasm_State *D = Dst_REF; + if (D->status == DASM_S_OK) { + int i; + for (i = 1; i <= 9; i++) { + if (D->lglabels[i] > 0) { D->status = DASM_S_UNDEF_LG|i; break; } + D->lglabels[i] = 0; + } + } + if (D->status == DASM_S_OK && secmatch >= 0 && + D->section != &D->sections[secmatch]) + D->status = DASM_S_MATCH_SEC|(D->section-D->sections); + return D->status; +} +#endif + diff --git a/ext/opcache/dynasm/dasm_arm64.lua b/ext/opcache/dynasm/dasm_arm64.lua new file mode 100644 index 0000000000000..4a7d8dfeeb54c --- /dev/null +++ b/ext/opcache/dynasm/dasm_arm64.lua @@ -0,0 +1,1166 @@ +------------------------------------------------------------------------------ +-- DynASM ARM64 module. +-- +-- Copyright (C) 2005-2016 Mike Pall. All rights reserved. +-- See dynasm.lua for full copyright notice. +------------------------------------------------------------------------------ + +-- Module information: +local _info = { + arch = "arm", + description = "DynASM ARM64 module", + version = "1.4.0", + vernum = 10400, + release = "2015-10-18", + author = "Mike Pall", + license = "MIT", +} + +-- Exported glue functions for the arch-specific module. +local _M = { _info = _info } + +-- Cache library functions. +local type, tonumber, pairs, ipairs = type, tonumber, pairs, ipairs +local assert, setmetatable, rawget = assert, setmetatable, rawget +local _s = string +local sub, format, byte, char = _s.sub, _s.format, _s.byte, _s.char +local match, gmatch, gsub = _s.match, _s.gmatch, _s.gsub +local concat, sort, insert = table.concat, table.sort, table.insert +local bit = bit or require("bit") +local band, shl, shr, sar = bit.band, bit.lshift, bit.rshift, bit.arshift +local ror, tohex = bit.ror, bit.tohex + +-- Inherited tables and callbacks. +local g_opt, g_arch +local wline, werror, wfatal, wwarn + +-- Action name list. +-- CHECK: Keep this in sync with the C code! +local action_names = { + "STOP", "SECTION", "ESC", "REL_EXT", + "ALIGN", "REL_LG", "LABEL_LG", + "REL_PC", "LABEL_PC", "IMM", "IMM6", "IMM12", "IMM13W", "IMM13X", "IMML", +} + +-- Maximum number of section buffer positions for dasm_put(). +-- CHECK: Keep this in sync with the C code! +local maxsecpos = 25 -- Keep this low, to avoid excessively long C lines. + +-- Action name -> action number. +local map_action = {} +for n,name in ipairs(action_names) do + map_action[name] = n-1 +end + +-- Action list buffer. +local actlist = {} + +-- Argument list for next dasm_put(). Start with offset 0 into action list. +local actargs = { 0 } + +-- Current number of section buffer positions for dasm_put(). +local secpos = 1 + +------------------------------------------------------------------------------ + +-- Dump action names and numbers. +local function dumpactions(out) + out:write("DynASM encoding engine action codes:\n") + for n,name in ipairs(action_names) do + local num = map_action[name] + out:write(format(" %-10s %02X %d\n", name, num, num)) + end + out:write("\n") +end + +-- Write action list buffer as a huge static C array. +local function writeactions(out, name) + local nn = #actlist + if nn == 0 then nn = 1; actlist[0] = map_action.STOP end + out:write("static const unsigned int ", name, "[", nn, "] = {\n") + for i = 1,nn-1 do + assert(out:write("0x", tohex(actlist[i]), ",\n")) + end + assert(out:write("0x", tohex(actlist[nn]), "\n};\n\n")) +end + +------------------------------------------------------------------------------ + +-- Add word to action list. +local function wputxw(n) + assert(n >= 0 and n <= 0xffffffff and n % 1 == 0, "word out of range") + actlist[#actlist+1] = n +end + +-- Add action to list with optional arg. Advance buffer pos, too. +local function waction(action, val, a, num) + local w = assert(map_action[action], "bad action name `"..action.."'") + wputxw(w * 0x10000 + (val or 0)) + if a then actargs[#actargs+1] = a end + if a or num then secpos = secpos + (num or 1) end +end + +-- Flush action list (intervening C code or buffer pos overflow). +local function wflush(term) + if #actlist == actargs[1] then return end -- Nothing to flush. + if not term then waction("STOP") end -- Terminate action list. + wline(format("dasm_put(Dst, %s);", concat(actargs, ", ")), true) + actargs = { #actlist } -- Actionlist offset is 1st arg to next dasm_put(). + secpos = 1 -- The actionlist offset occupies a buffer position, too. +end + +-- Put escaped word. +local function wputw(n) + if n <= 0x000fffff then waction("ESC") end + wputxw(n) +end + +-- Reserve position for word. +local function wpos() + local pos = #actlist+1 + actlist[pos] = "" + return pos +end + +-- Store word to reserved position. +local function wputpos(pos, n) + assert(n >= 0 and n <= 0xffffffff and n % 1 == 0, "word out of range") + if n <= 0x000fffff then + insert(actlist, pos+1, n) + n = map_action.ESC * 0x10000 + end + actlist[pos] = n +end + +------------------------------------------------------------------------------ + +-- Global label name -> global label number. With auto assignment on 1st use. +local next_global = 20 +local map_global = setmetatable({}, { __index = function(t, name) + if not match(name, "^[%a_][%w_]*$") then werror("bad global label") end + local n = next_global + if n > 2047 then werror("too many global labels") end + next_global = n + 1 + t[name] = n + return n +end}) + +-- Dump global labels. +local function dumpglobals(out, lvl) + local t = {} + for name, n in pairs(map_global) do t[n] = name end + out:write("Global labels:\n") + for i=20,next_global-1 do + out:write(format(" %s\n", t[i])) + end + out:write("\n") +end + +-- Write global label enum. +local function writeglobals(out, prefix) + local t = {} + for name, n in pairs(map_global) do t[n] = name end + out:write("enum {\n") + for i=20,next_global-1 do + out:write(" ", prefix, t[i], ",\n") + end + out:write(" ", prefix, "_MAX\n};\n") +end + +-- Write global label names. +local function writeglobalnames(out, name) + local t = {} + for name, n in pairs(map_global) do t[n] = name end + out:write("static const char *const ", name, "[] = {\n") + for i=20,next_global-1 do + out:write(" \"", t[i], "\",\n") + end + out:write(" (const char *)0\n};\n") +end + +------------------------------------------------------------------------------ + +-- Extern label name -> extern label number. With auto assignment on 1st use. +local next_extern = 0 +local map_extern_ = {} +local map_extern = setmetatable({}, { __index = function(t, name) + -- No restrictions on the name for now. + local n = next_extern + if n > 2047 then werror("too many extern labels") end + next_extern = n + 1 + t[name] = n + map_extern_[n] = name + return n +end}) + +-- Dump extern labels. +local function dumpexterns(out, lvl) + out:write("Extern labels:\n") + for i=0,next_extern-1 do + out:write(format(" %s\n", map_extern_[i])) + end + out:write("\n") +end + +-- Write extern label names. +local function writeexternnames(out, name) + out:write("static const char *const ", name, "[] = {\n") + for i=0,next_extern-1 do + out:write(" \"", map_extern_[i], "\",\n") + end + out:write(" (const char *)0\n};\n") +end + +------------------------------------------------------------------------------ + +-- Arch-specific maps. + +-- Ext. register name -> int. name. +local map_archdef = { xzr = "@x31", wzr = "@w31", lr = "x30", } + +-- Int. register name -> ext. name. +local map_reg_rev = { ["@x31"] = "xzr", ["@w31"] = "wzr", x30 = "lr", } + +local map_type = {} -- Type name -> { ctype, reg } +local ctypenum = 0 -- Type number (for Dt... macros). + +-- Reverse defines for registers. +function _M.revdef(s) + return map_reg_rev[s] or s +end + +local map_shift = { lsl = 0, lsr = 1, asr = 2, } + +local map_extend = { + uxtb = 0, uxth = 1, uxtw = 2, uxtx = 3, + sxtb = 4, sxth = 5, sxtw = 6, sxtx = 7, +} + +local map_cond = { + eq = 0, ne = 1, cs = 2, cc = 3, mi = 4, pl = 5, vs = 6, vc = 7, + hi = 8, ls = 9, ge = 10, lt = 11, gt = 12, le = 13, al = 14, + hs = 2, lo = 3, +} + +------------------------------------------------------------------------------ + +local parse_reg_type + +local function parse_reg(expr) + if not expr then werror("expected register name") end + local tname, ovreg = match(expr, "^([%w_]+):(@?%l%d+)$") + local tp = map_type[tname or expr] + if tp then + local reg = ovreg or tp.reg + if not reg then + werror("type `"..(tname or expr).."' needs a register override") + end + expr = reg + end + local ok31, rt, r = match(expr, "^(@?)([xwqdshb])([123]?[0-9])$") + if r then + r = tonumber(r) + if r <= 30 or (r == 31 and ok31 ~= "" or (rt ~= "w" and rt ~= "x")) then + if not parse_reg_type then + parse_reg_type = rt + elseif parse_reg_type ~= rt then + werror("register size mismatch") + end + return r, tp + end + end + werror("bad register name `"..expr.."'") +end + +local function parse_reg_base(expr) + if expr == "sp" then return 0x3e0 end + local base, tp = parse_reg(expr) + if parse_reg_type ~= "x" then werror("bad register type") end + parse_reg_type = false + return shl(base, 5), tp +end + +local parse_ctx = {} + +local loadenv = setfenv and function(s) + local code = loadstring(s, "") + if code then setfenv(code, parse_ctx) end + return code +end or function(s) + return load(s, "", nil, parse_ctx) +end + +-- Try to parse simple arithmetic, too, since some basic ops are aliases. +local function parse_number(n) + local x = tonumber(n) + if x then return x end + local code = loadenv("return "..n) + if code then + local ok, y = pcall(code) + if ok then return y end + end + return nil +end + +local function parse_imm(imm, bits, shift, scale, signed) + imm = match(imm, "^#(.*)$") + if not imm then werror("expected immediate operand") end + local n = parse_number(imm) + if n then + local m = sar(n, scale) + if shl(m, scale) == n then + if signed then + local s = sar(m, bits-1) + if s == 0 then return shl(m, shift) + elseif s == -1 then return shl(m + shl(1, bits), shift) end + else + if sar(m, bits) == 0 then return shl(m, shift) end + end + end + werror("out of range immediate `"..imm.."'") + else + waction("IMM", (signed and 32768 or 0)+scale*1024+bits*32+shift, imm) + return 0 + end +end + +local function parse_imm12(imm) + imm = match(imm, "^#(.*)$") + if not imm then werror("expected immediate operand") end + local n = parse_number(imm) + if n then + if shr(n, 12) == 0 then + return shl(n, 10) + elseif band(n, 0xff000fff) == 0 then + return shr(n, 2) + 0x00400000 + end + werror("out of range immediate `"..imm.."'") + else + waction("IMM12", 0, imm) + return 0 + end +end + +local function parse_imm13(imm) + imm = match(imm, "^#(.*)$") + if not imm then werror("expected immediate operand") end + local n = parse_number(imm) + local r64 = parse_reg_type == "x" + if n and n % 1 == 0 and n >= 0 and n <= 0xffffffff then + local inv = false + if band(n, 1) == 1 then n = bit.bnot(n); inv = true end + local t = {} + for i=1,32 do t[i] = band(n, 1); n = shr(n, 1) end + local b = table.concat(t) + b = b..(r64 and (inv and "1" or "0"):rep(32) or b) + local p0, p1, p0a, p1a = b:match("^(0+)(1+)(0*)(1*)") + if p0 then + local w = p1a == "" and (r64 and 64 or 32) or #p1+#p0a + if band(w, w-1) == 0 and b == b:sub(1, w):rep(64/w) then + local s = band(-2*w, 0x3f) - 1 + if w == 64 then s = s + 0x1000 end + if inv then + return shl(w-#p1-#p0, 16) + shl(s+w-#p1, 10) + else + return shl(w-#p0, 16) + shl(s+#p1, 10) + end + end + end + werror("out of range immediate `"..imm.."'") + elseif r64 then + waction("IMM13X", 0, format("(unsigned int)(%s)", imm)) + actargs[#actargs+1] = format("(unsigned int)((unsigned long long)(%s)>>32)", imm) + return 0 + else + waction("IMM13W", 0, imm) + return 0 + end +end + +local function parse_imm6(imm) + imm = match(imm, "^#(.*)$") + if not imm then werror("expected immediate operand") end + local n = parse_number(imm) + if n then + if n >= 0 and n <= 63 then + return shl(band(n, 0x1f), 19) + (n >= 32 and 0x80000000 or 0) + end + werror("out of range immediate `"..imm.."'") + else + waction("IMM6", 0, imm) + return 0 + end +end + +local function parse_imm_load(imm, scale) + local n = parse_number(imm) + if n then + local m = sar(n, scale) + if shl(m, scale) == n and m >= 0 and m < 0x1000 then + return shl(m, 10) + 0x01000000 -- Scaled, unsigned 12 bit offset. + elseif n >= -256 and n < 256 then + return shl(band(n, 511), 12) -- Unscaled, signed 9 bit offset. + end + werror("out of range immediate `"..imm.."'") + else + waction("IMML", 0, imm) + return 0 + end +end + +local function parse_fpimm(imm) + imm = match(imm, "^#(.*)$") + if not imm then werror("expected immediate operand") end + local n = parse_number(imm) + if n then + local m, e = math.frexp(n) + local s, e2 = 0, band(e-2, 7) + if m < 0 then m = -m; s = 0x00100000 end + m = m*32-16 + if m % 1 == 0 and m >= 0 and m <= 15 and sar(shl(e2, 29), 29)+2 == e then + return s + shl(e2, 17) + shl(m, 13) + end + werror("out of range immediate `"..imm.."'") + else + werror("NYI fpimm action") + end +end + +local function parse_shift(expr) + local s, s2 = match(expr, "^(%S+)%s*(.*)$") + s = map_shift[s] + if not s then werror("expected shift operand") end + return parse_imm(s2, 6, 10, 0, false) + shl(s, 22) +end + +local function parse_lslx16(expr) + local n = match(expr, "^lsl%s*#(%d+)$") + n = tonumber(n) + if not n then werror("expected shift operand") end + if band(n, parse_reg_type == "x" and 0xffffffcf or 0xffffffef) ~= 0 then + werror("bad shift amount") + end + return shl(n, 17) +end + +local function parse_extend(expr) + local s, s2 = match(expr, "^(%S+)%s*(.*)$") + if s == "lsl" then + s = parse_reg_type == "x" and 3 or 2 + else + s = map_extend[s] + end + if not s then werror("expected extend operand") end + return (s2 == "" and 0 or parse_imm(s2, 3, 10, 0, false)) + shl(s, 13) +end + +local function parse_cond(expr, inv) + local c = map_cond[expr] + if not c then werror("expected condition operand") end + return shl(bit.bxor(c, inv), 12) +end + +local function parse_load(params, nparams, n, op) + if params[n+2] then werror("too many operands") end + local pn, p2 = params[n], params[n+1] + local p1, wb = match(pn, "^%[%s*(.-)%s*%](!?)$") + if not p1 then + if not p2 then + local reg, tailr = match(pn, "^([%w_:]+)%s*(.*)$") + if reg and tailr ~= "" then + local base, tp = parse_reg_base(reg) + if tp then + waction("IMML", 0, format(tp.ctypefmt, tailr)) + return op + base + end + end + end + werror("expected address operand") + end + local scale = shr(op, 30) + if p2 then + if wb == "!" then werror("bad use of '!'") end + op = op + parse_reg_base(p1) + parse_imm(p2, 9, 12, 0, true) + 0x400 + elseif wb == "!" then + local p1a, p2a = match(p1, "^([^,%s]*)%s*,%s*(.*)$") + if not p1a then werror("bad use of '!'") end + op = op + parse_reg_base(p1a) + parse_imm(p2a, 9, 12, 0, true) + 0xc00 + else + local p1a, p2a = match(p1, "^([^,%s]*)%s*(.*)$") + op = op + parse_reg_base(p1a) + if p2a ~= "" then + local imm = match(p2a, "^,%s*#(.*)$") + if imm then + op = op + parse_imm_load(imm, scale) + else + local p2b, p3b, p3s = match(p2a, "^,%s*([^,%s]*)%s*,?%s*(%S*)%s*(.*)$") + op = op + shl(parse_reg(p2b), 16) + 0x00200800 + if parse_reg_type ~= "x" and parse_reg_type ~= "w" then + werror("bad index register type") + end + if p3b == "" then + if parse_reg_type ~= "x" then werror("bad index register type") end + op = op + 0x6000 + else + if p3s == "" or p3s == "#0" then + elseif p3s == "#"..scale then + op = op + 0x1000 + else + werror("bad scale") + end + if parse_reg_type == "x" then + if p3b == "lsl" and p3s ~= "" then op = op + 0x6000 + elseif p3b == "sxtx" then op = op + 0xe000 + else + werror("bad extend/shift specifier") + end + else + if p3b == "uxtw" then op = op + 0x4000 + elseif p3b == "sxtw" then op = op + 0xc000 + else + werror("bad extend/shift specifier") + end + end + end + end + else + if wb == "!" then werror("bad use of '!'") end + op = op + 0x01000000 + end + end + return op +end + +local function parse_load_pair(params, nparams, n, op) + if params[n+2] then werror("too many operands") end + local pn, p2 = params[n], params[n+1] + local scale = shr(op, 30) == 0 and 2 or 3 + local p1, wb = match(pn, "^%[%s*(.-)%s*%](!?)$") + if not p1 then + if not p2 then + local reg, tailr = match(pn, "^([%w_:]+)%s*(.*)$") + if reg and tailr ~= "" then + local base, tp = parse_reg_base(reg) + if tp then + waction("IMM", 32768+7*32+15+scale*1024, format(tp.ctypefmt, tailr)) + return op + base + 0x01000000 + end + end + end + werror("expected address operand") + end + if p2 then + if wb == "!" then werror("bad use of '!'") end + op = op + 0x00800000 + else + local p1a, p2a = match(p1, "^([^,%s]*)%s*,%s*(.*)$") + if p1a then p1, p2 = p1a, p2a else p2 = "#0" end + op = op + (wb == "!" and 0x01800000 or 0x01000000) + end + return op + parse_reg_base(p1) + parse_imm(p2, 7, 15, scale, true) +end + +local function parse_label(label, def) + local prefix = sub(label, 1, 2) + -- =>label (pc label reference) + if prefix == "=>" then + return "PC", 0, sub(label, 3) + end + -- ->name (global label reference) + if prefix == "->" then + return "LG", map_global[sub(label, 3)] + end + if def then + -- [1-9] (local label definition) + if match(label, "^[1-9]$") then + return "LG", 10+tonumber(label) + end + else + -- [<>][1-9] (local label reference) + local dir, lnum = match(label, "^([<>])([1-9])$") + if dir then -- Fwd: 1-9, Bkwd: 11-19. + return "LG", lnum + (dir == ">" and 0 or 10) + end + -- extern label (extern label reference) + local extname = match(label, "^extern%s+(%S+)$") + if extname then + return "EXT", map_extern[extname] + end + end + werror("bad label `"..label.."'") +end + +local function branch_type(op) + if band(op, 0x7c000000) == 0x14000000 then return 0 -- B, BL + elseif shr(op, 24) == 0x54 or band(op, 0x7e000000) == 0x34000000 or + band(op, 0x3b000000) == 0x18000000 then + return 0x800 -- B.cond, CBZ, CBNZ, LDR* literal + elseif band(op, 0x7e000000) == 0x36000000 then return 0x1000 -- TBZ, TBNZ + elseif band(op, 0x9f000000) == 0x10000000 then return 0x2000 -- ADR + elseif band(op, 0x9f000000) == band(0x90000000) then return 0x3000 -- ADRP + else + assert(false, "unknown branch type") + end +end + +------------------------------------------------------------------------------ + +local map_op, op_template + +local function op_alias(opname, f) + return function(params, nparams) + if not params then return "-> "..opname:sub(1, -3) end + f(params, nparams) + op_template(params, map_op[opname], nparams) + end +end + +local function alias_bfx(p) + p[4] = "#("..p[3]:sub(2)..")+("..p[4]:sub(2)..")-1" +end + +local function alias_bfiz(p) + parse_reg(p[1]) + if parse_reg_type == "w" then + p[3] = "#-("..p[3]:sub(2)..")%32" + p[4] = "#("..p[4]:sub(2)..")-1" + else + p[3] = "#-("..p[3]:sub(2)..")%64" + p[4] = "#("..p[4]:sub(2)..")-1" + end +end + +local alias_lslimm = op_alias("ubfm_4", function(p) + parse_reg(p[1]) + local sh = p[3]:sub(2) + if parse_reg_type == "w" then + p[3] = "#-("..sh..")%32" + p[4] = "#31-("..sh..")" + else + p[3] = "#-("..sh..")%64" + p[4] = "#63-("..sh..")" + end +end) + +-- Template strings for ARM instructions. +map_op = { + -- Basic data processing instructions. + add_3 = "0b000000DNMg|11000000pDpNIg|8b206000pDpNMx", + add_4 = "0b000000DNMSg|0b200000DNMXg|8b200000pDpNMXx|8b200000pDpNxMwX", + adds_3 = "2b000000DNMg|31000000DpNIg|ab206000DpNMx", + adds_4 = "2b000000DNMSg|2b200000DNMXg|ab200000DpNMXx|ab200000DpNxMwX", + cmn_2 = "2b00001fNMg|3100001fpNIg|ab20601fpNMx", + cmn_3 = "2b00001fNMSg|2b20001fNMXg|ab20001fpNMXx|ab20001fpNxMwX", + + sub_3 = "4b000000DNMg|51000000pDpNIg|cb206000pDpNMx", + sub_4 = "4b000000DNMSg|4b200000DNMXg|cb200000pDpNMXx|cb200000pDpNxMwX", + subs_3 = "6b000000DNMg|71000000DpNIg|eb206000DpNMx", + subs_4 = "6b000000DNMSg|6b200000DNMXg|eb200000DpNMXx|eb200000DpNxMwX", + cmp_2 = "6b00001fNMg|7100001fpNIg|eb20601fpNMx", + cmp_3 = "6b00001fNMSg|6b20001fNMXg|eb20001fpNMXx|eb20001fpNxMwX", + + neg_2 = "4b0003e0DMg", + neg_3 = "4b0003e0DMSg", + negs_2 = "6b0003e0DMg", + negs_3 = "6b0003e0DMSg", + + adc_3 = "1a000000DNMg", + adcs_3 = "3a000000DNMg", + sbc_3 = "5a000000DNMg", + sbcs_3 = "7a000000DNMg", + ngc_2 = "5a0003e0DMg", + ngcs_2 = "7a0003e0DMg", + + and_3 = "0a000000DNMg|12000000pDNig", + and_4 = "0a000000DNMSg", + orr_3 = "2a000000DNMg|32000000pDNig", + orr_4 = "2a000000DNMSg", + eor_3 = "4a000000DNMg|52000000pDNig", + eor_4 = "4a000000DNMSg", + ands_3 = "6a000000DNMg|72000000DNig", + ands_4 = "6a000000DNMSg", + tst_2 = "6a00001fNMg|7200001fNig", + tst_3 = "6a00001fNMSg", + + bic_3 = "0a200000DNMg", + bic_4 = "0a200000DNMSg", + orn_3 = "2a200000DNMg", + orn_4 = "2a200000DNMSg", + eon_3 = "4a200000DNMg", + eon_4 = "4a200000DNMSg", + bics_3 = "6a200000DNMg", + bics_4 = "6a200000DNMSg", + + movn_2 = "12800000DWg", + movn_3 = "12800000DWRg", + movz_2 = "52800000DWg", + movz_3 = "52800000DWRg", + movk_2 = "72800000DWg", + movk_3 = "72800000DWRg", + + -- TODO: this doesn't cover all valid immediates for mov reg, #imm. + mov_2 = "2a0003e0DMg|52800000DW|320003e0pDig|11000000pDpNg", + mov_3 = "2a0003e0DMSg", + mvn_2 = "2a2003e0DMg", + mvn_3 = "2a2003e0DMSg", + + adr_2 = "10000000DBx", + adrp_2 = "90000000DBx", + + csel_4 = "1a800000DNMCg", + csinc_4 = "1a800400DNMCg", + csinv_4 = "5a800000DNMCg", + csneg_4 = "5a800400DNMCg", + cset_2 = "1a9f07e0Dcg", + csetm_2 = "5a9f03e0Dcg", + cinc_3 = "1a800400DNmcg", + cinv_3 = "5a800000DNmcg", + cneg_3 = "5a800400DNmcg", + + ccmn_4 = "3a400000NMVCg|3a400800N5VCg", + ccmp_4 = "7a400000NMVCg|7a400800N5VCg", + + madd_4 = "1b000000DNMAg", + msub_4 = "1b008000DNMAg", + mul_3 = "1b007c00DNMg", + mneg_3 = "1b00fc00DNMg", + + smaddl_4 = "9b200000DxNMwAx", + smsubl_4 = "9b208000DxNMwAx", + smull_3 = "9b207c00DxNMw", + smnegl_3 = "9b20fc00DxNMw", + smulh_3 = "9b407c00DNMx", + umaddl_4 = "9ba00000DxNMwAx", + umsubl_4 = "9ba08000DxNMwAx", + umull_3 = "9ba07c00DxNMw", + umnegl_3 = "9ba0fc00DxNMw", + umulh_3 = "9bc07c00DNMx", + + udiv_3 = "1ac00800DNMg", + sdiv_3 = "1ac00c00DNMg", + + -- Bit operations. + sbfm_4 = "13000000DN12w|93400000DN12x", + bfm_4 = "33000000DN12w|b3400000DN12x", + ubfm_4 = "53000000DN12w|d3400000DN12x", + extr_4 = "13800000DNM2w|93c00000DNM2x", + + sxtb_2 = "13001c00DNw|93401c00DNx", + sxth_2 = "13003c00DNw|93403c00DNx", + sxtw_2 = "93407c00DxNw", + uxtb_2 = "53001c00DNw", + uxth_2 = "53003c00DNw", + + sbfx_4 = op_alias("sbfm_4", alias_bfx), + bfxil_4 = op_alias("bfm_4", alias_bfx), + ubfx_4 = op_alias("ubfm_4", alias_bfx), + sbfiz_4 = op_alias("sbfm_4", alias_bfiz), + bfi_4 = op_alias("bfm_4", alias_bfiz), + ubfiz_4 = op_alias("ubfm_4", alias_bfiz), + + lsl_3 = function(params, nparams) + if params and params[3]:byte() == 35 then + return alias_lslimm(params, nparams) + else + return op_template(params, "1ac02000DNMg", nparams) + end + end, + lsr_3 = "1ac02400DNMg|53007c00DN1w|d340fc00DN1x", + asr_3 = "1ac02800DNMg|13007c00DN1w|9340fc00DN1x", + ror_3 = "1ac02c00DNMg|13800000DNm2w|93c00000DNm2x", + + clz_2 = "5ac01000DNg", + cls_2 = "5ac01400DNg", + rbit_2 = "5ac00000DNg", + rev_2 = "5ac00800DNw|dac00c00DNx", + rev16_2 = "5ac00400DNg", + rev32_2 = "dac00800DNx", + + -- Loads and stores. + ["strb_*"] = "38000000DwL", + ["ldrb_*"] = "38400000DwL", + ["ldrsb_*"] = "38c00000DwL|38800000DxL", + ["strh_*"] = "78000000DwL", + ["ldrh_*"] = "78400000DwL", + ["ldrsh_*"] = "78c00000DwL|78800000DxL", + ["str_*"] = "b8000000DwL|f8000000DxL|bc000000DsL|fc000000DdL", + ["ldr_*"] = "18000000DwB|58000000DxB|1c000000DsB|5c000000DdB|b8400000DwL|f8400000DxL|bc400000DsL|fc400000DdL", + ["ldrsw_*"] = "98000000DxB|b8800000DxL", + -- NOTE: ldur etc. are handled by ldr et al. + + ["stp_*"] = "28000000DAwP|a8000000DAxP|2c000000DAsP|6c000000DAdP", + ["ldp_*"] = "28400000DAwP|a8400000DAxP|2c400000DAsP|6c400000DAdP", + ["ldpsw_*"] = "68400000DAxP", + + -- Branches. + b_1 = "14000000B", + bl_1 = "94000000B", + blr_1 = "d63f0000Nx", + br_1 = "d61f0000Nx", + ret_0 = "d65f03c0", + ret_1 = "d65f0000Nx", + -- b.cond is added below. + cbz_2 = "34000000DBg", + cbnz_2 = "35000000DBg", + tbz_3 = "36000000DTBw|36000000DTBx", + tbnz_3 = "37000000DTBw|37000000DTBx", + + -- Miscellaneous instructions. + -- TODO: hlt, hvc, smc, svc, eret, dcps[123], drps, mrs, msr + -- TODO: sys, sysl, ic, dc, at, tlbi + -- TODO: hint, yield, wfe, wfi, sev, sevl + -- TODO: clrex, dsb, dmb, isb + nop_0 = "d503201f", + brk_0 = "d4200000", + brk_1 = "d4200000W", + + -- Floating point instructions. + fmov_2 = "1e204000DNf|1e260000DwNs|1e270000DsNw|9e660000DxNd|9e670000DdNx|1e201000DFf", + fabs_2 = "1e20c000DNf", + fneg_2 = "1e214000DNf", + fsqrt_2 = "1e21c000DNf", + + fcvt_2 = "1e22c000DdNs|1e624000DsNd", + + -- TODO: half-precision and fixed-point conversions. + fcvtas_2 = "1e240000DwNs|9e240000DxNs|1e640000DwNd|9e640000DxNd", + fcvtau_2 = "1e250000DwNs|9e250000DxNs|1e650000DwNd|9e650000DxNd", + fcvtms_2 = "1e300000DwNs|9e300000DxNs|1e700000DwNd|9e700000DxNd", + fcvtmu_2 = "1e310000DwNs|9e310000DxNs|1e710000DwNd|9e710000DxNd", + fcvtns_2 = "1e200000DwNs|9e200000DxNs|1e600000DwNd|9e600000DxNd", + fcvtnu_2 = "1e210000DwNs|9e210000DxNs|1e610000DwNd|9e610000DxNd", + fcvtps_2 = "1e280000DwNs|9e280000DxNs|1e680000DwNd|9e680000DxNd", + fcvtpu_2 = "1e290000DwNs|9e290000DxNs|1e690000DwNd|9e690000DxNd", + fcvtzs_2 = "1e380000DwNs|9e380000DxNs|1e780000DwNd|9e780000DxNd", + fcvtzu_2 = "1e390000DwNs|9e390000DxNs|1e790000DwNd|9e790000DxNd", + + scvtf_2 = "1e220000DsNw|9e220000DsNx|1e620000DdNw|9e620000DdNx", + ucvtf_2 = "1e230000DsNw|9e230000DsNx|1e630000DdNw|9e630000DdNx", + + frintn_2 = "1e244000DNf", + frintp_2 = "1e24c000DNf", + frintm_2 = "1e254000DNf", + frintz_2 = "1e25c000DNf", + frinta_2 = "1e264000DNf", + frintx_2 = "1e274000DNf", + frinti_2 = "1e27c000DNf", + + fadd_3 = "1e202800DNMf", + fsub_3 = "1e203800DNMf", + fmul_3 = "1e200800DNMf", + fnmul_3 = "1e208800DNMf", + fdiv_3 = "1e201800DNMf", + + fmadd_4 = "1f000000DNMAf", + fmsub_4 = "1f008000DNMAf", + fnmadd_4 = "1f200000DNMAf", + fnmsub_4 = "1f208000DNMAf", + + fmax_3 = "1e204800DNMf", + fmaxnm_3 = "1e206800DNMf", + fmin_3 = "1e205800DNMf", + fminnm_3 = "1e207800DNMf", + + fcmp_2 = "1e202000NMf|1e202008NZf", + fcmpe_2 = "1e202010NMf|1e202018NZf", + + fccmp_4 = "1e200400NMVCf", + fccmpe_4 = "1e200410NMVCf", + + fcsel_4 = "1e200c00DNMCf", + + -- TODO: crc32*, aes*, sha*, pmull + -- TODO: SIMD instructions. +} + +for cond,c in pairs(map_cond) do + map_op["b"..cond.."_1"] = tohex(0x54000000+c).."B" +end + +------------------------------------------------------------------------------ + +-- Handle opcodes defined with template strings. +local function parse_template(params, template, nparams, pos) + local op = tonumber(sub(template, 1, 8), 16) + local n = 1 + local rtt = {} + + parse_reg_type = false + + -- Process each character. + for p in gmatch(sub(template, 9), ".") do + local q = params[n] + if p == "D" then + op = op + parse_reg(q); n = n + 1 + elseif p == "N" then + op = op + shl(parse_reg(q), 5); n = n + 1 + elseif p == "M" then + op = op + shl(parse_reg(q), 16); n = n + 1 + elseif p == "A" then + op = op + shl(parse_reg(q), 10); n = n + 1 + elseif p == "m" then + op = op + shl(parse_reg(params[n-1]), 16) + + elseif p == "p" then + if q == "sp" then params[n] = "@x31" end + elseif p == "g" then + if parse_reg_type == "x" then + op = op + 0x80000000 + elseif parse_reg_type ~= "w" then + werror("bad register type") + end + parse_reg_type = false + elseif p == "f" then + if parse_reg_type == "d" then + op = op + 0x00400000 + elseif parse_reg_type ~= "s" then + werror("bad register type") + end + parse_reg_type = false + elseif p == "x" or p == "w" or p == "d" or p == "s" then + if parse_reg_type ~= p then + werror("register size mismatch") + end + parse_reg_type = false + + elseif p == "L" then + op = parse_load(params, nparams, n, op) + elseif p == "P" then + op = parse_load_pair(params, nparams, n, op) + + elseif p == "B" then + local mode, v, s = parse_label(q, false); n = n + 1 + local m = branch_type(op) + waction("REL_"..mode, v+m, s, 1) + + elseif p == "I" then + op = op + parse_imm12(q); n = n + 1 + elseif p == "i" then + op = op + parse_imm13(q); n = n + 1 + elseif p == "W" then + op = op + parse_imm(q, 16, 5, 0, false); n = n + 1 + elseif p == "T" then + op = op + parse_imm6(q); n = n + 1 + elseif p == "1" then + op = op + parse_imm(q, 6, 16, 0, false); n = n + 1 + elseif p == "2" then + op = op + parse_imm(q, 6, 10, 0, false); n = n + 1 + elseif p == "5" then + op = op + parse_imm(q, 5, 16, 0, false); n = n + 1 + elseif p == "V" then + op = op + parse_imm(q, 4, 0, 0, false); n = n + 1 + elseif p == "F" then + op = op + parse_fpimm(q); n = n + 1 + elseif p == "Z" then + if q ~= "#0" and q ~= "#0.0" then werror("expected zero immediate") end + n = n + 1 + + elseif p == "S" then + op = op + parse_shift(q); n = n + 1 + elseif p == "X" then + op = op + parse_extend(q); n = n + 1 + elseif p == "R" then + op = op + parse_lslx16(q); n = n + 1 + elseif p == "C" then + op = op + parse_cond(q, 0); n = n + 1 + elseif p == "c" then + op = op + parse_cond(q, 1); n = n + 1 + + else + assert(false) + end + end + wputpos(pos, op) +end + +function op_template(params, template, nparams) + if not params then return template:gsub("%x%x%x%x%x%x%x%x", "") end + + -- Limit number of section buffer positions used by a single dasm_put(). + -- A single opcode needs a maximum of 3 positions. + if secpos+3 > maxsecpos then wflush() end + local pos = wpos() + local lpos, apos, spos = #actlist, #actargs, secpos + + local ok, err + for t in gmatch(template, "[^|]+") do + ok, err = pcall(parse_template, params, t, nparams, pos) + if ok then return end + secpos = spos + actlist[lpos+1] = nil + actlist[lpos+2] = nil + actlist[lpos+3] = nil + actargs[apos+1] = nil + actargs[apos+2] = nil + actargs[apos+3] = nil + end + error(err, 0) +end + +map_op[".template__"] = op_template + +------------------------------------------------------------------------------ + +-- Pseudo-opcode to mark the position where the action list is to be emitted. +map_op[".actionlist_1"] = function(params) + if not params then return "cvar" end + local name = params[1] -- No syntax check. You get to keep the pieces. + wline(function(out) writeactions(out, name) end) +end + +-- Pseudo-opcode to mark the position where the global enum is to be emitted. +map_op[".globals_1"] = function(params) + if not params then return "prefix" end + local prefix = params[1] -- No syntax check. You get to keep the pieces. + wline(function(out) writeglobals(out, prefix) end) +end + +-- Pseudo-opcode to mark the position where the global names are to be emitted. +map_op[".globalnames_1"] = function(params) + if not params then return "cvar" end + local name = params[1] -- No syntax check. You get to keep the pieces. + wline(function(out) writeglobalnames(out, name) end) +end + +-- Pseudo-opcode to mark the position where the extern names are to be emitted. +map_op[".externnames_1"] = function(params) + if not params then return "cvar" end + local name = params[1] -- No syntax check. You get to keep the pieces. + wline(function(out) writeexternnames(out, name) end) +end + +------------------------------------------------------------------------------ + +-- Label pseudo-opcode (converted from trailing colon form). +map_op[".label_1"] = function(params) + if not params then return "[1-9] | ->global | =>pcexpr" end + if secpos+1 > maxsecpos then wflush() end + local mode, n, s = parse_label(params[1], true) + if mode == "EXT" then werror("bad label definition") end + waction("LABEL_"..mode, n, s, 1) +end + +------------------------------------------------------------------------------ + +-- Pseudo-opcodes for data storage. +map_op[".long_*"] = function(params) + if not params then return "imm..." end + for _,p in ipairs(params) do + local n = tonumber(p) + if not n then werror("bad immediate `"..p.."'") end + if n < 0 then n = n + 2^32 end + wputw(n) + if secpos+2 > maxsecpos then wflush() end + end +end + +-- Alignment pseudo-opcode. +map_op[".align_1"] = function(params) + if not params then return "numpow2" end + if secpos+1 > maxsecpos then wflush() end + local align = tonumber(params[1]) + if align then + local x = align + -- Must be a power of 2 in the range (2 ... 256). + for i=1,8 do + x = x / 2 + if x == 1 then + waction("ALIGN", align-1, nil, 1) -- Action byte is 2**n-1. + return + end + end + end + werror("bad alignment") +end + +------------------------------------------------------------------------------ + +-- Pseudo-opcode for (primitive) type definitions (map to C types). +map_op[".type_3"] = function(params, nparams) + if not params then + return nparams == 2 and "name, ctype" or "name, ctype, reg" + end + local name, ctype, reg = params[1], params[2], params[3] + if not match(name, "^[%a_][%w_]*$") then + werror("bad type name `"..name.."'") + end + local tp = map_type[name] + if tp then + werror("duplicate type `"..name.."'") + end + -- Add #type to defines. A bit unclean to put it in map_archdef. + map_archdef["#"..name] = "sizeof("..ctype..")" + -- Add new type and emit shortcut define. + local num = ctypenum + 1 + map_type[name] = { + ctype = ctype, + ctypefmt = format("Dt%X(%%s)", num), + reg = reg, + } + wline(format("#define Dt%X(_V) (int)(ptrdiff_t)&(((%s *)0)_V)", num, ctype)) + ctypenum = num +end +map_op[".type_2"] = map_op[".type_3"] + +-- Dump type definitions. +local function dumptypes(out, lvl) + local t = {} + for name in pairs(map_type) do t[#t+1] = name end + sort(t) + out:write("Type definitions:\n") + for _,name in ipairs(t) do + local tp = map_type[name] + local reg = tp.reg or "" + out:write(format(" %-20s %-20s %s\n", name, tp.ctype, reg)) + end + out:write("\n") +end + +------------------------------------------------------------------------------ + +-- Set the current section. +function _M.section(num) + waction("SECTION", num) + wflush(true) -- SECTION is a terminal action. +end + +------------------------------------------------------------------------------ + +-- Dump architecture description. +function _M.dumparch(out) + out:write(format("DynASM %s version %s, released %s\n\n", + _info.arch, _info.version, _info.release)) + dumpactions(out) +end + +-- Dump all user defined elements. +function _M.dumpdef(out, lvl) + dumptypes(out, lvl) + dumpglobals(out, lvl) + dumpexterns(out, lvl) +end + +------------------------------------------------------------------------------ + +-- Pass callbacks from/to the DynASM core. +function _M.passcb(wl, we, wf, ww) + wline, werror, wfatal, wwarn = wl, we, wf, ww + return wflush +end + +-- Setup the arch-specific module. +function _M.setup(arch, opt) + g_arch, g_opt = arch, opt +end + +-- Merge the core maps and the arch-specific maps. +function _M.mergemaps(map_coreop, map_def) + setmetatable(map_op, { __index = map_coreop }) + setmetatable(map_def, { __index = map_archdef }) + return map_op, map_def +end + +return _M + +------------------------------------------------------------------------------ + diff --git a/ext/opcache/dynasm/dasm_mips.h b/ext/opcache/dynasm/dasm_mips.h new file mode 100644 index 0000000000000..f3b43211de239 --- /dev/null +++ b/ext/opcache/dynasm/dasm_mips.h @@ -0,0 +1,419 @@ +/* +** DynASM MIPS encoding engine. +** Copyright (C) 2005-2016 Mike Pall. All rights reserved. +** Released under the MIT license. See dynasm.lua for full copyright notice. +*/ + +#include +#include +#include +#include + +#define DASM_ARCH "mips" + +#ifndef DASM_EXTERN +#define DASM_EXTERN(a,b,c,d) 0 +#endif + +/* Action definitions. */ +enum { + DASM_STOP, DASM_SECTION, DASM_ESC, DASM_REL_EXT, + /* The following actions need a buffer position. */ + DASM_ALIGN, DASM_REL_LG, DASM_LABEL_LG, + /* The following actions also have an argument. */ + DASM_REL_PC, DASM_LABEL_PC, DASM_IMM, DASM_IMMS, + DASM__MAX +}; + +/* Maximum number of section buffer positions for a single dasm_put() call. */ +#define DASM_MAXSECPOS 25 + +/* DynASM encoder status codes. Action list offset or number are or'ed in. */ +#define DASM_S_OK 0x00000000 +#define DASM_S_NOMEM 0x01000000 +#define DASM_S_PHASE 0x02000000 +#define DASM_S_MATCH_SEC 0x03000000 +#define DASM_S_RANGE_I 0x11000000 +#define DASM_S_RANGE_SEC 0x12000000 +#define DASM_S_RANGE_LG 0x13000000 +#define DASM_S_RANGE_PC 0x14000000 +#define DASM_S_RANGE_REL 0x15000000 +#define DASM_S_UNDEF_LG 0x21000000 +#define DASM_S_UNDEF_PC 0x22000000 + +/* Macros to convert positions (8 bit section + 24 bit index). */ +#define DASM_POS2IDX(pos) ((pos)&0x00ffffff) +#define DASM_POS2BIAS(pos) ((pos)&0xff000000) +#define DASM_SEC2POS(sec) ((sec)<<24) +#define DASM_POS2SEC(pos) ((pos)>>24) +#define DASM_POS2PTR(D, pos) (D->sections[DASM_POS2SEC(pos)].rbuf + (pos)) + +/* Action list type. */ +typedef const unsigned int *dasm_ActList; + +/* Per-section structure. */ +typedef struct dasm_Section { + int *rbuf; /* Biased buffer pointer (negative section bias). */ + int *buf; /* True buffer pointer. */ + size_t bsize; /* Buffer size in bytes. */ + int pos; /* Biased buffer position. */ + int epos; /* End of biased buffer position - max single put. */ + int ofs; /* Byte offset into section. */ +} dasm_Section; + +/* Core structure holding the DynASM encoding state. */ +struct dasm_State { + size_t psize; /* Allocated size of this structure. */ + dasm_ActList actionlist; /* Current actionlist pointer. */ + int *lglabels; /* Local/global chain/pos ptrs. */ + size_t lgsize; + int *pclabels; /* PC label chains/pos ptrs. */ + size_t pcsize; + void **globals; /* Array of globals (bias -10). */ + dasm_Section *section; /* Pointer to active section. */ + size_t codesize; /* Total size of all code sections. */ + int maxsection; /* 0 <= sectionidx < maxsection. */ + int status; /* Status code. */ + dasm_Section sections[1]; /* All sections. Alloc-extended. */ +}; + +/* The size of the core structure depends on the max. number of sections. */ +#define DASM_PSZ(ms) (sizeof(dasm_State)+(ms-1)*sizeof(dasm_Section)) + + +/* Initialize DynASM state. */ +void dasm_init(Dst_DECL, int maxsection) +{ + dasm_State *D; + size_t psz = 0; + int i; + Dst_REF = NULL; + DASM_M_GROW(Dst, struct dasm_State, Dst_REF, psz, DASM_PSZ(maxsection)); + D = Dst_REF; + D->psize = psz; + D->lglabels = NULL; + D->lgsize = 0; + D->pclabels = NULL; + D->pcsize = 0; + D->globals = NULL; + D->maxsection = maxsection; + for (i = 0; i < maxsection; i++) { + D->sections[i].buf = NULL; /* Need this for pass3. */ + D->sections[i].rbuf = D->sections[i].buf - DASM_SEC2POS(i); + D->sections[i].bsize = 0; + D->sections[i].epos = 0; /* Wrong, but is recalculated after resize. */ + } +} + +/* Free DynASM state. */ +void dasm_free(Dst_DECL) +{ + dasm_State *D = Dst_REF; + int i; + for (i = 0; i < D->maxsection; i++) + if (D->sections[i].buf) + DASM_M_FREE(Dst, D->sections[i].buf, D->sections[i].bsize); + if (D->pclabels) DASM_M_FREE(Dst, D->pclabels, D->pcsize); + if (D->lglabels) DASM_M_FREE(Dst, D->lglabels, D->lgsize); + DASM_M_FREE(Dst, D, D->psize); +} + +/* Setup global label array. Must be called before dasm_setup(). */ +void dasm_setupglobal(Dst_DECL, void **gl, unsigned int maxgl) +{ + dasm_State *D = Dst_REF; + D->globals = gl - 10; /* Negative bias to compensate for locals. */ + DASM_M_GROW(Dst, int, D->lglabels, D->lgsize, (10+maxgl)*sizeof(int)); +} + +/* Grow PC label array. Can be called after dasm_setup(), too. */ +void dasm_growpc(Dst_DECL, unsigned int maxpc) +{ + dasm_State *D = Dst_REF; + size_t osz = D->pcsize; + DASM_M_GROW(Dst, int, D->pclabels, D->pcsize, maxpc*sizeof(int)); + memset((void *)(((unsigned char *)D->pclabels)+osz), 0, D->pcsize-osz); +} + +/* Setup encoder. */ +void dasm_setup(Dst_DECL, const void *actionlist) +{ + dasm_State *D = Dst_REF; + int i; + D->actionlist = (dasm_ActList)actionlist; + D->status = DASM_S_OK; + D->section = &D->sections[0]; + memset((void *)D->lglabels, 0, D->lgsize); + if (D->pclabels) memset((void *)D->pclabels, 0, D->pcsize); + for (i = 0; i < D->maxsection; i++) { + D->sections[i].pos = DASM_SEC2POS(i); + D->sections[i].ofs = 0; + } +} + + +#ifdef DASM_CHECKS +#define CK(x, st) \ + do { if (!(x)) { \ + D->status = DASM_S_##st|(p-D->actionlist-1); return; } } while (0) +#define CKPL(kind, st) \ + do { if ((size_t)((char *)pl-(char *)D->kind##labels) >= D->kind##size) { \ + D->status = DASM_S_RANGE_##st|(p-D->actionlist-1); return; } } while (0) +#else +#define CK(x, st) ((void)0) +#define CKPL(kind, st) ((void)0) +#endif + +/* Pass 1: Store actions and args, link branches/labels, estimate offsets. */ +void dasm_put(Dst_DECL, int start, ...) +{ + va_list ap; + dasm_State *D = Dst_REF; + dasm_ActList p = D->actionlist + start; + dasm_Section *sec = D->section; + int pos = sec->pos, ofs = sec->ofs; + int *b; + + if (pos >= sec->epos) { + DASM_M_GROW(Dst, int, sec->buf, sec->bsize, + sec->bsize + 2*DASM_MAXSECPOS*sizeof(int)); + sec->rbuf = sec->buf - DASM_POS2BIAS(pos); + sec->epos = (int)sec->bsize/sizeof(int) - DASM_MAXSECPOS+DASM_POS2BIAS(pos); + } + + b = sec->rbuf; + b[pos++] = start; + + va_start(ap, start); + while (1) { + unsigned int ins = *p++; + unsigned int action = (ins >> 16) - 0xff00; + if (action >= DASM__MAX) { + ofs += 4; + } else { + int *pl, n = action >= DASM_REL_PC ? va_arg(ap, int) : 0; + switch (action) { + case DASM_STOP: goto stop; + case DASM_SECTION: + n = (ins & 255); CK(n < D->maxsection, RANGE_SEC); + D->section = &D->sections[n]; goto stop; + case DASM_ESC: p++; ofs += 4; break; + case DASM_REL_EXT: break; + case DASM_ALIGN: ofs += (ins & 255); b[pos++] = ofs; break; + case DASM_REL_LG: + n = (ins & 2047) - 10; pl = D->lglabels + n; + /* Bkwd rel or global. */ + if (n >= 0) { CK(n>=10||*pl<0, RANGE_LG); CKPL(lg, LG); goto putrel; } + pl += 10; n = *pl; + if (n < 0) n = 0; /* Start new chain for fwd rel if label exists. */ + goto linkrel; + case DASM_REL_PC: + pl = D->pclabels + n; CKPL(pc, PC); + putrel: + n = *pl; + if (n < 0) { /* Label exists. Get label pos and store it. */ + b[pos] = -n; + } else { + linkrel: + b[pos] = n; /* Else link to rel chain, anchored at label. */ + *pl = pos; + } + pos++; + break; + case DASM_LABEL_LG: + pl = D->lglabels + (ins & 2047) - 10; CKPL(lg, LG); goto putlabel; + case DASM_LABEL_PC: + pl = D->pclabels + n; CKPL(pc, PC); + putlabel: + n = *pl; /* n > 0: Collapse rel chain and replace with label pos. */ + while (n > 0) { int *pb = DASM_POS2PTR(D, n); n = *pb; *pb = pos; + } + *pl = -pos; /* Label exists now. */ + b[pos++] = ofs; /* Store pass1 offset estimate. */ + break; + case DASM_IMM: case DASM_IMMS: +#ifdef DASM_CHECKS + CK((n & ((1<<((ins>>10)&31))-1)) == 0, RANGE_I); +#endif + n >>= ((ins>>10)&31); +#ifdef DASM_CHECKS + if (ins & 0x8000) + CK(((n + (1<<(((ins>>5)&31)-1)))>>((ins>>5)&31)) == 0, RANGE_I); + else + CK((n>>((ins>>5)&31)) == 0, RANGE_I); +#endif + b[pos++] = n; + break; + } + } + } +stop: + va_end(ap); + sec->pos = pos; + sec->ofs = ofs; +} +#undef CK + +/* Pass 2: Link sections, shrink aligns, fix label offsets. */ +int dasm_link(Dst_DECL, size_t *szp) +{ + dasm_State *D = Dst_REF; + int secnum; + int ofs = 0; + +#ifdef DASM_CHECKS + *szp = 0; + if (D->status != DASM_S_OK) return D->status; + { + int pc; + for (pc = 0; pc*sizeof(int) < D->pcsize; pc++) + if (D->pclabels[pc] > 0) return DASM_S_UNDEF_PC|pc; + } +#endif + + { /* Handle globals not defined in this translation unit. */ + int idx; + for (idx = 20; idx*sizeof(int) < D->lgsize; idx++) { + int n = D->lglabels[idx]; + /* Undefined label: Collapse rel chain and replace with marker (< 0). */ + while (n > 0) { int *pb = DASM_POS2PTR(D, n); n = *pb; *pb = -idx; } + } + } + + /* Combine all code sections. No support for data sections (yet). */ + for (secnum = 0; secnum < D->maxsection; secnum++) { + dasm_Section *sec = D->sections + secnum; + int *b = sec->rbuf; + int pos = DASM_SEC2POS(secnum); + int lastpos = sec->pos; + + while (pos != lastpos) { + dasm_ActList p = D->actionlist + b[pos++]; + while (1) { + unsigned int ins = *p++; + unsigned int action = (ins >> 16) - 0xff00; + switch (action) { + case DASM_STOP: case DASM_SECTION: goto stop; + case DASM_ESC: p++; break; + case DASM_REL_EXT: break; + case DASM_ALIGN: ofs -= (b[pos++] + ofs) & (ins & 255); break; + case DASM_REL_LG: case DASM_REL_PC: pos++; break; + case DASM_LABEL_LG: case DASM_LABEL_PC: b[pos++] += ofs; break; + case DASM_IMM: case DASM_IMMS: pos++; break; + } + } + stop: (void)0; + } + ofs += sec->ofs; /* Next section starts right after current section. */ + } + + D->codesize = ofs; /* Total size of all code sections */ + *szp = ofs; + return DASM_S_OK; +} + +#ifdef DASM_CHECKS +#define CK(x, st) \ + do { if (!(x)) return DASM_S_##st|(p-D->actionlist-1); } while (0) +#else +#define CK(x, st) ((void)0) +#endif + +/* Pass 3: Encode sections. */ +int dasm_encode(Dst_DECL, void *buffer) +{ + dasm_State *D = Dst_REF; + char *base = (char *)buffer; + unsigned int *cp = (unsigned int *)buffer; + int secnum; + + /* Encode all code sections. No support for data sections (yet). */ + for (secnum = 0; secnum < D->maxsection; secnum++) { + dasm_Section *sec = D->sections + secnum; + int *b = sec->buf; + int *endb = sec->rbuf + sec->pos; + + while (b != endb) { + dasm_ActList p = D->actionlist + *b++; + while (1) { + unsigned int ins = *p++; + unsigned int action = (ins >> 16) - 0xff00; + int n = (action >= DASM_ALIGN && action < DASM__MAX) ? *b++ : 0; + switch (action) { + case DASM_STOP: case DASM_SECTION: goto stop; + case DASM_ESC: *cp++ = *p++; break; + case DASM_REL_EXT: + n = DASM_EXTERN(Dst, (unsigned char *)cp, (ins & 2047), 1); + goto patchrel; + case DASM_ALIGN: + ins &= 255; while ((((char *)cp - base) & ins)) *cp++ = 0x60000000; + break; + case DASM_REL_LG: + CK(n >= 0, UNDEF_LG); + case DASM_REL_PC: + CK(n >= 0, UNDEF_PC); + n = *DASM_POS2PTR(D, n); + if (ins & 2048) + n = n - (int)((char *)cp - base); + else + n = (n + (int)(size_t)base) & 0x0fffffff; + patchrel: + CK((n & 3) == 0 && + ((n + ((ins & 2048) ? 0x00020000 : 0)) >> + ((ins & 2048) ? 18 : 28)) == 0, RANGE_REL); + cp[-1] |= ((n>>2) & ((ins & 2048) ? 0x0000ffff: 0x03ffffff)); + break; + case DASM_LABEL_LG: + ins &= 2047; if (ins >= 20) D->globals[ins-10] = (void *)(base + n); + break; + case DASM_LABEL_PC: break; + case DASM_IMMS: + cp[-1] |= ((n>>3) & 4); n &= 0x1f; + /* fallthrough */ + case DASM_IMM: + cp[-1] |= (n & ((1<<((ins>>5)&31))-1)) << (ins&31); + break; + default: *cp++ = ins; break; + } + } + stop: (void)0; + } + } + + if (base + D->codesize != (char *)cp) /* Check for phase errors. */ + return DASM_S_PHASE; + return DASM_S_OK; +} +#undef CK + +/* Get PC label offset. */ +int dasm_getpclabel(Dst_DECL, unsigned int pc) +{ + dasm_State *D = Dst_REF; + if (pc*sizeof(int) < D->pcsize) { + int pos = D->pclabels[pc]; + if (pos < 0) return *DASM_POS2PTR(D, -pos); + if (pos > 0) return -1; /* Undefined. */ + } + return -2; /* Unused or out of range. */ +} + +#ifdef DASM_CHECKS +/* Optional sanity checker to call between isolated encoding steps. */ +int dasm_checkstep(Dst_DECL, int secmatch) +{ + dasm_State *D = Dst_REF; + if (D->status == DASM_S_OK) { + int i; + for (i = 1; i <= 9; i++) { + if (D->lglabels[i] > 0) { D->status = DASM_S_UNDEF_LG|i; break; } + D->lglabels[i] = 0; + } + } + if (D->status == DASM_S_OK && secmatch >= 0 && + D->section != &D->sections[secmatch]) + D->status = DASM_S_MATCH_SEC|(D->section-D->sections); + return D->status; +} +#endif + diff --git a/ext/opcache/dynasm/dasm_mips.lua b/ext/opcache/dynasm/dasm_mips.lua new file mode 100644 index 0000000000000..c8010561db10c --- /dev/null +++ b/ext/opcache/dynasm/dasm_mips.lua @@ -0,0 +1,1008 @@ +------------------------------------------------------------------------------ +-- DynASM MIPS32/MIPS64 module. +-- +-- Copyright (C) 2005-2016 Mike Pall. All rights reserved. +-- See dynasm.lua for full copyright notice. +------------------------------------------------------------------------------ + +local mips64 = mips64 + +-- Module information: +local _info = { + arch = mips64 and "mips64" or "mips", + description = "DynASM MIPS32/MIPS64 module", + version = "1.4.0", + vernum = 10400, + release = "2016-05-24", + author = "Mike Pall", + license = "MIT", +} + +-- Exported glue functions for the arch-specific module. +local _M = { _info = _info } + +-- Cache library functions. +local type, tonumber, pairs, ipairs = type, tonumber, pairs, ipairs +local assert, setmetatable = assert, setmetatable +local _s = string +local sub, format, byte, char = _s.sub, _s.format, _s.byte, _s.char +local match, gmatch = _s.match, _s.gmatch +local concat, sort = table.concat, table.sort +local bit = bit or require("bit") +local band, shl, shr, sar = bit.band, bit.lshift, bit.rshift, bit.arshift +local tohex = bit.tohex + +-- Inherited tables and callbacks. +local g_opt, g_arch +local wline, werror, wfatal, wwarn + +-- Action name list. +-- CHECK: Keep this in sync with the C code! +local action_names = { + "STOP", "SECTION", "ESC", "REL_EXT", + "ALIGN", "REL_LG", "LABEL_LG", + "REL_PC", "LABEL_PC", "IMM", "IMMS", +} + +-- Maximum number of section buffer positions for dasm_put(). +-- CHECK: Keep this in sync with the C code! +local maxsecpos = 25 -- Keep this low, to avoid excessively long C lines. + +-- Action name -> action number. +local map_action = {} +for n,name in ipairs(action_names) do + map_action[name] = n-1 +end + +-- Action list buffer. +local actlist = {} + +-- Argument list for next dasm_put(). Start with offset 0 into action list. +local actargs = { 0 } + +-- Current number of section buffer positions for dasm_put(). +local secpos = 1 + +------------------------------------------------------------------------------ + +-- Dump action names and numbers. +local function dumpactions(out) + out:write("DynASM encoding engine action codes:\n") + for n,name in ipairs(action_names) do + local num = map_action[name] + out:write(format(" %-10s %02X %d\n", name, num, num)) + end + out:write("\n") +end + +-- Write action list buffer as a huge static C array. +local function writeactions(out, name) + local nn = #actlist + if nn == 0 then nn = 1; actlist[0] = map_action.STOP end + out:write("static const unsigned int ", name, "[", nn, "] = {\n") + for i = 1,nn-1 do + assert(out:write("0x", tohex(actlist[i]), ",\n")) + end + assert(out:write("0x", tohex(actlist[nn]), "\n};\n\n")) +end + +------------------------------------------------------------------------------ + +-- Add word to action list. +local function wputxw(n) + assert(n >= 0 and n <= 0xffffffff and n % 1 == 0, "word out of range") + actlist[#actlist+1] = n +end + +-- Add action to list with optional arg. Advance buffer pos, too. +local function waction(action, val, a, num) + local w = assert(map_action[action], "bad action name `"..action.."'") + wputxw(0xff000000 + w * 0x10000 + (val or 0)) + if a then actargs[#actargs+1] = a end + if a or num then secpos = secpos + (num or 1) end +end + +-- Flush action list (intervening C code or buffer pos overflow). +local function wflush(term) + if #actlist == actargs[1] then return end -- Nothing to flush. + if not term then waction("STOP") end -- Terminate action list. + wline(format("dasm_put(Dst, %s);", concat(actargs, ", ")), true) + actargs = { #actlist } -- Actionlist offset is 1st arg to next dasm_put(). + secpos = 1 -- The actionlist offset occupies a buffer position, too. +end + +-- Put escaped word. +local function wputw(n) + if n >= 0xff000000 then waction("ESC") end + wputxw(n) +end + +-- Reserve position for word. +local function wpos() + local pos = #actlist+1 + actlist[pos] = "" + return pos +end + +-- Store word to reserved position. +local function wputpos(pos, n) + assert(n >= 0 and n <= 0xffffffff and n % 1 == 0, "word out of range") + actlist[pos] = n +end + +------------------------------------------------------------------------------ + +-- Global label name -> global label number. With auto assignment on 1st use. +local next_global = 20 +local map_global = setmetatable({}, { __index = function(t, name) + if not match(name, "^[%a_][%w_]*$") then werror("bad global label") end + local n = next_global + if n > 2047 then werror("too many global labels") end + next_global = n + 1 + t[name] = n + return n +end}) + +-- Dump global labels. +local function dumpglobals(out, lvl) + local t = {} + for name, n in pairs(map_global) do t[n] = name end + out:write("Global labels:\n") + for i=20,next_global-1 do + out:write(format(" %s\n", t[i])) + end + out:write("\n") +end + +-- Write global label enum. +local function writeglobals(out, prefix) + local t = {} + for name, n in pairs(map_global) do t[n] = name end + out:write("enum {\n") + for i=20,next_global-1 do + out:write(" ", prefix, t[i], ",\n") + end + out:write(" ", prefix, "_MAX\n};\n") +end + +-- Write global label names. +local function writeglobalnames(out, name) + local t = {} + for name, n in pairs(map_global) do t[n] = name end + out:write("static const char *const ", name, "[] = {\n") + for i=20,next_global-1 do + out:write(" \"", t[i], "\",\n") + end + out:write(" (const char *)0\n};\n") +end + +------------------------------------------------------------------------------ + +-- Extern label name -> extern label number. With auto assignment on 1st use. +local next_extern = 0 +local map_extern_ = {} +local map_extern = setmetatable({}, { __index = function(t, name) + -- No restrictions on the name for now. + local n = next_extern + if n > 2047 then werror("too many extern labels") end + next_extern = n + 1 + t[name] = n + map_extern_[n] = name + return n +end}) + +-- Dump extern labels. +local function dumpexterns(out, lvl) + out:write("Extern labels:\n") + for i=0,next_extern-1 do + out:write(format(" %s\n", map_extern_[i])) + end + out:write("\n") +end + +-- Write extern label names. +local function writeexternnames(out, name) + out:write("static const char *const ", name, "[] = {\n") + for i=0,next_extern-1 do + out:write(" \"", map_extern_[i], "\",\n") + end + out:write(" (const char *)0\n};\n") +end + +------------------------------------------------------------------------------ + +-- Arch-specific maps. +local map_archdef = { sp="r29", ra="r31" } -- Ext. register name -> int. name. + +local map_type = {} -- Type name -> { ctype, reg } +local ctypenum = 0 -- Type number (for Dt... macros). + +-- Reverse defines for registers. +function _M.revdef(s) + if s == "r29" then return "sp" + elseif s == "r31" then return "ra" end + return s +end + +------------------------------------------------------------------------------ + +-- Template strings for MIPS instructions. +local map_op = { + -- First-level opcodes. + j_1 = "08000000J", + jal_1 = "0c000000J", + b_1 = "10000000B", + beqz_2 = "10000000SB", + beq_3 = "10000000STB", + bnez_2 = "14000000SB", + bne_3 = "14000000STB", + blez_2 = "18000000SB", + bgtz_2 = "1c000000SB", + addi_3 = "20000000TSI", + li_2 = "24000000TI", + addiu_3 = "24000000TSI", + slti_3 = "28000000TSI", + sltiu_3 = "2c000000TSI", + andi_3 = "30000000TSU", + lu_2 = "34000000TU", + ori_3 = "34000000TSU", + xori_3 = "38000000TSU", + lui_2 = "3c000000TU", + beqzl_2 = "50000000SB", + beql_3 = "50000000STB", + bnezl_2 = "54000000SB", + bnel_3 = "54000000STB", + blezl_2 = "58000000SB", + bgtzl_2 = "5c000000SB", + daddi_3 = mips64 and "60000000TSI", + daddiu_3 = mips64 and "64000000TSI", + ldl_2 = mips64 and "68000000TO", + ldr_2 = mips64 and "6c000000TO", + lb_2 = "80000000TO", + lh_2 = "84000000TO", + lwl_2 = "88000000TO", + lw_2 = "8c000000TO", + lbu_2 = "90000000TO", + lhu_2 = "94000000TO", + lwr_2 = "98000000TO", + lwu_2 = mips64 and "9c000000TO", + sb_2 = "a0000000TO", + sh_2 = "a4000000TO", + swl_2 = "a8000000TO", + sw_2 = "ac000000TO", + sdl_2 = mips64 and "b0000000TO", + sdr_2 = mips64 and "b1000000TO", + swr_2 = "b8000000TO", + cache_2 = "bc000000NO", + ll_2 = "c0000000TO", + lwc1_2 = "c4000000HO", + pref_2 = "cc000000NO", + ldc1_2 = "d4000000HO", + ld_2 = mips64 and "dc000000TO", + sc_2 = "e0000000TO", + swc1_2 = "e4000000HO", + scd_2 = mips64 and "f0000000TO", + sdc1_2 = "f4000000HO", + sd_2 = mips64 and "fc000000TO", + + -- Opcode SPECIAL. + nop_0 = "00000000", + sll_3 = "00000000DTA", + sextw_2 = "00000000DT", + movf_2 = "00000001DS", + movf_3 = "00000001DSC", + movt_2 = "00010001DS", + movt_3 = "00010001DSC", + srl_3 = "00000002DTA", + rotr_3 = "00200002DTA", + sra_3 = "00000003DTA", + sllv_3 = "00000004DTS", + srlv_3 = "00000006DTS", + rotrv_3 = "00000046DTS", + drotrv_3 = mips64 and "00000056DTS", + srav_3 = "00000007DTS", + jr_1 = "00000008S", + jalr_1 = "0000f809S", + jalr_2 = "00000009DS", + movz_3 = "0000000aDST", + movn_3 = "0000000bDST", + syscall_0 = "0000000c", + syscall_1 = "0000000cY", + break_0 = "0000000d", + break_1 = "0000000dY", + sync_0 = "0000000f", + mfhi_1 = "00000010D", + mthi_1 = "00000011S", + mflo_1 = "00000012D", + mtlo_1 = "00000013S", + dsllv_3 = mips64 and "00000014DTS", + dsrlv_3 = mips64 and "00000016DTS", + dsrav_3 = mips64 and "00000017DTS", + mult_2 = "00000018ST", + multu_2 = "00000019ST", + div_2 = "0000001aST", + divu_2 = "0000001bST", + dmult_2 = mips64 and "0000001cST", + dmultu_2 = mips64 and "0000001dST", + ddiv_2 = mips64 and "0000001eST", + ddivu_2 = mips64 and "0000001fST", + add_3 = "00000020DST", + move_2 = mips64 and "00000025DS" or "00000021DS", + addu_3 = "00000021DST", + sub_3 = "00000022DST", + negu_2 = mips64 and "0000002fDT" or "00000023DT", + subu_3 = "00000023DST", + and_3 = "00000024DST", + or_3 = "00000025DST", + xor_3 = "00000026DST", + not_2 = "00000027DS", + nor_3 = "00000027DST", + slt_3 = "0000002aDST", + sltu_3 = "0000002bDST", + dadd_3 = mips64 and "0000002cDST", + daddu_3 = mips64 and "0000002dDST", + dsub_3 = mips64 and "0000002eDST", + dsubu_3 = mips64 and "0000002fDST", + tge_2 = "00000030ST", + tge_3 = "00000030STZ", + tgeu_2 = "00000031ST", + tgeu_3 = "00000031STZ", + tlt_2 = "00000032ST", + tlt_3 = "00000032STZ", + tltu_2 = "00000033ST", + tltu_3 = "00000033STZ", + teq_2 = "00000034ST", + teq_3 = "00000034STZ", + tne_2 = "00000036ST", + tne_3 = "00000036STZ", + dsll_3 = mips64 and "00000038DTa", + dsrl_3 = mips64 and "0000003aDTa", + drotr_3 = mips64 and "0020003aDTa", + dsra_3 = mips64 and "0000003bDTa", + dsll32_3 = mips64 and "0000003cDTA", + dsrl32_3 = mips64 and "0000003eDTA", + drotr32_3 = mips64 and "0020003eDTA", + dsra32_3 = mips64 and "0000003fDTA", + + -- Opcode REGIMM. + bltz_2 = "04000000SB", + bgez_2 = "04010000SB", + bltzl_2 = "04020000SB", + bgezl_2 = "04030000SB", + tgei_2 = "04080000SI", + tgeiu_2 = "04090000SI", + tlti_2 = "040a0000SI", + tltiu_2 = "040b0000SI", + teqi_2 = "040c0000SI", + tnei_2 = "040e0000SI", + bltzal_2 = "04100000SB", + bal_1 = "04110000B", + bgezal_2 = "04110000SB", + bltzall_2 = "04120000SB", + bgezall_2 = "04130000SB", + synci_1 = "041f0000O", + + -- Opcode SPECIAL2. + madd_2 = "70000000ST", + maddu_2 = "70000001ST", + mul_3 = "70000002DST", + msub_2 = "70000004ST", + msubu_2 = "70000005ST", + clz_2 = "70000020DS=", + clo_2 = "70000021DS=", + dclz_2 = mips64 and "70000024DS=", + dclo_2 = mips64 and "70000025DS=", + sdbbp_0 = "7000003f", + sdbbp_1 = "7000003fY", + + -- Opcode SPECIAL3. + ext_4 = "7c000000TSAM", -- Note: last arg is msbd = size-1 + dextm_4 = mips64 and "7c000001TSAM", -- Args: pos | size-1-32 + dextu_4 = mips64 and "7c000002TSAM", -- Args: pos-32 | size-1 + dext_4 = mips64 and "7c000003TSAM", -- Args: pos | size-1 + zextw_2 = mips64 and "7c00f803TS", + ins_4 = "7c000004TSAM", -- Note: last arg is msb = pos+size-1 + dinsm_4 = mips64 and "7c000005TSAM", -- Args: pos | pos+size-33 + dinsu_4 = mips64 and "7c000006TSAM", -- Args: pos-32 | pos+size-33 + dins_4 = mips64 and "7c000007TSAM", -- Args: pos | pos+size-1 + wsbh_2 = "7c0000a0DT", + dsbh_2 = mips64 and "7c0000a4DT", + dshd_2 = mips64 and "7c000164DT", + seb_2 = "7c000420DT", + seh_2 = "7c000620DT", + rdhwr_2 = "7c00003bTD", + + -- Opcode COP0. + mfc0_2 = "40000000TD", + mfc0_3 = "40000000TDW", + dmfc0_2 = mips64 and "40200000TD", + dmfc0_3 = mips64 and "40200000TDW", + mtc0_2 = "40800000TD", + mtc0_3 = "40800000TDW", + dmtc0_2 = mips64 and "40a00000TD", + dmtc0_3 = mips64 and "40a00000TDW", + rdpgpr_2 = "41400000DT", + di_0 = "41606000", + di_1 = "41606000T", + ei_0 = "41606020", + ei_1 = "41606020T", + wrpgpr_2 = "41c00000DT", + tlbr_0 = "42000001", + tlbwi_0 = "42000002", + tlbwr_0 = "42000006", + tlbp_0 = "42000008", + eret_0 = "42000018", + deret_0 = "4200001f", + wait_0 = "42000020", + + -- Opcode COP1. + mfc1_2 = "44000000TG", + dmfc1_2 = mips64 and "44200000TG", + cfc1_2 = "44400000TG", + mfhc1_2 = "44600000TG", + mtc1_2 = "44800000TG", + dmtc1_2 = mips64 and "44a00000TG", + ctc1_2 = "44c00000TG", + mthc1_2 = "44e00000TG", + + bc1f_1 = "45000000B", + bc1f_2 = "45000000CB", + bc1t_1 = "45010000B", + bc1t_2 = "45010000CB", + bc1fl_1 = "45020000B", + bc1fl_2 = "45020000CB", + bc1tl_1 = "45030000B", + bc1tl_2 = "45030000CB", + + ["add.s_3"] = "46000000FGH", + ["sub.s_3"] = "46000001FGH", + ["mul.s_3"] = "46000002FGH", + ["div.s_3"] = "46000003FGH", + ["sqrt.s_2"] = "46000004FG", + ["abs.s_2"] = "46000005FG", + ["mov.s_2"] = "46000006FG", + ["neg.s_2"] = "46000007FG", + ["round.l.s_2"] = "46000008FG", + ["trunc.l.s_2"] = "46000009FG", + ["ceil.l.s_2"] = "4600000aFG", + ["floor.l.s_2"] = "4600000bFG", + ["round.w.s_2"] = "4600000cFG", + ["trunc.w.s_2"] = "4600000dFG", + ["ceil.w.s_2"] = "4600000eFG", + ["floor.w.s_2"] = "4600000fFG", + ["movf.s_2"] = "46000011FG", + ["movf.s_3"] = "46000011FGC", + ["movt.s_2"] = "46010011FG", + ["movt.s_3"] = "46010011FGC", + ["movz.s_3"] = "46000012FGT", + ["movn.s_3"] = "46000013FGT", + ["recip.s_2"] = "46000015FG", + ["rsqrt.s_2"] = "46000016FG", + ["cvt.d.s_2"] = "46000021FG", + ["cvt.w.s_2"] = "46000024FG", + ["cvt.l.s_2"] = "46000025FG", + ["cvt.ps.s_3"] = "46000026FGH", + ["c.f.s_2"] = "46000030GH", + ["c.f.s_3"] = "46000030VGH", + ["c.un.s_2"] = "46000031GH", + ["c.un.s_3"] = "46000031VGH", + ["c.eq.s_2"] = "46000032GH", + ["c.eq.s_3"] = "46000032VGH", + ["c.ueq.s_2"] = "46000033GH", + ["c.ueq.s_3"] = "46000033VGH", + ["c.olt.s_2"] = "46000034GH", + ["c.olt.s_3"] = "46000034VGH", + ["c.ult.s_2"] = "46000035GH", + ["c.ult.s_3"] = "46000035VGH", + ["c.ole.s_2"] = "46000036GH", + ["c.ole.s_3"] = "46000036VGH", + ["c.ule.s_2"] = "46000037GH", + ["c.ule.s_3"] = "46000037VGH", + ["c.sf.s_2"] = "46000038GH", + ["c.sf.s_3"] = "46000038VGH", + ["c.ngle.s_2"] = "46000039GH", + ["c.ngle.s_3"] = "46000039VGH", + ["c.seq.s_2"] = "4600003aGH", + ["c.seq.s_3"] = "4600003aVGH", + ["c.ngl.s_2"] = "4600003bGH", + ["c.ngl.s_3"] = "4600003bVGH", + ["c.lt.s_2"] = "4600003cGH", + ["c.lt.s_3"] = "4600003cVGH", + ["c.nge.s_2"] = "4600003dGH", + ["c.nge.s_3"] = "4600003dVGH", + ["c.le.s_2"] = "4600003eGH", + ["c.le.s_3"] = "4600003eVGH", + ["c.ngt.s_2"] = "4600003fGH", + ["c.ngt.s_3"] = "4600003fVGH", + + ["add.d_3"] = "46200000FGH", + ["sub.d_3"] = "46200001FGH", + ["mul.d_3"] = "46200002FGH", + ["div.d_3"] = "46200003FGH", + ["sqrt.d_2"] = "46200004FG", + ["abs.d_2"] = "46200005FG", + ["mov.d_2"] = "46200006FG", + ["neg.d_2"] = "46200007FG", + ["round.l.d_2"] = "46200008FG", + ["trunc.l.d_2"] = "46200009FG", + ["ceil.l.d_2"] = "4620000aFG", + ["floor.l.d_2"] = "4620000bFG", + ["round.w.d_2"] = "4620000cFG", + ["trunc.w.d_2"] = "4620000dFG", + ["ceil.w.d_2"] = "4620000eFG", + ["floor.w.d_2"] = "4620000fFG", + ["movf.d_2"] = "46200011FG", + ["movf.d_3"] = "46200011FGC", + ["movt.d_2"] = "46210011FG", + ["movt.d_3"] = "46210011FGC", + ["movz.d_3"] = "46200012FGT", + ["movn.d_3"] = "46200013FGT", + ["recip.d_2"] = "46200015FG", + ["rsqrt.d_2"] = "46200016FG", + ["cvt.s.d_2"] = "46200020FG", + ["cvt.w.d_2"] = "46200024FG", + ["cvt.l.d_2"] = "46200025FG", + ["c.f.d_2"] = "46200030GH", + ["c.f.d_3"] = "46200030VGH", + ["c.un.d_2"] = "46200031GH", + ["c.un.d_3"] = "46200031VGH", + ["c.eq.d_2"] = "46200032GH", + ["c.eq.d_3"] = "46200032VGH", + ["c.ueq.d_2"] = "46200033GH", + ["c.ueq.d_3"] = "46200033VGH", + ["c.olt.d_2"] = "46200034GH", + ["c.olt.d_3"] = "46200034VGH", + ["c.ult.d_2"] = "46200035GH", + ["c.ult.d_3"] = "46200035VGH", + ["c.ole.d_2"] = "46200036GH", + ["c.ole.d_3"] = "46200036VGH", + ["c.ule.d_2"] = "46200037GH", + ["c.ule.d_3"] = "46200037VGH", + ["c.sf.d_2"] = "46200038GH", + ["c.sf.d_3"] = "46200038VGH", + ["c.ngle.d_2"] = "46200039GH", + ["c.ngle.d_3"] = "46200039VGH", + ["c.seq.d_2"] = "4620003aGH", + ["c.seq.d_3"] = "4620003aVGH", + ["c.ngl.d_2"] = "4620003bGH", + ["c.ngl.d_3"] = "4620003bVGH", + ["c.lt.d_2"] = "4620003cGH", + ["c.lt.d_3"] = "4620003cVGH", + ["c.nge.d_2"] = "4620003dGH", + ["c.nge.d_3"] = "4620003dVGH", + ["c.le.d_2"] = "4620003eGH", + ["c.le.d_3"] = "4620003eVGH", + ["c.ngt.d_2"] = "4620003fGH", + ["c.ngt.d_3"] = "4620003fVGH", + + ["add.ps_3"] = "46c00000FGH", + ["sub.ps_3"] = "46c00001FGH", + ["mul.ps_3"] = "46c00002FGH", + ["abs.ps_2"] = "46c00005FG", + ["mov.ps_2"] = "46c00006FG", + ["neg.ps_2"] = "46c00007FG", + ["movf.ps_2"] = "46c00011FG", + ["movf.ps_3"] = "46c00011FGC", + ["movt.ps_2"] = "46c10011FG", + ["movt.ps_3"] = "46c10011FGC", + ["movz.ps_3"] = "46c00012FGT", + ["movn.ps_3"] = "46c00013FGT", + ["cvt.s.pu_2"] = "46c00020FG", + ["cvt.s.pl_2"] = "46c00028FG", + ["pll.ps_3"] = "46c0002cFGH", + ["plu.ps_3"] = "46c0002dFGH", + ["pul.ps_3"] = "46c0002eFGH", + ["puu.ps_3"] = "46c0002fFGH", + ["c.f.ps_2"] = "46c00030GH", + ["c.f.ps_3"] = "46c00030VGH", + ["c.un.ps_2"] = "46c00031GH", + ["c.un.ps_3"] = "46c00031VGH", + ["c.eq.ps_2"] = "46c00032GH", + ["c.eq.ps_3"] = "46c00032VGH", + ["c.ueq.ps_2"] = "46c00033GH", + ["c.ueq.ps_3"] = "46c00033VGH", + ["c.olt.ps_2"] = "46c00034GH", + ["c.olt.ps_3"] = "46c00034VGH", + ["c.ult.ps_2"] = "46c00035GH", + ["c.ult.ps_3"] = "46c00035VGH", + ["c.ole.ps_2"] = "46c00036GH", + ["c.ole.ps_3"] = "46c00036VGH", + ["c.ule.ps_2"] = "46c00037GH", + ["c.ule.ps_3"] = "46c00037VGH", + ["c.sf.ps_2"] = "46c00038GH", + ["c.sf.ps_3"] = "46c00038VGH", + ["c.ngle.ps_2"] = "46c00039GH", + ["c.ngle.ps_3"] = "46c00039VGH", + ["c.seq.ps_2"] = "46c0003aGH", + ["c.seq.ps_3"] = "46c0003aVGH", + ["c.ngl.ps_2"] = "46c0003bGH", + ["c.ngl.ps_3"] = "46c0003bVGH", + ["c.lt.ps_2"] = "46c0003cGH", + ["c.lt.ps_3"] = "46c0003cVGH", + ["c.nge.ps_2"] = "46c0003dGH", + ["c.nge.ps_3"] = "46c0003dVGH", + ["c.le.ps_2"] = "46c0003eGH", + ["c.le.ps_3"] = "46c0003eVGH", + ["c.ngt.ps_2"] = "46c0003fGH", + ["c.ngt.ps_3"] = "46c0003fVGH", + + ["cvt.s.w_2"] = "46800020FG", + ["cvt.d.w_2"] = "46800021FG", + + ["cvt.s.l_2"] = "46a00020FG", + ["cvt.d.l_2"] = "46a00021FG", + + -- Opcode COP1X. + lwxc1_2 = "4c000000FX", + ldxc1_2 = "4c000001FX", + luxc1_2 = "4c000005FX", + swxc1_2 = "4c000008FX", + sdxc1_2 = "4c000009FX", + suxc1_2 = "4c00000dFX", + prefx_2 = "4c00000fMX", + ["alnv.ps_4"] = "4c00001eFGHS", + ["madd.s_4"] = "4c000020FRGH", + ["madd.d_4"] = "4c000021FRGH", + ["madd.ps_4"] = "4c000026FRGH", + ["msub.s_4"] = "4c000028FRGH", + ["msub.d_4"] = "4c000029FRGH", + ["msub.ps_4"] = "4c00002eFRGH", + ["nmadd.s_4"] = "4c000030FRGH", + ["nmadd.d_4"] = "4c000031FRGH", + ["nmadd.ps_4"] = "4c000036FRGH", + ["nmsub.s_4"] = "4c000038FRGH", + ["nmsub.d_4"] = "4c000039FRGH", + ["nmsub.ps_4"] = "4c00003eFRGH", +} + +------------------------------------------------------------------------------ + +local function parse_gpr(expr) + local tname, ovreg = match(expr, "^([%w_]+):(r[1-3]?[0-9])$") + local tp = map_type[tname or expr] + if tp then + local reg = ovreg or tp.reg + if not reg then + werror("type `"..(tname or expr).."' needs a register override") + end + expr = reg + end + local r = match(expr, "^r([1-3]?[0-9])$") + if r then + r = tonumber(r) + if r <= 31 then return r, tp end + end + werror("bad register name `"..expr.."'") +end + +local function parse_fpr(expr) + local r = match(expr, "^f([1-3]?[0-9])$") + if r then + r = tonumber(r) + if r <= 31 then return r end + end + werror("bad register name `"..expr.."'") +end + +local function parse_imm(imm, bits, shift, scale, signed, action) + local n = tonumber(imm) + if n then + local m = sar(n, scale) + if shl(m, scale) == n then + if signed then + local s = sar(m, bits-1) + if s == 0 then return shl(m, shift) + elseif s == -1 then return shl(m + shl(1, bits), shift) end + else + if sar(m, bits) == 0 then return shl(m, shift) end + end + end + werror("out of range immediate `"..imm.."'") + elseif match(imm, "^[rf]([1-3]?[0-9])$") or + match(imm, "^([%w_]+):([rf][1-3]?[0-9])$") then + werror("expected immediate operand, got register") + else + waction(action or "IMM", + (signed and 32768 or 0)+shl(scale, 10)+shl(bits, 5)+shift, imm) + return 0 + end +end + +local function parse_disp(disp) + local imm, reg = match(disp, "^(.*)%(([%w_:]+)%)$") + if imm then + local r = shl(parse_gpr(reg), 21) + local extname = match(imm, "^extern%s+(%S+)$") + if extname then + waction("REL_EXT", map_extern[extname], nil, 1) + return r + else + return r + parse_imm(imm, 16, 0, 0, true) + end + end + local reg, tailr = match(disp, "^([%w_:]+)%s*(.*)$") + if reg and tailr ~= "" then + local r, tp = parse_gpr(reg) + if tp then + waction("IMM", 32768+16*32, format(tp.ctypefmt, tailr)) + return shl(r, 21) + end + end + werror("bad displacement `"..disp.."'") +end + +local function parse_index(idx) + local rt, rs = match(idx, "^(.*)%(([%w_:]+)%)$") + if rt then + rt = parse_gpr(rt) + rs = parse_gpr(rs) + return shl(rt, 16) + shl(rs, 21) + end + werror("bad index `"..idx.."'") +end + +local function parse_label(label, def) + local prefix = sub(label, 1, 2) + -- =>label (pc label reference) + if prefix == "=>" then + return "PC", 0, sub(label, 3) + end + -- ->name (global label reference) + if prefix == "->" then + return "LG", map_global[sub(label, 3)] + end + if def then + -- [1-9] (local label definition) + if match(label, "^[1-9]$") then + return "LG", 10+tonumber(label) + end + else + -- [<>][1-9] (local label reference) + local dir, lnum = match(label, "^([<>])([1-9])$") + if dir then -- Fwd: 1-9, Bkwd: 11-19. + return "LG", lnum + (dir == ">" and 0 or 10) + end + -- extern label (extern label reference) + local extname = match(label, "^extern%s+(%S+)$") + if extname then + return "EXT", map_extern[extname] + end + end + werror("bad label `"..label.."'") +end + +------------------------------------------------------------------------------ + +-- Handle opcodes defined with template strings. +map_op[".template__"] = function(params, template, nparams) + if not params then return sub(template, 9) end + local op = tonumber(sub(template, 1, 8), 16) + local n = 1 + + -- Limit number of section buffer positions used by a single dasm_put(). + -- A single opcode needs a maximum of 2 positions (ins/ext). + if secpos+2 > maxsecpos then wflush() end + local pos = wpos() + + -- Process each character. + for p in gmatch(sub(template, 9), ".") do + if p == "D" then + op = op + shl(parse_gpr(params[n]), 11); n = n + 1 + elseif p == "T" then + op = op + shl(parse_gpr(params[n]), 16); n = n + 1 + elseif p == "S" then + op = op + shl(parse_gpr(params[n]), 21); n = n + 1 + elseif p == "F" then + op = op + shl(parse_fpr(params[n]), 6); n = n + 1 + elseif p == "G" then + op = op + shl(parse_fpr(params[n]), 11); n = n + 1 + elseif p == "H" then + op = op + shl(parse_fpr(params[n]), 16); n = n + 1 + elseif p == "R" then + op = op + shl(parse_fpr(params[n]), 21); n = n + 1 + elseif p == "I" then + op = op + parse_imm(params[n], 16, 0, 0, true); n = n + 1 + elseif p == "U" then + op = op + parse_imm(params[n], 16, 0, 0, false); n = n + 1 + elseif p == "O" then + op = op + parse_disp(params[n]); n = n + 1 + elseif p == "X" then + op = op + parse_index(params[n]); n = n + 1 + elseif p == "B" or p == "J" then + local mode, n, s = parse_label(params[n], false) + if p == "B" then n = n + 2048 end + waction("REL_"..mode, n, s, 1) + n = n + 1 + elseif p == "A" then + op = op + parse_imm(params[n], 5, 6, 0, false); n = n + 1 + elseif p == "a" then + local m = parse_imm(params[n], 6, 6, 0, false, "IMMS"); n = n + 1 + op = op + band(m, 0x7c0) + band(shr(m, 9), 4) + elseif p == "M" then + op = op + parse_imm(params[n], 5, 11, 0, false); n = n + 1 + elseif p == "N" then + op = op + parse_imm(params[n], 5, 16, 0, false); n = n + 1 + elseif p == "C" then + op = op + parse_imm(params[n], 3, 18, 0, false); n = n + 1 + elseif p == "V" then + op = op + parse_imm(params[n], 3, 8, 0, false); n = n + 1 + elseif p == "W" then + op = op + parse_imm(params[n], 3, 0, 0, false); n = n + 1 + elseif p == "Y" then + op = op + parse_imm(params[n], 20, 6, 0, false); n = n + 1 + elseif p == "Z" then + op = op + parse_imm(params[n], 10, 6, 0, false); n = n + 1 + elseif p == "=" then + op = op + shl(band(op, 0xf800), 5) -- Copy D to T for clz, clo. + else + assert(false) + end + end + wputpos(pos, op) +end + +------------------------------------------------------------------------------ + +-- Pseudo-opcode to mark the position where the action list is to be emitted. +map_op[".actionlist_1"] = function(params) + if not params then return "cvar" end + local name = params[1] -- No syntax check. You get to keep the pieces. + wline(function(out) writeactions(out, name) end) +end + +-- Pseudo-opcode to mark the position where the global enum is to be emitted. +map_op[".globals_1"] = function(params) + if not params then return "prefix" end + local prefix = params[1] -- No syntax check. You get to keep the pieces. + wline(function(out) writeglobals(out, prefix) end) +end + +-- Pseudo-opcode to mark the position where the global names are to be emitted. +map_op[".globalnames_1"] = function(params) + if not params then return "cvar" end + local name = params[1] -- No syntax check. You get to keep the pieces. + wline(function(out) writeglobalnames(out, name) end) +end + +-- Pseudo-opcode to mark the position where the extern names are to be emitted. +map_op[".externnames_1"] = function(params) + if not params then return "cvar" end + local name = params[1] -- No syntax check. You get to keep the pieces. + wline(function(out) writeexternnames(out, name) end) +end + +------------------------------------------------------------------------------ + +-- Label pseudo-opcode (converted from trailing colon form). +map_op[".label_1"] = function(params) + if not params then return "[1-9] | ->global | =>pcexpr" end + if secpos+1 > maxsecpos then wflush() end + local mode, n, s = parse_label(params[1], true) + if mode == "EXT" then werror("bad label definition") end + waction("LABEL_"..mode, n, s, 1) +end + +------------------------------------------------------------------------------ + +-- Pseudo-opcodes for data storage. +map_op[".long_*"] = function(params) + if not params then return "imm..." end + for _,p in ipairs(params) do + local n = tonumber(p) + if not n then werror("bad immediate `"..p.."'") end + if n < 0 then n = n + 2^32 end + wputw(n) + if secpos+2 > maxsecpos then wflush() end + end +end + +-- Alignment pseudo-opcode. +map_op[".align_1"] = function(params) + if not params then return "numpow2" end + if secpos+1 > maxsecpos then wflush() end + local align = tonumber(params[1]) + if align then + local x = align + -- Must be a power of 2 in the range (2 ... 256). + for i=1,8 do + x = x / 2 + if x == 1 then + waction("ALIGN", align-1, nil, 1) -- Action byte is 2**n-1. + return + end + end + end + werror("bad alignment") +end + +------------------------------------------------------------------------------ + +-- Pseudo-opcode for (primitive) type definitions (map to C types). +map_op[".type_3"] = function(params, nparams) + if not params then + return nparams == 2 and "name, ctype" or "name, ctype, reg" + end + local name, ctype, reg = params[1], params[2], params[3] + if not match(name, "^[%a_][%w_]*$") then + werror("bad type name `"..name.."'") + end + local tp = map_type[name] + if tp then + werror("duplicate type `"..name.."'") + end + -- Add #type to defines. A bit unclean to put it in map_archdef. + map_archdef["#"..name] = "sizeof("..ctype..")" + -- Add new type and emit shortcut define. + local num = ctypenum + 1 + map_type[name] = { + ctype = ctype, + ctypefmt = format("Dt%X(%%s)", num), + reg = reg, + } + wline(format("#define Dt%X(_V) (int)(ptrdiff_t)&(((%s *)0)_V)", num, ctype)) + ctypenum = num +end +map_op[".type_2"] = map_op[".type_3"] + +-- Dump type definitions. +local function dumptypes(out, lvl) + local t = {} + for name in pairs(map_type) do t[#t+1] = name end + sort(t) + out:write("Type definitions:\n") + for _,name in ipairs(t) do + local tp = map_type[name] + local reg = tp.reg or "" + out:write(format(" %-20s %-20s %s\n", name, tp.ctype, reg)) + end + out:write("\n") +end + +------------------------------------------------------------------------------ + +-- Set the current section. +function _M.section(num) + waction("SECTION", num) + wflush(true) -- SECTION is a terminal action. +end + +------------------------------------------------------------------------------ + +-- Dump architecture description. +function _M.dumparch(out) + out:write(format("DynASM %s version %s, released %s\n\n", + _info.arch, _info.version, _info.release)) + dumpactions(out) +end + +-- Dump all user defined elements. +function _M.dumpdef(out, lvl) + dumptypes(out, lvl) + dumpglobals(out, lvl) + dumpexterns(out, lvl) +end + +------------------------------------------------------------------------------ + +-- Pass callbacks from/to the DynASM core. +function _M.passcb(wl, we, wf, ww) + wline, werror, wfatal, wwarn = wl, we, wf, ww + return wflush +end + +-- Setup the arch-specific module. +function _M.setup(arch, opt) + g_arch, g_opt = arch, opt +end + +-- Merge the core maps and the arch-specific maps. +function _M.mergemaps(map_coreop, map_def) + setmetatable(map_op, { __index = map_coreop }) + setmetatable(map_def, { __index = map_archdef }) + return map_op, map_def +end + +return _M + +------------------------------------------------------------------------------ + diff --git a/ext/opcache/dynasm/dasm_mips64.lua b/ext/opcache/dynasm/dasm_mips64.lua new file mode 100644 index 0000000000000..94f21921fc504 --- /dev/null +++ b/ext/opcache/dynasm/dasm_mips64.lua @@ -0,0 +1,12 @@ +------------------------------------------------------------------------------ +-- DynASM MIPS64 module. +-- +-- Copyright (C) 2005-2016 Mike Pall. All rights reserved. +-- See dynasm.lua for full copyright notice. +------------------------------------------------------------------------------ +-- This module just sets 64 bit mode for the combined MIPS/MIPS64 module. +-- All the interesting stuff is there. +------------------------------------------------------------------------------ + +mips64 = true -- Using a global is an ugly, but effective solution. +return require("dasm_mips") diff --git a/ext/opcache/dynasm/dasm_ppc.h b/ext/opcache/dynasm/dasm_ppc.h new file mode 100644 index 0000000000000..3f267fb824077 --- /dev/null +++ b/ext/opcache/dynasm/dasm_ppc.h @@ -0,0 +1,419 @@ +/* +** DynASM PPC/PPC64 encoding engine. +** Copyright (C) 2005-2016 Mike Pall. All rights reserved. +** Released under the MIT license. See dynasm.lua for full copyright notice. +*/ + +#include +#include +#include +#include + +#define DASM_ARCH "ppc" + +#ifndef DASM_EXTERN +#define DASM_EXTERN(a,b,c,d) 0 +#endif + +/* Action definitions. */ +enum { + DASM_STOP, DASM_SECTION, DASM_ESC, DASM_REL_EXT, + /* The following actions need a buffer position. */ + DASM_ALIGN, DASM_REL_LG, DASM_LABEL_LG, + /* The following actions also have an argument. */ + DASM_REL_PC, DASM_LABEL_PC, DASM_IMM, DASM_IMMSH, + DASM__MAX +}; + +/* Maximum number of section buffer positions for a single dasm_put() call. */ +#define DASM_MAXSECPOS 25 + +/* DynASM encoder status codes. Action list offset or number are or'ed in. */ +#define DASM_S_OK 0x00000000 +#define DASM_S_NOMEM 0x01000000 +#define DASM_S_PHASE 0x02000000 +#define DASM_S_MATCH_SEC 0x03000000 +#define DASM_S_RANGE_I 0x11000000 +#define DASM_S_RANGE_SEC 0x12000000 +#define DASM_S_RANGE_LG 0x13000000 +#define DASM_S_RANGE_PC 0x14000000 +#define DASM_S_RANGE_REL 0x15000000 +#define DASM_S_UNDEF_LG 0x21000000 +#define DASM_S_UNDEF_PC 0x22000000 + +/* Macros to convert positions (8 bit section + 24 bit index). */ +#define DASM_POS2IDX(pos) ((pos)&0x00ffffff) +#define DASM_POS2BIAS(pos) ((pos)&0xff000000) +#define DASM_SEC2POS(sec) ((sec)<<24) +#define DASM_POS2SEC(pos) ((pos)>>24) +#define DASM_POS2PTR(D, pos) (D->sections[DASM_POS2SEC(pos)].rbuf + (pos)) + +/* Action list type. */ +typedef const unsigned int *dasm_ActList; + +/* Per-section structure. */ +typedef struct dasm_Section { + int *rbuf; /* Biased buffer pointer (negative section bias). */ + int *buf; /* True buffer pointer. */ + size_t bsize; /* Buffer size in bytes. */ + int pos; /* Biased buffer position. */ + int epos; /* End of biased buffer position - max single put. */ + int ofs; /* Byte offset into section. */ +} dasm_Section; + +/* Core structure holding the DynASM encoding state. */ +struct dasm_State { + size_t psize; /* Allocated size of this structure. */ + dasm_ActList actionlist; /* Current actionlist pointer. */ + int *lglabels; /* Local/global chain/pos ptrs. */ + size_t lgsize; + int *pclabels; /* PC label chains/pos ptrs. */ + size_t pcsize; + void **globals; /* Array of globals (bias -10). */ + dasm_Section *section; /* Pointer to active section. */ + size_t codesize; /* Total size of all code sections. */ + int maxsection; /* 0 <= sectionidx < maxsection. */ + int status; /* Status code. */ + dasm_Section sections[1]; /* All sections. Alloc-extended. */ +}; + +/* The size of the core structure depends on the max. number of sections. */ +#define DASM_PSZ(ms) (sizeof(dasm_State)+(ms-1)*sizeof(dasm_Section)) + + +/* Initialize DynASM state. */ +void dasm_init(Dst_DECL, int maxsection) +{ + dasm_State *D; + size_t psz = 0; + int i; + Dst_REF = NULL; + DASM_M_GROW(Dst, struct dasm_State, Dst_REF, psz, DASM_PSZ(maxsection)); + D = Dst_REF; + D->psize = psz; + D->lglabels = NULL; + D->lgsize = 0; + D->pclabels = NULL; + D->pcsize = 0; + D->globals = NULL; + D->maxsection = maxsection; + for (i = 0; i < maxsection; i++) { + D->sections[i].buf = NULL; /* Need this for pass3. */ + D->sections[i].rbuf = D->sections[i].buf - DASM_SEC2POS(i); + D->sections[i].bsize = 0; + D->sections[i].epos = 0; /* Wrong, but is recalculated after resize. */ + } +} + +/* Free DynASM state. */ +void dasm_free(Dst_DECL) +{ + dasm_State *D = Dst_REF; + int i; + for (i = 0; i < D->maxsection; i++) + if (D->sections[i].buf) + DASM_M_FREE(Dst, D->sections[i].buf, D->sections[i].bsize); + if (D->pclabels) DASM_M_FREE(Dst, D->pclabels, D->pcsize); + if (D->lglabels) DASM_M_FREE(Dst, D->lglabels, D->lgsize); + DASM_M_FREE(Dst, D, D->psize); +} + +/* Setup global label array. Must be called before dasm_setup(). */ +void dasm_setupglobal(Dst_DECL, void **gl, unsigned int maxgl) +{ + dasm_State *D = Dst_REF; + D->globals = gl - 10; /* Negative bias to compensate for locals. */ + DASM_M_GROW(Dst, int, D->lglabels, D->lgsize, (10+maxgl)*sizeof(int)); +} + +/* Grow PC label array. Can be called after dasm_setup(), too. */ +void dasm_growpc(Dst_DECL, unsigned int maxpc) +{ + dasm_State *D = Dst_REF; + size_t osz = D->pcsize; + DASM_M_GROW(Dst, int, D->pclabels, D->pcsize, maxpc*sizeof(int)); + memset((void *)(((unsigned char *)D->pclabels)+osz), 0, D->pcsize-osz); +} + +/* Setup encoder. */ +void dasm_setup(Dst_DECL, const void *actionlist) +{ + dasm_State *D = Dst_REF; + int i; + D->actionlist = (dasm_ActList)actionlist; + D->status = DASM_S_OK; + D->section = &D->sections[0]; + memset((void *)D->lglabels, 0, D->lgsize); + if (D->pclabels) memset((void *)D->pclabels, 0, D->pcsize); + for (i = 0; i < D->maxsection; i++) { + D->sections[i].pos = DASM_SEC2POS(i); + D->sections[i].ofs = 0; + } +} + + +#ifdef DASM_CHECKS +#define CK(x, st) \ + do { if (!(x)) { \ + D->status = DASM_S_##st|(p-D->actionlist-1); return; } } while (0) +#define CKPL(kind, st) \ + do { if ((size_t)((char *)pl-(char *)D->kind##labels) >= D->kind##size) { \ + D->status = DASM_S_RANGE_##st|(p-D->actionlist-1); return; } } while (0) +#else +#define CK(x, st) ((void)0) +#define CKPL(kind, st) ((void)0) +#endif + +/* Pass 1: Store actions and args, link branches/labels, estimate offsets. */ +void dasm_put(Dst_DECL, int start, ...) +{ + va_list ap; + dasm_State *D = Dst_REF; + dasm_ActList p = D->actionlist + start; + dasm_Section *sec = D->section; + int pos = sec->pos, ofs = sec->ofs; + int *b; + + if (pos >= sec->epos) { + DASM_M_GROW(Dst, int, sec->buf, sec->bsize, + sec->bsize + 2*DASM_MAXSECPOS*sizeof(int)); + sec->rbuf = sec->buf - DASM_POS2BIAS(pos); + sec->epos = (int)sec->bsize/sizeof(int) - DASM_MAXSECPOS+DASM_POS2BIAS(pos); + } + + b = sec->rbuf; + b[pos++] = start; + + va_start(ap, start); + while (1) { + unsigned int ins = *p++; + unsigned int action = (ins >> 16); + if (action >= DASM__MAX) { + ofs += 4; + } else { + int *pl, n = action >= DASM_REL_PC ? va_arg(ap, int) : 0; + switch (action) { + case DASM_STOP: goto stop; + case DASM_SECTION: + n = (ins & 255); CK(n < D->maxsection, RANGE_SEC); + D->section = &D->sections[n]; goto stop; + case DASM_ESC: p++; ofs += 4; break; + case DASM_REL_EXT: break; + case DASM_ALIGN: ofs += (ins & 255); b[pos++] = ofs; break; + case DASM_REL_LG: + n = (ins & 2047) - 10; pl = D->lglabels + n; + /* Bkwd rel or global. */ + if (n >= 0) { CK(n>=10||*pl<0, RANGE_LG); CKPL(lg, LG); goto putrel; } + pl += 10; n = *pl; + if (n < 0) n = 0; /* Start new chain for fwd rel if label exists. */ + goto linkrel; + case DASM_REL_PC: + pl = D->pclabels + n; CKPL(pc, PC); + putrel: + n = *pl; + if (n < 0) { /* Label exists. Get label pos and store it. */ + b[pos] = -n; + } else { + linkrel: + b[pos] = n; /* Else link to rel chain, anchored at label. */ + *pl = pos; + } + pos++; + break; + case DASM_LABEL_LG: + pl = D->lglabels + (ins & 2047) - 10; CKPL(lg, LG); goto putlabel; + case DASM_LABEL_PC: + pl = D->pclabels + n; CKPL(pc, PC); + putlabel: + n = *pl; /* n > 0: Collapse rel chain and replace with label pos. */ + while (n > 0) { int *pb = DASM_POS2PTR(D, n); n = *pb; *pb = pos; + } + *pl = -pos; /* Label exists now. */ + b[pos++] = ofs; /* Store pass1 offset estimate. */ + break; + case DASM_IMM: +#ifdef DASM_CHECKS + CK((n & ((1<<((ins>>10)&31))-1)) == 0, RANGE_I); +#endif + n >>= ((ins>>10)&31); +#ifdef DASM_CHECKS + if (ins & 0x8000) + CK(((n + (1<<(((ins>>5)&31)-1)))>>((ins>>5)&31)) == 0, RANGE_I); + else + CK((n>>((ins>>5)&31)) == 0, RANGE_I); +#endif + b[pos++] = n; + break; + case DASM_IMMSH: + CK((n >> 6) == 0, RANGE_I); + b[pos++] = n; + break; + } + } + } +stop: + va_end(ap); + sec->pos = pos; + sec->ofs = ofs; +} +#undef CK + +/* Pass 2: Link sections, shrink aligns, fix label offsets. */ +int dasm_link(Dst_DECL, size_t *szp) +{ + dasm_State *D = Dst_REF; + int secnum; + int ofs = 0; + +#ifdef DASM_CHECKS + *szp = 0; + if (D->status != DASM_S_OK) return D->status; + { + int pc; + for (pc = 0; pc*sizeof(int) < D->pcsize; pc++) + if (D->pclabels[pc] > 0) return DASM_S_UNDEF_PC|pc; + } +#endif + + { /* Handle globals not defined in this translation unit. */ + int idx; + for (idx = 20; idx*sizeof(int) < D->lgsize; idx++) { + int n = D->lglabels[idx]; + /* Undefined label: Collapse rel chain and replace with marker (< 0). */ + while (n > 0) { int *pb = DASM_POS2PTR(D, n); n = *pb; *pb = -idx; } + } + } + + /* Combine all code sections. No support for data sections (yet). */ + for (secnum = 0; secnum < D->maxsection; secnum++) { + dasm_Section *sec = D->sections + secnum; + int *b = sec->rbuf; + int pos = DASM_SEC2POS(secnum); + int lastpos = sec->pos; + + while (pos != lastpos) { + dasm_ActList p = D->actionlist + b[pos++]; + while (1) { + unsigned int ins = *p++; + unsigned int action = (ins >> 16); + switch (action) { + case DASM_STOP: case DASM_SECTION: goto stop; + case DASM_ESC: p++; break; + case DASM_REL_EXT: break; + case DASM_ALIGN: ofs -= (b[pos++] + ofs) & (ins & 255); break; + case DASM_REL_LG: case DASM_REL_PC: pos++; break; + case DASM_LABEL_LG: case DASM_LABEL_PC: b[pos++] += ofs; break; + case DASM_IMM: case DASM_IMMSH: pos++; break; + } + } + stop: (void)0; + } + ofs += sec->ofs; /* Next section starts right after current section. */ + } + + D->codesize = ofs; /* Total size of all code sections */ + *szp = ofs; + return DASM_S_OK; +} + +#ifdef DASM_CHECKS +#define CK(x, st) \ + do { if (!(x)) return DASM_S_##st|(p-D->actionlist-1); } while (0) +#else +#define CK(x, st) ((void)0) +#endif + +/* Pass 3: Encode sections. */ +int dasm_encode(Dst_DECL, void *buffer) +{ + dasm_State *D = Dst_REF; + char *base = (char *)buffer; + unsigned int *cp = (unsigned int *)buffer; + int secnum; + + /* Encode all code sections. No support for data sections (yet). */ + for (secnum = 0; secnum < D->maxsection; secnum++) { + dasm_Section *sec = D->sections + secnum; + int *b = sec->buf; + int *endb = sec->rbuf + sec->pos; + + while (b != endb) { + dasm_ActList p = D->actionlist + *b++; + while (1) { + unsigned int ins = *p++; + unsigned int action = (ins >> 16); + int n = (action >= DASM_ALIGN && action < DASM__MAX) ? *b++ : 0; + switch (action) { + case DASM_STOP: case DASM_SECTION: goto stop; + case DASM_ESC: *cp++ = *p++; break; + case DASM_REL_EXT: + n = DASM_EXTERN(Dst, (unsigned char *)cp, (ins & 2047), 1) - 4; + goto patchrel; + case DASM_ALIGN: + ins &= 255; while ((((char *)cp - base) & ins)) *cp++ = 0x60000000; + break; + case DASM_REL_LG: + CK(n >= 0, UNDEF_LG); + case DASM_REL_PC: + CK(n >= 0, UNDEF_PC); + n = *DASM_POS2PTR(D, n) - (int)((char *)cp - base); + patchrel: + CK((n & 3) == 0 && + (((n+4) + ((ins & 2048) ? 0x00008000 : 0x02000000)) >> + ((ins & 2048) ? 16 : 26)) == 0, RANGE_REL); + cp[-1] |= ((n+4) & ((ins & 2048) ? 0x0000fffc: 0x03fffffc)); + break; + case DASM_LABEL_LG: + ins &= 2047; if (ins >= 20) D->globals[ins-10] = (void *)(base + n); + break; + case DASM_LABEL_PC: break; + case DASM_IMM: + cp[-1] |= (n & ((1<<((ins>>5)&31))-1)) << (ins&31); + break; + case DASM_IMMSH: + cp[-1] |= (ins & 1) ? ((n&31)<<11)|((n&32)>>4) : ((n&31)<<6)|(n&32); + break; + default: *cp++ = ins; break; + } + } + stop: (void)0; + } + } + + if (base + D->codesize != (char *)cp) /* Check for phase errors. */ + return DASM_S_PHASE; + return DASM_S_OK; +} +#undef CK + +/* Get PC label offset. */ +int dasm_getpclabel(Dst_DECL, unsigned int pc) +{ + dasm_State *D = Dst_REF; + if (pc*sizeof(int) < D->pcsize) { + int pos = D->pclabels[pc]; + if (pos < 0) return *DASM_POS2PTR(D, -pos); + if (pos > 0) return -1; /* Undefined. */ + } + return -2; /* Unused or out of range. */ +} + +#ifdef DASM_CHECKS +/* Optional sanity checker to call between isolated encoding steps. */ +int dasm_checkstep(Dst_DECL, int secmatch) +{ + dasm_State *D = Dst_REF; + if (D->status == DASM_S_OK) { + int i; + for (i = 1; i <= 9; i++) { + if (D->lglabels[i] > 0) { D->status = DASM_S_UNDEF_LG|i; break; } + D->lglabels[i] = 0; + } + } + if (D->status == DASM_S_OK && secmatch >= 0 && + D->section != &D->sections[secmatch]) + D->status = DASM_S_MATCH_SEC|(D->section-D->sections); + return D->status; +} +#endif + diff --git a/ext/opcache/dynasm/dasm_ppc.lua b/ext/opcache/dynasm/dasm_ppc.lua new file mode 100644 index 0000000000000..e2f704ec18abc --- /dev/null +++ b/ext/opcache/dynasm/dasm_ppc.lua @@ -0,0 +1,1919 @@ +------------------------------------------------------------------------------ +-- DynASM PPC/PPC64 module. +-- +-- Copyright (C) 2005-2016 Mike Pall. All rights reserved. +-- See dynasm.lua for full copyright notice. +-- +-- Support for various extensions contributed by Caio Souza Oliveira. +------------------------------------------------------------------------------ + +-- Module information: +local _info = { + arch = "ppc", + description = "DynASM PPC module", + version = "1.4.0", + vernum = 10400, + release = "2015-10-18", + author = "Mike Pall", + license = "MIT", +} + +-- Exported glue functions for the arch-specific module. +local _M = { _info = _info } + +-- Cache library functions. +local type, tonumber, pairs, ipairs = type, tonumber, pairs, ipairs +local assert, setmetatable = assert, setmetatable +local _s = string +local sub, format, byte, char = _s.sub, _s.format, _s.byte, _s.char +local match, gmatch = _s.match, _s.gmatch +local concat, sort = table.concat, table.sort +local bit = bit or require("bit") +local band, shl, shr, sar = bit.band, bit.lshift, bit.rshift, bit.arshift +local tohex = bit.tohex + +-- Inherited tables and callbacks. +local g_opt, g_arch +local wline, werror, wfatal, wwarn + +-- Action name list. +-- CHECK: Keep this in sync with the C code! +local action_names = { + "STOP", "SECTION", "ESC", "REL_EXT", + "ALIGN", "REL_LG", "LABEL_LG", + "REL_PC", "LABEL_PC", "IMM", "IMMSH" +} + +-- Maximum number of section buffer positions for dasm_put(). +-- CHECK: Keep this in sync with the C code! +local maxsecpos = 25 -- Keep this low, to avoid excessively long C lines. + +-- Action name -> action number. +local map_action = {} +for n,name in ipairs(action_names) do + map_action[name] = n-1 +end + +-- Action list buffer. +local actlist = {} + +-- Argument list for next dasm_put(). Start with offset 0 into action list. +local actargs = { 0 } + +-- Current number of section buffer positions for dasm_put(). +local secpos = 1 + +------------------------------------------------------------------------------ + +-- Dump action names and numbers. +local function dumpactions(out) + out:write("DynASM encoding engine action codes:\n") + for n,name in ipairs(action_names) do + local num = map_action[name] + out:write(format(" %-10s %02X %d\n", name, num, num)) + end + out:write("\n") +end + +-- Write action list buffer as a huge static C array. +local function writeactions(out, name) + local nn = #actlist + if nn == 0 then nn = 1; actlist[0] = map_action.STOP end + out:write("static const unsigned int ", name, "[", nn, "] = {\n") + for i = 1,nn-1 do + assert(out:write("0x", tohex(actlist[i]), ",\n")) + end + assert(out:write("0x", tohex(actlist[nn]), "\n};\n\n")) +end + +------------------------------------------------------------------------------ + +-- Add word to action list. +local function wputxw(n) + assert(n >= 0 and n <= 0xffffffff and n % 1 == 0, "word out of range") + actlist[#actlist+1] = n +end + +-- Add action to list with optional arg. Advance buffer pos, too. +local function waction(action, val, a, num) + local w = assert(map_action[action], "bad action name `"..action.."'") + wputxw(w * 0x10000 + (val or 0)) + if a then actargs[#actargs+1] = a end + if a or num then secpos = secpos + (num or 1) end +end + +-- Flush action list (intervening C code or buffer pos overflow). +local function wflush(term) + if #actlist == actargs[1] then return end -- Nothing to flush. + if not term then waction("STOP") end -- Terminate action list. + wline(format("dasm_put(Dst, %s);", concat(actargs, ", ")), true) + actargs = { #actlist } -- Actionlist offset is 1st arg to next dasm_put(). + secpos = 1 -- The actionlist offset occupies a buffer position, too. +end + +-- Put escaped word. +local function wputw(n) + if n <= 0xffffff then waction("ESC") end + wputxw(n) +end + +-- Reserve position for word. +local function wpos() + local pos = #actlist+1 + actlist[pos] = "" + return pos +end + +-- Store word to reserved position. +local function wputpos(pos, n) + assert(n >= 0 and n <= 0xffffffff and n % 1 == 0, "word out of range") + actlist[pos] = n +end + +------------------------------------------------------------------------------ + +-- Global label name -> global label number. With auto assignment on 1st use. +local next_global = 20 +local map_global = setmetatable({}, { __index = function(t, name) + if not match(name, "^[%a_][%w_]*$") then werror("bad global label") end + local n = next_global + if n > 2047 then werror("too many global labels") end + next_global = n + 1 + t[name] = n + return n +end}) + +-- Dump global labels. +local function dumpglobals(out, lvl) + local t = {} + for name, n in pairs(map_global) do t[n] = name end + out:write("Global labels:\n") + for i=20,next_global-1 do + out:write(format(" %s\n", t[i])) + end + out:write("\n") +end + +-- Write global label enum. +local function writeglobals(out, prefix) + local t = {} + for name, n in pairs(map_global) do t[n] = name end + out:write("enum {\n") + for i=20,next_global-1 do + out:write(" ", prefix, t[i], ",\n") + end + out:write(" ", prefix, "_MAX\n};\n") +end + +-- Write global label names. +local function writeglobalnames(out, name) + local t = {} + for name, n in pairs(map_global) do t[n] = name end + out:write("static const char *const ", name, "[] = {\n") + for i=20,next_global-1 do + out:write(" \"", t[i], "\",\n") + end + out:write(" (const char *)0\n};\n") +end + +------------------------------------------------------------------------------ + +-- Extern label name -> extern label number. With auto assignment on 1st use. +local next_extern = 0 +local map_extern_ = {} +local map_extern = setmetatable({}, { __index = function(t, name) + -- No restrictions on the name for now. + local n = next_extern + if n > 2047 then werror("too many extern labels") end + next_extern = n + 1 + t[name] = n + map_extern_[n] = name + return n +end}) + +-- Dump extern labels. +local function dumpexterns(out, lvl) + out:write("Extern labels:\n") + for i=0,next_extern-1 do + out:write(format(" %s\n", map_extern_[i])) + end + out:write("\n") +end + +-- Write extern label names. +local function writeexternnames(out, name) + out:write("static const char *const ", name, "[] = {\n") + for i=0,next_extern-1 do + out:write(" \"", map_extern_[i], "\",\n") + end + out:write(" (const char *)0\n};\n") +end + +------------------------------------------------------------------------------ + +-- Arch-specific maps. +local map_archdef = { sp = "r1" } -- Ext. register name -> int. name. + +local map_type = {} -- Type name -> { ctype, reg } +local ctypenum = 0 -- Type number (for Dt... macros). + +-- Reverse defines for registers. +function _M.revdef(s) + if s == "r1" then return "sp" end + return s +end + +local map_cond = { + lt = 0, gt = 1, eq = 2, so = 3, + ge = 4, le = 5, ne = 6, ns = 7, +} + +------------------------------------------------------------------------------ + +local map_op, op_template + +local function op_alias(opname, f) + return function(params, nparams) + if not params then return "-> "..opname:sub(1, -3) end + f(params, nparams) + op_template(params, map_op[opname], nparams) + end +end + +-- Template strings for PPC instructions. +map_op = { + tdi_3 = "08000000ARI", + twi_3 = "0c000000ARI", + mulli_3 = "1c000000RRI", + subfic_3 = "20000000RRI", + cmplwi_3 = "28000000XRU", + cmplwi_2 = "28000000-RU", + cmpldi_3 = "28200000XRU", + cmpldi_2 = "28200000-RU", + cmpwi_3 = "2c000000XRI", + cmpwi_2 = "2c000000-RI", + cmpdi_3 = "2c200000XRI", + cmpdi_2 = "2c200000-RI", + addic_3 = "30000000RRI", + ["addic._3"] = "34000000RRI", + addi_3 = "38000000RR0I", + li_2 = "38000000RI", + la_2 = "38000000RD", + addis_3 = "3c000000RR0I", + lis_2 = "3c000000RI", + lus_2 = "3c000000RU", + bc_3 = "40000000AAK", + bcl_3 = "40000001AAK", + bdnz_1 = "42000000K", + bdz_1 = "42400000K", + sc_0 = "44000000", + b_1 = "48000000J", + bl_1 = "48000001J", + rlwimi_5 = "50000000RR~AAA.", + rlwinm_5 = "54000000RR~AAA.", + rlwnm_5 = "5c000000RR~RAA.", + ori_3 = "60000000RR~U", + nop_0 = "60000000", + oris_3 = "64000000RR~U", + xori_3 = "68000000RR~U", + xoris_3 = "6c000000RR~U", + ["andi._3"] = "70000000RR~U", + ["andis._3"] = "74000000RR~U", + lwz_2 = "80000000RD", + lwzu_2 = "84000000RD", + lbz_2 = "88000000RD", + lbzu_2 = "8c000000RD", + stw_2 = "90000000RD", + stwu_2 = "94000000RD", + stb_2 = "98000000RD", + stbu_2 = "9c000000RD", + lhz_2 = "a0000000RD", + lhzu_2 = "a4000000RD", + lha_2 = "a8000000RD", + lhau_2 = "ac000000RD", + sth_2 = "b0000000RD", + sthu_2 = "b4000000RD", + lmw_2 = "b8000000RD", + stmw_2 = "bc000000RD", + lfs_2 = "c0000000FD", + lfsu_2 = "c4000000FD", + lfd_2 = "c8000000FD", + lfdu_2 = "cc000000FD", + stfs_2 = "d0000000FD", + stfsu_2 = "d4000000FD", + stfd_2 = "d8000000FD", + stfdu_2 = "dc000000FD", + ld_2 = "e8000000RD", -- NYI: displacement must be divisible by 4. + ldu_2 = "e8000001RD", + lwa_2 = "e8000002RD", + std_2 = "f8000000RD", + stdu_2 = "f8000001RD", + + subi_3 = op_alias("addi_3", function(p) p[3] = "-("..p[3]..")" end), + subis_3 = op_alias("addis_3", function(p) p[3] = "-("..p[3]..")" end), + subic_3 = op_alias("addic_3", function(p) p[3] = "-("..p[3]..")" end), + ["subic._3"] = op_alias("addic._3", function(p) p[3] = "-("..p[3]..")" end), + + rotlwi_3 = op_alias("rlwinm_5", function(p) + p[4] = "0"; p[5] = "31" + end), + rotrwi_3 = op_alias("rlwinm_5", function(p) + p[3] = "32-("..p[3]..")"; p[4] = "0"; p[5] = "31" + end), + rotlw_3 = op_alias("rlwnm_5", function(p) + p[4] = "0"; p[5] = "31" + end), + slwi_3 = op_alias("rlwinm_5", function(p) + p[5] = "31-("..p[3]..")"; p[4] = "0" + end), + srwi_3 = op_alias("rlwinm_5", function(p) + p[4] = p[3]; p[3] = "32-("..p[3]..")"; p[5] = "31" + end), + clrlwi_3 = op_alias("rlwinm_5", function(p) + p[4] = p[3]; p[3] = "0"; p[5] = "31" + end), + clrrwi_3 = op_alias("rlwinm_5", function(p) + p[5] = "31-("..p[3]..")"; p[3] = "0"; p[4] = "0" + end), + + -- Primary opcode 4: + mulhhwu_3 = "10000010RRR.", + machhwu_3 = "10000018RRR.", + mulhhw_3 = "10000050RRR.", + nmachhw_3 = "1000005cRRR.", + machhwsu_3 = "10000098RRR.", + machhws_3 = "100000d8RRR.", + nmachhws_3 = "100000dcRRR.", + mulchwu_3 = "10000110RRR.", + macchwu_3 = "10000118RRR.", + mulchw_3 = "10000150RRR.", + macchw_3 = "10000158RRR.", + nmacchw_3 = "1000015cRRR.", + macchwsu_3 = "10000198RRR.", + macchws_3 = "100001d8RRR.", + nmacchws_3 = "100001dcRRR.", + mullhw_3 = "10000350RRR.", + maclhw_3 = "10000358RRR.", + nmaclhw_3 = "1000035cRRR.", + maclhwsu_3 = "10000398RRR.", + maclhws_3 = "100003d8RRR.", + nmaclhws_3 = "100003dcRRR.", + machhwuo_3 = "10000418RRR.", + nmachhwo_3 = "1000045cRRR.", + machhwsuo_3 = "10000498RRR.", + machhwso_3 = "100004d8RRR.", + nmachhwso_3 = "100004dcRRR.", + macchwuo_3 = "10000518RRR.", + macchwo_3 = "10000558RRR.", + nmacchwo_3 = "1000055cRRR.", + macchwsuo_3 = "10000598RRR.", + macchwso_3 = "100005d8RRR.", + nmacchwso_3 = "100005dcRRR.", + maclhwo_3 = "10000758RRR.", + nmaclhwo_3 = "1000075cRRR.", + maclhwsuo_3 = "10000798RRR.", + maclhwso_3 = "100007d8RRR.", + nmaclhwso_3 = "100007dcRRR.", + + vaddubm_3 = "10000000VVV", + vmaxub_3 = "10000002VVV", + vrlb_3 = "10000004VVV", + vcmpequb_3 = "10000006VVV", + vmuloub_3 = "10000008VVV", + vaddfp_3 = "1000000aVVV", + vmrghb_3 = "1000000cVVV", + vpkuhum_3 = "1000000eVVV", + vmhaddshs_4 = "10000020VVVV", + vmhraddshs_4 = "10000021VVVV", + vmladduhm_4 = "10000022VVVV", + vmsumubm_4 = "10000024VVVV", + vmsummbm_4 = "10000025VVVV", + vmsumuhm_4 = "10000026VVVV", + vmsumuhs_4 = "10000027VVVV", + vmsumshm_4 = "10000028VVVV", + vmsumshs_4 = "10000029VVVV", + vsel_4 = "1000002aVVVV", + vperm_4 = "1000002bVVVV", + vsldoi_4 = "1000002cVVVP", + vpermxor_4 = "1000002dVVVV", + vmaddfp_4 = "1000002eVVVV~", + vnmsubfp_4 = "1000002fVVVV~", + vaddeuqm_4 = "1000003cVVVV", + vaddecuq_4 = "1000003dVVVV", + vsubeuqm_4 = "1000003eVVVV", + vsubecuq_4 = "1000003fVVVV", + vadduhm_3 = "10000040VVV", + vmaxuh_3 = "10000042VVV", + vrlh_3 = "10000044VVV", + vcmpequh_3 = "10000046VVV", + vmulouh_3 = "10000048VVV", + vsubfp_3 = "1000004aVVV", + vmrghh_3 = "1000004cVVV", + vpkuwum_3 = "1000004eVVV", + vadduwm_3 = "10000080VVV", + vmaxuw_3 = "10000082VVV", + vrlw_3 = "10000084VVV", + vcmpequw_3 = "10000086VVV", + vmulouw_3 = "10000088VVV", + vmuluwm_3 = "10000089VVV", + vmrghw_3 = "1000008cVVV", + vpkuhus_3 = "1000008eVVV", + vaddudm_3 = "100000c0VVV", + vmaxud_3 = "100000c2VVV", + vrld_3 = "100000c4VVV", + vcmpeqfp_3 = "100000c6VVV", + vcmpequd_3 = "100000c7VVV", + vpkuwus_3 = "100000ceVVV", + vadduqm_3 = "10000100VVV", + vmaxsb_3 = "10000102VVV", + vslb_3 = "10000104VVV", + vmulosb_3 = "10000108VVV", + vrefp_2 = "1000010aV-V", + vmrglb_3 = "1000010cVVV", + vpkshus_3 = "1000010eVVV", + vaddcuq_3 = "10000140VVV", + vmaxsh_3 = "10000142VVV", + vslh_3 = "10000144VVV", + vmulosh_3 = "10000148VVV", + vrsqrtefp_2 = "1000014aV-V", + vmrglh_3 = "1000014cVVV", + vpkswus_3 = "1000014eVVV", + vaddcuw_3 = "10000180VVV", + vmaxsw_3 = "10000182VVV", + vslw_3 = "10000184VVV", + vmulosw_3 = "10000188VVV", + vexptefp_2 = "1000018aV-V", + vmrglw_3 = "1000018cVVV", + vpkshss_3 = "1000018eVVV", + vmaxsd_3 = "100001c2VVV", + vsl_3 = "100001c4VVV", + vcmpgefp_3 = "100001c6VVV", + vlogefp_2 = "100001caV-V", + vpkswss_3 = "100001ceVVV", + vadduhs_3 = "10000240VVV", + vminuh_3 = "10000242VVV", + vsrh_3 = "10000244VVV", + vcmpgtuh_3 = "10000246VVV", + vmuleuh_3 = "10000248VVV", + vrfiz_2 = "1000024aV-V", + vsplth_3 = "1000024cVV3", + vupkhsh_2 = "1000024eV-V", + vminuw_3 = "10000282VVV", + vminud_3 = "100002c2VVV", + vcmpgtud_3 = "100002c7VVV", + vrfim_2 = "100002caV-V", + vcmpgtsb_3 = "10000306VVV", + vcfux_3 = "1000030aVVA~", + vaddshs_3 = "10000340VVV", + vminsh_3 = "10000342VVV", + vsrah_3 = "10000344VVV", + vcmpgtsh_3 = "10000346VVV", + vmulesh_3 = "10000348VVV", + vcfsx_3 = "1000034aVVA~", + vspltish_2 = "1000034cVS", + vupkhpx_2 = "1000034eV-V", + vaddsws_3 = "10000380VVV", + vminsw_3 = "10000382VVV", + vsraw_3 = "10000384VVV", + vcmpgtsw_3 = "10000386VVV", + vmulesw_3 = "10000388VVV", + vctuxs_3 = "1000038aVVA~", + vspltisw_2 = "1000038cVS", + vminsd_3 = "100003c2VVV", + vsrad_3 = "100003c4VVV", + vcmpbfp_3 = "100003c6VVV", + vcmpgtsd_3 = "100003c7VVV", + vctsxs_3 = "100003caVVA~", + vupklpx_2 = "100003ceV-V", + vsububm_3 = "10000400VVV", + ["bcdadd._4"] = "10000401VVVy.", + vavgub_3 = "10000402VVV", + vand_3 = "10000404VVV", + ["vcmpequb._3"] = "10000406VVV", + vmaxfp_3 = "1000040aVVV", + vsubuhm_3 = "10000440VVV", + ["bcdsub._4"] = "10000441VVVy.", + vavguh_3 = "10000442VVV", + vandc_3 = "10000444VVV", + ["vcmpequh._3"] = "10000446VVV", + vminfp_3 = "1000044aVVV", + vpkudum_3 = "1000044eVVV", + vsubuwm_3 = "10000480VVV", + vavguw_3 = "10000482VVV", + vor_3 = "10000484VVV", + ["vcmpequw._3"] = "10000486VVV", + vpmsumw_3 = "10000488VVV", + ["vcmpeqfp._3"] = "100004c6VVV", + ["vcmpequd._3"] = "100004c7VVV", + vpkudus_3 = "100004ceVVV", + vavgsb_3 = "10000502VVV", + vavgsh_3 = "10000542VVV", + vorc_3 = "10000544VVV", + vbpermq_3 = "1000054cVVV", + vpksdus_3 = "1000054eVVV", + vavgsw_3 = "10000582VVV", + vsld_3 = "100005c4VVV", + ["vcmpgefp._3"] = "100005c6VVV", + vpksdss_3 = "100005ceVVV", + vsububs_3 = "10000600VVV", + mfvscr_1 = "10000604V--", + vsum4ubs_3 = "10000608VVV", + vsubuhs_3 = "10000640VVV", + mtvscr_1 = "10000644--V", + ["vcmpgtuh._3"] = "10000646VVV", + vsum4shs_3 = "10000648VVV", + vupkhsw_2 = "1000064eV-V", + vsubuws_3 = "10000680VVV", + vshasigmaw_4 = "10000682VVYp", + veqv_3 = "10000684VVV", + vsum2sws_3 = "10000688VVV", + vmrgow_3 = "1000068cVVV", + vshasigmad_4 = "100006c2VVYp", + vsrd_3 = "100006c4VVV", + ["vcmpgtud._3"] = "100006c7VVV", + vupklsw_2 = "100006ceV-V", + vupkslw_2 = "100006ceV-V", + vsubsbs_3 = "10000700VVV", + vclzb_2 = "10000702V-V", + vpopcntb_2 = "10000703V-V", + ["vcmpgtsb._3"] = "10000706VVV", + vsum4sbs_3 = "10000708VVV", + vsubshs_3 = "10000740VVV", + vclzh_2 = "10000742V-V", + vpopcnth_2 = "10000743V-V", + ["vcmpgtsh._3"] = "10000746VVV", + vsubsws_3 = "10000780VVV", + vclzw_2 = "10000782V-V", + vpopcntw_2 = "10000783V-V", + ["vcmpgtsw._3"] = "10000786VVV", + vsumsws_3 = "10000788VVV", + vmrgew_3 = "1000078cVVV", + vclzd_2 = "100007c2V-V", + vpopcntd_2 = "100007c3V-V", + ["vcmpbfp._3"] = "100007c6VVV", + ["vcmpgtsd._3"] = "100007c7VVV", + + -- Primary opcode 19: + mcrf_2 = "4c000000XX", + isync_0 = "4c00012c", + crnor_3 = "4c000042CCC", + crnot_2 = "4c000042CC=", + crandc_3 = "4c000102CCC", + crxor_3 = "4c000182CCC", + crclr_1 = "4c000182C==", + crnand_3 = "4c0001c2CCC", + crand_3 = "4c000202CCC", + creqv_3 = "4c000242CCC", + crset_1 = "4c000242C==", + crorc_3 = "4c000342CCC", + cror_3 = "4c000382CCC", + crmove_2 = "4c000382CC=", + bclr_2 = "4c000020AA", + bclrl_2 = "4c000021AA", + bcctr_2 = "4c000420AA", + bcctrl_2 = "4c000421AA", + bctar_2 = "4c000460AA", + bctarl_2 = "4c000461AA", + blr_0 = "4e800020", + blrl_0 = "4e800021", + bctr_0 = "4e800420", + bctrl_0 = "4e800421", + + -- Primary opcode 31: + cmpw_3 = "7c000000XRR", + cmpw_2 = "7c000000-RR", + cmpd_3 = "7c200000XRR", + cmpd_2 = "7c200000-RR", + tw_3 = "7c000008ARR", + lvsl_3 = "7c00000cVRR", + subfc_3 = "7c000010RRR.", + subc_3 = "7c000010RRR~.", + mulhdu_3 = "7c000012RRR.", + addc_3 = "7c000014RRR.", + mulhwu_3 = "7c000016RRR.", + isel_4 = "7c00001eRRRC", + isellt_3 = "7c00001eRRR", + iselgt_3 = "7c00005eRRR", + iseleq_3 = "7c00009eRRR", + mfcr_1 = "7c000026R", + mfocrf_2 = "7c100026RG", + mtcrf_2 = "7c000120GR", + mtocrf_2 = "7c100120GR", + lwarx_3 = "7c000028RR0R", + ldx_3 = "7c00002aRR0R", + lwzx_3 = "7c00002eRR0R", + slw_3 = "7c000030RR~R.", + cntlzw_2 = "7c000034RR~", + sld_3 = "7c000036RR~R.", + and_3 = "7c000038RR~R.", + cmplw_3 = "7c000040XRR", + cmplw_2 = "7c000040-RR", + cmpld_3 = "7c200040XRR", + cmpld_2 = "7c200040-RR", + lvsr_3 = "7c00004cVRR", + subf_3 = "7c000050RRR.", + sub_3 = "7c000050RRR~.", + lbarx_3 = "7c000068RR0R", + ldux_3 = "7c00006aRR0R", + dcbst_2 = "7c00006c-RR", + lwzux_3 = "7c00006eRR0R", + cntlzd_2 = "7c000074RR~", + andc_3 = "7c000078RR~R.", + td_3 = "7c000088ARR", + lvewx_3 = "7c00008eVRR", + mulhd_3 = "7c000092RRR.", + addg6s_3 = "7c000094RRR", + mulhw_3 = "7c000096RRR.", + dlmzb_3 = "7c00009cRR~R.", + ldarx_3 = "7c0000a8RR0R", + dcbf_2 = "7c0000ac-RR", + lbzx_3 = "7c0000aeRR0R", + lvx_3 = "7c0000ceVRR", + neg_2 = "7c0000d0RR.", + lharx_3 = "7c0000e8RR0R", + lbzux_3 = "7c0000eeRR0R", + popcntb_2 = "7c0000f4RR~", + not_2 = "7c0000f8RR~%.", + nor_3 = "7c0000f8RR~R.", + stvebx_3 = "7c00010eVRR", + subfe_3 = "7c000110RRR.", + sube_3 = "7c000110RRR~.", + adde_3 = "7c000114RRR.", + stdx_3 = "7c00012aRR0R", + ["stwcx._3"] = "7c00012dRR0R.", + stwx_3 = "7c00012eRR0R", + prtyw_2 = "7c000134RR~", + stvehx_3 = "7c00014eVRR", + stdux_3 = "7c00016aRR0R", + ["stqcx._3"] = "7c00016dR:R0R.", + stwux_3 = "7c00016eRR0R", + prtyd_2 = "7c000174RR~", + stvewx_3 = "7c00018eVRR", + subfze_2 = "7c000190RR.", + addze_2 = "7c000194RR.", + ["stdcx._3"] = "7c0001adRR0R.", + stbx_3 = "7c0001aeRR0R", + stvx_3 = "7c0001ceVRR", + subfme_2 = "7c0001d0RR.", + mulld_3 = "7c0001d2RRR.", + addme_2 = "7c0001d4RR.", + mullw_3 = "7c0001d6RRR.", + dcbtst_2 = "7c0001ec-RR", + stbux_3 = "7c0001eeRR0R", + bpermd_3 = "7c0001f8RR~R", + lvepxl_3 = "7c00020eVRR", + add_3 = "7c000214RRR.", + lqarx_3 = "7c000228R:R0R", + dcbt_2 = "7c00022c-RR", + lhzx_3 = "7c00022eRR0R", + cdtbcd_2 = "7c000234RR~", + eqv_3 = "7c000238RR~R.", + lvepx_3 = "7c00024eVRR", + eciwx_3 = "7c00026cRR0R", + lhzux_3 = "7c00026eRR0R", + cbcdtd_2 = "7c000274RR~", + xor_3 = "7c000278RR~R.", + mfspefscr_1 = "7c0082a6R", + mfxer_1 = "7c0102a6R", + mflr_1 = "7c0802a6R", + mfctr_1 = "7c0902a6R", + lwax_3 = "7c0002aaRR0R", + lhax_3 = "7c0002aeRR0R", + mftb_1 = "7c0c42e6R", + mftbu_1 = "7c0d42e6R", + lvxl_3 = "7c0002ceVRR", + lwaux_3 = "7c0002eaRR0R", + lhaux_3 = "7c0002eeRR0R", + popcntw_2 = "7c0002f4RR~", + divdeu_3 = "7c000312RRR.", + divweu_3 = "7c000316RRR.", + sthx_3 = "7c00032eRR0R", + orc_3 = "7c000338RR~R.", + ecowx_3 = "7c00036cRR0R", + sthux_3 = "7c00036eRR0R", + or_3 = "7c000378RR~R.", + mr_2 = "7c000378RR~%.", + divdu_3 = "7c000392RRR.", + divwu_3 = "7c000396RRR.", + mtspefscr_1 = "7c0083a6R", + mtxer_1 = "7c0103a6R", + mtlr_1 = "7c0803a6R", + mtctr_1 = "7c0903a6R", + dcbi_2 = "7c0003ac-RR", + nand_3 = "7c0003b8RR~R.", + dsn_2 = "7c0003c6-RR", + stvxl_3 = "7c0003ceVRR", + divd_3 = "7c0003d2RRR.", + divw_3 = "7c0003d6RRR.", + popcntd_2 = "7c0003f4RR~", + cmpb_3 = "7c0003f8RR~R.", + mcrxr_1 = "7c000400X", + lbdx_3 = "7c000406RRR", + subfco_3 = "7c000410RRR.", + subco_3 = "7c000410RRR~.", + addco_3 = "7c000414RRR.", + ldbrx_3 = "7c000428RR0R", + lswx_3 = "7c00042aRR0R", + lwbrx_3 = "7c00042cRR0R", + lfsx_3 = "7c00042eFR0R", + srw_3 = "7c000430RR~R.", + srd_3 = "7c000436RR~R.", + lhdx_3 = "7c000446RRR", + subfo_3 = "7c000450RRR.", + subo_3 = "7c000450RRR~.", + lfsux_3 = "7c00046eFR0R", + lwdx_3 = "7c000486RRR", + lswi_3 = "7c0004aaRR0A", + sync_0 = "7c0004ac", + lwsync_0 = "7c2004ac", + ptesync_0 = "7c4004ac", + lfdx_3 = "7c0004aeFR0R", + lddx_3 = "7c0004c6RRR", + nego_2 = "7c0004d0RR.", + lfdux_3 = "7c0004eeFR0R", + stbdx_3 = "7c000506RRR", + subfeo_3 = "7c000510RRR.", + subeo_3 = "7c000510RRR~.", + addeo_3 = "7c000514RRR.", + stdbrx_3 = "7c000528RR0R", + stswx_3 = "7c00052aRR0R", + stwbrx_3 = "7c00052cRR0R", + stfsx_3 = "7c00052eFR0R", + sthdx_3 = "7c000546RRR", + ["stbcx._3"] = "7c00056dRRR", + stfsux_3 = "7c00056eFR0R", + stwdx_3 = "7c000586RRR", + subfzeo_2 = "7c000590RR.", + addzeo_2 = "7c000594RR.", + stswi_3 = "7c0005aaRR0A", + ["sthcx._3"] = "7c0005adRRR", + stfdx_3 = "7c0005aeFR0R", + stddx_3 = "7c0005c6RRR", + subfmeo_2 = "7c0005d0RR.", + mulldo_3 = "7c0005d2RRR.", + addmeo_2 = "7c0005d4RR.", + mullwo_3 = "7c0005d6RRR.", + dcba_2 = "7c0005ec-RR", + stfdux_3 = "7c0005eeFR0R", + stvepxl_3 = "7c00060eVRR", + addo_3 = "7c000614RRR.", + lhbrx_3 = "7c00062cRR0R", + lfdpx_3 = "7c00062eF:RR", + sraw_3 = "7c000630RR~R.", + srad_3 = "7c000634RR~R.", + lfddx_3 = "7c000646FRR", + stvepx_3 = "7c00064eVRR", + srawi_3 = "7c000670RR~A.", + sradi_3 = "7c000674RR~H.", + eieio_0 = "7c0006ac", + lfiwax_3 = "7c0006aeFR0R", + divdeuo_3 = "7c000712RRR.", + divweuo_3 = "7c000716RRR.", + sthbrx_3 = "7c00072cRR0R", + stfdpx_3 = "7c00072eF:RR", + extsh_2 = "7c000734RR~.", + stfddx_3 = "7c000746FRR", + divdeo_3 = "7c000752RRR.", + divweo_3 = "7c000756RRR.", + extsb_2 = "7c000774RR~.", + divduo_3 = "7c000792RRR.", + divwou_3 = "7c000796RRR.", + icbi_2 = "7c0007ac-RR", + stfiwx_3 = "7c0007aeFR0R", + extsw_2 = "7c0007b4RR~.", + divdo_3 = "7c0007d2RRR.", + divwo_3 = "7c0007d6RRR.", + dcbz_2 = "7c0007ec-RR", + + ["tbegin._1"] = "7c00051d1", + ["tbegin._0"] = "7c00051d", + ["tend._1"] = "7c00055dY", + ["tend._0"] = "7c00055d", + ["tendall._0"] = "7e00055d", + tcheck_1 = "7c00059cX", + ["tsr._1"] = "7c0005dd1", + ["tsuspend._0"] = "7c0005dd", + ["tresume._0"] = "7c2005dd", + ["tabortwc._3"] = "7c00061dARR", + ["tabortdc._3"] = "7c00065dARR", + ["tabortwci._3"] = "7c00069dARS", + ["tabortdci._3"] = "7c0006ddARS", + ["tabort._1"] = "7c00071d-R-", + ["treclaim._1"] = "7c00075d-R", + ["trechkpt._0"] = "7c0007dd", + + lxsiwzx_3 = "7c000018QRR", + lxsiwax_3 = "7c000098QRR", + mfvsrd_2 = "7c000066-Rq", + mfvsrwz_2 = "7c0000e6-Rq", + stxsiwx_3 = "7c000118QRR", + mtvsrd_2 = "7c000166QR", + mtvsrwa_2 = "7c0001a6QR", + lxvdsx_3 = "7c000298QRR", + lxsspx_3 = "7c000418QRR", + lxsdx_3 = "7c000498QRR", + stxsspx_3 = "7c000518QRR", + stxsdx_3 = "7c000598QRR", + lxvw4x_3 = "7c000618QRR", + lxvd2x_3 = "7c000698QRR", + stxvw4x_3 = "7c000718QRR", + stxvd2x_3 = "7c000798QRR", + + -- Primary opcode 30: + rldicl_4 = "78000000RR~HM.", + rldicr_4 = "78000004RR~HM.", + rldic_4 = "78000008RR~HM.", + rldimi_4 = "7800000cRR~HM.", + rldcl_4 = "78000010RR~RM.", + rldcr_4 = "78000012RR~RM.", + + rotldi_3 = op_alias("rldicl_4", function(p) + p[4] = "0" + end), + rotrdi_3 = op_alias("rldicl_4", function(p) + p[3] = "64-("..p[3]..")"; p[4] = "0" + end), + rotld_3 = op_alias("rldcl_4", function(p) + p[4] = "0" + end), + sldi_3 = op_alias("rldicr_4", function(p) + p[4] = "63-("..p[3]..")" + end), + srdi_3 = op_alias("rldicl_4", function(p) + p[4] = p[3]; p[3] = "64-("..p[3]..")" + end), + clrldi_3 = op_alias("rldicl_4", function(p) + p[4] = p[3]; p[3] = "0" + end), + clrrdi_3 = op_alias("rldicr_4", function(p) + p[4] = "63-("..p[3]..")"; p[3] = "0" + end), + + -- Primary opcode 56: + lq_2 = "e0000000R:D", -- NYI: displacement must be divisible by 8. + + -- Primary opcode 57: + lfdp_2 = "e4000000F:D", -- NYI: displacement must be divisible by 4. + + -- Primary opcode 59: + fdivs_3 = "ec000024FFF.", + fsubs_3 = "ec000028FFF.", + fadds_3 = "ec00002aFFF.", + fsqrts_2 = "ec00002cF-F.", + fres_2 = "ec000030F-F.", + fmuls_3 = "ec000032FF-F.", + frsqrtes_2 = "ec000034F-F.", + fmsubs_4 = "ec000038FFFF~.", + fmadds_4 = "ec00003aFFFF~.", + fnmsubs_4 = "ec00003cFFFF~.", + fnmadds_4 = "ec00003eFFFF~.", + fcfids_2 = "ec00069cF-F.", + fcfidus_2 = "ec00079cF-F.", + + dadd_3 = "ec000004FFF.", + dqua_4 = "ec000006FFFZ.", + dmul_3 = "ec000044FFF.", + drrnd_4 = "ec000046FFFZ.", + dscli_3 = "ec000084FF6.", + dquai_4 = "ec000086SF~FZ.", + dscri_3 = "ec0000c4FF6.", + drintx_4 = "ec0000c61F~FZ.", + dcmpo_3 = "ec000104XFF", + dtstex_3 = "ec000144XFF", + dtstdc_3 = "ec000184XF6", + dtstdg_3 = "ec0001c4XF6", + drintn_4 = "ec0001c61F~FZ.", + dctdp_2 = "ec000204F-F.", + dctfix_2 = "ec000244F-F.", + ddedpd_3 = "ec000284ZF~F.", + dxex_2 = "ec0002c4F-F.", + dsub_3 = "ec000404FFF.", + ddiv_3 = "ec000444FFF.", + dcmpu_3 = "ec000504XFF", + dtstsf_3 = "ec000544XFF", + drsp_2 = "ec000604F-F.", + dcffix_2 = "ec000644F-F.", + denbcd_3 = "ec000684YF~F.", + diex_3 = "ec0006c4FFF.", + + -- Primary opcode 60: + xsaddsp_3 = "f0000000QQQ", + xsmaddasp_3 = "f0000008QQQ", + xxsldwi_4 = "f0000010QQQz", + xsrsqrtesp_2 = "f0000028Q-Q", + xssqrtsp_2 = "f000002cQ-Q", + xxsel_4 = "f0000030QQQQ", + xssubsp_3 = "f0000040QQQ", + xsmaddmsp_3 = "f0000048QQQ", + xxpermdi_4 = "f0000050QQQz", + xsresp_2 = "f0000068Q-Q", + xsmulsp_3 = "f0000080QQQ", + xsmsubasp_3 = "f0000088QQQ", + xxmrghw_3 = "f0000090QQQ", + xsdivsp_3 = "f00000c0QQQ", + xsmsubmsp_3 = "f00000c8QQQ", + xsadddp_3 = "f0000100QQQ", + xsmaddadp_3 = "f0000108QQQ", + xscmpudp_3 = "f0000118XQQ", + xscvdpuxws_2 = "f0000120Q-Q", + xsrdpi_2 = "f0000124Q-Q", + xsrsqrtedp_2 = "f0000128Q-Q", + xssqrtdp_2 = "f000012cQ-Q", + xssubdp_3 = "f0000140QQQ", + xsmaddmdp_3 = "f0000148QQQ", + xscmpodp_3 = "f0000158XQQ", + xscvdpsxws_2 = "f0000160Q-Q", + xsrdpiz_2 = "f0000164Q-Q", + xsredp_2 = "f0000168Q-Q", + xsmuldp_3 = "f0000180QQQ", + xsmsubadp_3 = "f0000188QQQ", + xxmrglw_3 = "f0000190QQQ", + xsrdpip_2 = "f00001a4Q-Q", + xstsqrtdp_2 = "f00001a8X-Q", + xsrdpic_2 = "f00001acQ-Q", + xsdivdp_3 = "f00001c0QQQ", + xsmsubmdp_3 = "f00001c8QQQ", + xsrdpim_2 = "f00001e4Q-Q", + xstdivdp_3 = "f00001e8XQQ", + xvaddsp_3 = "f0000200QQQ", + xvmaddasp_3 = "f0000208QQQ", + xvcmpeqsp_3 = "f0000218QQQ", + xvcvspuxws_2 = "f0000220Q-Q", + xvrspi_2 = "f0000224Q-Q", + xvrsqrtesp_2 = "f0000228Q-Q", + xvsqrtsp_2 = "f000022cQ-Q", + xvsubsp_3 = "f0000240QQQ", + xvmaddmsp_3 = "f0000248QQQ", + xvcmpgtsp_3 = "f0000258QQQ", + xvcvspsxws_2 = "f0000260Q-Q", + xvrspiz_2 = "f0000264Q-Q", + xvresp_2 = "f0000268Q-Q", + xvmulsp_3 = "f0000280QQQ", + xvmsubasp_3 = "f0000288QQQ", + xxspltw_3 = "f0000290QQg~", + xvcmpgesp_3 = "f0000298QQQ", + xvcvuxwsp_2 = "f00002a0Q-Q", + xvrspip_2 = "f00002a4Q-Q", + xvtsqrtsp_2 = "f00002a8X-Q", + xvrspic_2 = "f00002acQ-Q", + xvdivsp_3 = "f00002c0QQQ", + xvmsubmsp_3 = "f00002c8QQQ", + xvcvsxwsp_2 = "f00002e0Q-Q", + xvrspim_2 = "f00002e4Q-Q", + xvtdivsp_3 = "f00002e8XQQ", + xvadddp_3 = "f0000300QQQ", + xvmaddadp_3 = "f0000308QQQ", + xvcmpeqdp_3 = "f0000318QQQ", + xvcvdpuxws_2 = "f0000320Q-Q", + xvrdpi_2 = "f0000324Q-Q", + xvrsqrtedp_2 = "f0000328Q-Q", + xvsqrtdp_2 = "f000032cQ-Q", + xvsubdp_3 = "f0000340QQQ", + xvmaddmdp_3 = "f0000348QQQ", + xvcmpgtdp_3 = "f0000358QQQ", + xvcvdpsxws_2 = "f0000360Q-Q", + xvrdpiz_2 = "f0000364Q-Q", + xvredp_2 = "f0000368Q-Q", + xvmuldp_3 = "f0000380QQQ", + xvmsubadp_3 = "f0000388QQQ", + xvcmpgedp_3 = "f0000398QQQ", + xvcvuxwdp_2 = "f00003a0Q-Q", + xvrdpip_2 = "f00003a4Q-Q", + xvtsqrtdp_2 = "f00003a8X-Q", + xvrdpic_2 = "f00003acQ-Q", + xvdivdp_3 = "f00003c0QQQ", + xvmsubmdp_3 = "f00003c8QQQ", + xvcvsxwdp_2 = "f00003e0Q-Q", + xvrdpim_2 = "f00003e4Q-Q", + xvtdivdp_3 = "f00003e8XQQ", + xsnmaddasp_3 = "f0000408QQQ", + xxland_3 = "f0000410QQQ", + xscvdpsp_2 = "f0000424Q-Q", + xscvdpspn_2 = "f000042cQ-Q", + xsnmaddmsp_3 = "f0000448QQQ", + xxlandc_3 = "f0000450QQQ", + xsrsp_2 = "f0000464Q-Q", + xsnmsubasp_3 = "f0000488QQQ", + xxlor_3 = "f0000490QQQ", + xscvuxdsp_2 = "f00004a0Q-Q", + xsnmsubmsp_3 = "f00004c8QQQ", + xxlxor_3 = "f00004d0QQQ", + xscvsxdsp_2 = "f00004e0Q-Q", + xsmaxdp_3 = "f0000500QQQ", + xsnmaddadp_3 = "f0000508QQQ", + xxlnor_3 = "f0000510QQQ", + xscvdpuxds_2 = "f0000520Q-Q", + xscvspdp_2 = "f0000524Q-Q", + xscvspdpn_2 = "f000052cQ-Q", + xsmindp_3 = "f0000540QQQ", + xsnmaddmdp_3 = "f0000548QQQ", + xxlorc_3 = "f0000550QQQ", + xscvdpsxds_2 = "f0000560Q-Q", + xsabsdp_2 = "f0000564Q-Q", + xscpsgndp_3 = "f0000580QQQ", + xsnmsubadp_3 = "f0000588QQQ", + xxlnand_3 = "f0000590QQQ", + xscvuxddp_2 = "f00005a0Q-Q", + xsnabsdp_2 = "f00005a4Q-Q", + xsnmsubmdp_3 = "f00005c8QQQ", + xxleqv_3 = "f00005d0QQQ", + xscvsxddp_2 = "f00005e0Q-Q", + xsnegdp_2 = "f00005e4Q-Q", + xvmaxsp_3 = "f0000600QQQ", + xvnmaddasp_3 = "f0000608QQQ", + ["xvcmpeqsp._3"] = "f0000618QQQ", + xvcvspuxds_2 = "f0000620Q-Q", + xvcvdpsp_2 = "f0000624Q-Q", + xvminsp_3 = "f0000640QQQ", + xvnmaddmsp_3 = "f0000648QQQ", + ["xvcmpgtsp._3"] = "f0000658QQQ", + xvcvspsxds_2 = "f0000660Q-Q", + xvabssp_2 = "f0000664Q-Q", + xvcpsgnsp_3 = "f0000680QQQ", + xvnmsubasp_3 = "f0000688QQQ", + ["xvcmpgesp._3"] = "f0000698QQQ", + xvcvuxdsp_2 = "f00006a0Q-Q", + xvnabssp_2 = "f00006a4Q-Q", + xvnmsubmsp_3 = "f00006c8QQQ", + xvcvsxdsp_2 = "f00006e0Q-Q", + xvnegsp_2 = "f00006e4Q-Q", + xvmaxdp_3 = "f0000700QQQ", + xvnmaddadp_3 = "f0000708QQQ", + ["xvcmpeqdp._3"] = "f0000718QQQ", + xvcvdpuxds_2 = "f0000720Q-Q", + xvcvspdp_2 = "f0000724Q-Q", + xvmindp_3 = "f0000740QQQ", + xvnmaddmdp_3 = "f0000748QQQ", + ["xvcmpgtdp._3"] = "f0000758QQQ", + xvcvdpsxds_2 = "f0000760Q-Q", + xvabsdp_2 = "f0000764Q-Q", + xvcpsgndp_3 = "f0000780QQQ", + xvnmsubadp_3 = "f0000788QQQ", + ["xvcmpgedp._3"] = "f0000798QQQ", + xvcvuxddp_2 = "f00007a0Q-Q", + xvnabsdp_2 = "f00007a4Q-Q", + xvnmsubmdp_3 = "f00007c8QQQ", + xvcvsxddp_2 = "f00007e0Q-Q", + xvnegdp_2 = "f00007e4Q-Q", + + -- Primary opcode 61: + stfdp_2 = "f4000000F:D", -- NYI: displacement must be divisible by 4. + + -- Primary opcode 62: + stq_2 = "f8000002R:D", -- NYI: displacement must be divisible by 8. + + -- Primary opcode 63: + fdiv_3 = "fc000024FFF.", + fsub_3 = "fc000028FFF.", + fadd_3 = "fc00002aFFF.", + fsqrt_2 = "fc00002cF-F.", + fsel_4 = "fc00002eFFFF~.", + fre_2 = "fc000030F-F.", + fmul_3 = "fc000032FF-F.", + frsqrte_2 = "fc000034F-F.", + fmsub_4 = "fc000038FFFF~.", + fmadd_4 = "fc00003aFFFF~.", + fnmsub_4 = "fc00003cFFFF~.", + fnmadd_4 = "fc00003eFFFF~.", + fcmpu_3 = "fc000000XFF", + fcpsgn_3 = "fc000010FFF.", + fcmpo_3 = "fc000040XFF", + mtfsb1_1 = "fc00004cA", + fneg_2 = "fc000050F-F.", + mcrfs_2 = "fc000080XX", + mtfsb0_1 = "fc00008cA", + fmr_2 = "fc000090F-F.", + frsp_2 = "fc000018F-F.", + fctiw_2 = "fc00001cF-F.", + fctiwz_2 = "fc00001eF-F.", + ftdiv_2 = "fc000100X-F.", + fctiwu_2 = "fc00011cF-F.", + fctiwuz_2 = "fc00011eF-F.", + mtfsfi_2 = "fc00010cAA", -- NYI: upshift. + fnabs_2 = "fc000110F-F.", + ftsqrt_2 = "fc000140X-F.", + fabs_2 = "fc000210F-F.", + frin_2 = "fc000310F-F.", + friz_2 = "fc000350F-F.", + frip_2 = "fc000390F-F.", + frim_2 = "fc0003d0F-F.", + mffs_1 = "fc00048eF.", + -- NYI: mtfsf, mtfsb0, mtfsb1. + fctid_2 = "fc00065cF-F.", + fctidz_2 = "fc00065eF-F.", + fmrgow_3 = "fc00068cFFF", + fcfid_2 = "fc00069cF-F.", + fctidu_2 = "fc00075cF-F.", + fctiduz_2 = "fc00075eF-F.", + fmrgew_3 = "fc00078cFFF", + fcfidu_2 = "fc00079cF-F.", + + daddq_3 = "fc000004F:F:F:.", + dquaq_4 = "fc000006F:F:F:Z.", + dmulq_3 = "fc000044F:F:F:.", + drrndq_4 = "fc000046F:F:F:Z.", + dscliq_3 = "fc000084F:F:6.", + dquaiq_4 = "fc000086SF:~F:Z.", + dscriq_3 = "fc0000c4F:F:6.", + drintxq_4 = "fc0000c61F:~F:Z.", + dcmpoq_3 = "fc000104XF:F:", + dtstexq_3 = "fc000144XF:F:", + dtstdcq_3 = "fc000184XF:6", + dtstdgq_3 = "fc0001c4XF:6", + drintnq_4 = "fc0001c61F:~F:Z.", + dctqpq_2 = "fc000204F:-F:.", + dctfixq_2 = "fc000244F:-F:.", + ddedpdq_3 = "fc000284ZF:~F:.", + dxexq_2 = "fc0002c4F:-F:.", + dsubq_3 = "fc000404F:F:F:.", + ddivq_3 = "fc000444F:F:F:.", + dcmpuq_3 = "fc000504XF:F:", + dtstsfq_3 = "fc000544XF:F:", + drdpq_2 = "fc000604F:-F:.", + dcffixq_2 = "fc000644F:-F:.", + denbcdq_3 = "fc000684YF:~F:.", + diexq_3 = "fc0006c4F:FF:.", + + -- Primary opcode 4, SPE APU extension: + evaddw_3 = "10000200RRR", + evaddiw_3 = "10000202RAR~", + evsubw_3 = "10000204RRR~", + evsubiw_3 = "10000206RAR~", + evabs_2 = "10000208RR", + evneg_2 = "10000209RR", + evextsb_2 = "1000020aRR", + evextsh_2 = "1000020bRR", + evrndw_2 = "1000020cRR", + evcntlzw_2 = "1000020dRR", + evcntlsw_2 = "1000020eRR", + brinc_3 = "1000020fRRR", + evand_3 = "10000211RRR", + evandc_3 = "10000212RRR", + evxor_3 = "10000216RRR", + evor_3 = "10000217RRR", + evmr_2 = "10000217RR=", + evnor_3 = "10000218RRR", + evnot_2 = "10000218RR=", + eveqv_3 = "10000219RRR", + evorc_3 = "1000021bRRR", + evnand_3 = "1000021eRRR", + evsrwu_3 = "10000220RRR", + evsrws_3 = "10000221RRR", + evsrwiu_3 = "10000222RRA", + evsrwis_3 = "10000223RRA", + evslw_3 = "10000224RRR", + evslwi_3 = "10000226RRA", + evrlw_3 = "10000228RRR", + evsplati_2 = "10000229RS", + evrlwi_3 = "1000022aRRA", + evsplatfi_2 = "1000022bRS", + evmergehi_3 = "1000022cRRR", + evmergelo_3 = "1000022dRRR", + evcmpgtu_3 = "10000230XRR", + evcmpgtu_2 = "10000230-RR", + evcmpgts_3 = "10000231XRR", + evcmpgts_2 = "10000231-RR", + evcmpltu_3 = "10000232XRR", + evcmpltu_2 = "10000232-RR", + evcmplts_3 = "10000233XRR", + evcmplts_2 = "10000233-RR", + evcmpeq_3 = "10000234XRR", + evcmpeq_2 = "10000234-RR", + evsel_4 = "10000278RRRW", + evsel_3 = "10000278RRR", + evfsadd_3 = "10000280RRR", + evfssub_3 = "10000281RRR", + evfsabs_2 = "10000284RR", + evfsnabs_2 = "10000285RR", + evfsneg_2 = "10000286RR", + evfsmul_3 = "10000288RRR", + evfsdiv_3 = "10000289RRR", + evfscmpgt_3 = "1000028cXRR", + evfscmpgt_2 = "1000028c-RR", + evfscmplt_3 = "1000028dXRR", + evfscmplt_2 = "1000028d-RR", + evfscmpeq_3 = "1000028eXRR", + evfscmpeq_2 = "1000028e-RR", + evfscfui_2 = "10000290R-R", + evfscfsi_2 = "10000291R-R", + evfscfuf_2 = "10000292R-R", + evfscfsf_2 = "10000293R-R", + evfsctui_2 = "10000294R-R", + evfsctsi_2 = "10000295R-R", + evfsctuf_2 = "10000296R-R", + evfsctsf_2 = "10000297R-R", + evfsctuiz_2 = "10000298R-R", + evfsctsiz_2 = "1000029aR-R", + evfststgt_3 = "1000029cXRR", + evfststgt_2 = "1000029c-RR", + evfststlt_3 = "1000029dXRR", + evfststlt_2 = "1000029d-RR", + evfststeq_3 = "1000029eXRR", + evfststeq_2 = "1000029e-RR", + efsadd_3 = "100002c0RRR", + efssub_3 = "100002c1RRR", + efsabs_2 = "100002c4RR", + efsnabs_2 = "100002c5RR", + efsneg_2 = "100002c6RR", + efsmul_3 = "100002c8RRR", + efsdiv_3 = "100002c9RRR", + efscmpgt_3 = "100002ccXRR", + efscmpgt_2 = "100002cc-RR", + efscmplt_3 = "100002cdXRR", + efscmplt_2 = "100002cd-RR", + efscmpeq_3 = "100002ceXRR", + efscmpeq_2 = "100002ce-RR", + efscfd_2 = "100002cfR-R", + efscfui_2 = "100002d0R-R", + efscfsi_2 = "100002d1R-R", + efscfuf_2 = "100002d2R-R", + efscfsf_2 = "100002d3R-R", + efsctui_2 = "100002d4R-R", + efsctsi_2 = "100002d5R-R", + efsctuf_2 = "100002d6R-R", + efsctsf_2 = "100002d7R-R", + efsctuiz_2 = "100002d8R-R", + efsctsiz_2 = "100002daR-R", + efststgt_3 = "100002dcXRR", + efststgt_2 = "100002dc-RR", + efststlt_3 = "100002ddXRR", + efststlt_2 = "100002dd-RR", + efststeq_3 = "100002deXRR", + efststeq_2 = "100002de-RR", + efdadd_3 = "100002e0RRR", + efdsub_3 = "100002e1RRR", + efdcfuid_2 = "100002e2R-R", + efdcfsid_2 = "100002e3R-R", + efdabs_2 = "100002e4RR", + efdnabs_2 = "100002e5RR", + efdneg_2 = "100002e6RR", + efdmul_3 = "100002e8RRR", + efddiv_3 = "100002e9RRR", + efdctuidz_2 = "100002eaR-R", + efdctsidz_2 = "100002ebR-R", + efdcmpgt_3 = "100002ecXRR", + efdcmpgt_2 = "100002ec-RR", + efdcmplt_3 = "100002edXRR", + efdcmplt_2 = "100002ed-RR", + efdcmpeq_3 = "100002eeXRR", + efdcmpeq_2 = "100002ee-RR", + efdcfs_2 = "100002efR-R", + efdcfui_2 = "100002f0R-R", + efdcfsi_2 = "100002f1R-R", + efdcfuf_2 = "100002f2R-R", + efdcfsf_2 = "100002f3R-R", + efdctui_2 = "100002f4R-R", + efdctsi_2 = "100002f5R-R", + efdctuf_2 = "100002f6R-R", + efdctsf_2 = "100002f7R-R", + efdctuiz_2 = "100002f8R-R", + efdctsiz_2 = "100002faR-R", + efdtstgt_3 = "100002fcXRR", + efdtstgt_2 = "100002fc-RR", + efdtstlt_3 = "100002fdXRR", + efdtstlt_2 = "100002fd-RR", + efdtsteq_3 = "100002feXRR", + efdtsteq_2 = "100002fe-RR", + evlddx_3 = "10000300RR0R", + evldd_2 = "10000301R8", + evldwx_3 = "10000302RR0R", + evldw_2 = "10000303R8", + evldhx_3 = "10000304RR0R", + evldh_2 = "10000305R8", + evlwhex_3 = "10000310RR0R", + evlwhe_2 = "10000311R4", + evlwhoux_3 = "10000314RR0R", + evlwhou_2 = "10000315R4", + evlwhosx_3 = "10000316RR0R", + evlwhos_2 = "10000317R4", + evstddx_3 = "10000320RR0R", + evstdd_2 = "10000321R8", + evstdwx_3 = "10000322RR0R", + evstdw_2 = "10000323R8", + evstdhx_3 = "10000324RR0R", + evstdh_2 = "10000325R8", + evstwhex_3 = "10000330RR0R", + evstwhe_2 = "10000331R4", + evstwhox_3 = "10000334RR0R", + evstwho_2 = "10000335R4", + evstwwex_3 = "10000338RR0R", + evstwwe_2 = "10000339R4", + evstwwox_3 = "1000033cRR0R", + evstwwo_2 = "1000033dR4", + evmhessf_3 = "10000403RRR", + evmhossf_3 = "10000407RRR", + evmheumi_3 = "10000408RRR", + evmhesmi_3 = "10000409RRR", + evmhesmf_3 = "1000040bRRR", + evmhoumi_3 = "1000040cRRR", + evmhosmi_3 = "1000040dRRR", + evmhosmf_3 = "1000040fRRR", + evmhessfa_3 = "10000423RRR", + evmhossfa_3 = "10000427RRR", + evmheumia_3 = "10000428RRR", + evmhesmia_3 = "10000429RRR", + evmhesmfa_3 = "1000042bRRR", + evmhoumia_3 = "1000042cRRR", + evmhosmia_3 = "1000042dRRR", + evmhosmfa_3 = "1000042fRRR", + evmwhssf_3 = "10000447RRR", + evmwlumi_3 = "10000448RRR", + evmwhumi_3 = "1000044cRRR", + evmwhsmi_3 = "1000044dRRR", + evmwhsmf_3 = "1000044fRRR", + evmwssf_3 = "10000453RRR", + evmwumi_3 = "10000458RRR", + evmwsmi_3 = "10000459RRR", + evmwsmf_3 = "1000045bRRR", + evmwhssfa_3 = "10000467RRR", + evmwlumia_3 = "10000468RRR", + evmwhumia_3 = "1000046cRRR", + evmwhsmia_3 = "1000046dRRR", + evmwhsmfa_3 = "1000046fRRR", + evmwssfa_3 = "10000473RRR", + evmwumia_3 = "10000478RRR", + evmwsmia_3 = "10000479RRR", + evmwsmfa_3 = "1000047bRRR", + evmra_2 = "100004c4RR", + evdivws_3 = "100004c6RRR", + evdivwu_3 = "100004c7RRR", + evmwssfaa_3 = "10000553RRR", + evmwumiaa_3 = "10000558RRR", + evmwsmiaa_3 = "10000559RRR", + evmwsmfaa_3 = "1000055bRRR", + evmwssfan_3 = "100005d3RRR", + evmwumian_3 = "100005d8RRR", + evmwsmian_3 = "100005d9RRR", + evmwsmfan_3 = "100005dbRRR", + evmergehilo_3 = "1000022eRRR", + evmergelohi_3 = "1000022fRRR", + evlhhesplatx_3 = "10000308RR0R", + evlhhesplat_2 = "10000309R2", + evlhhousplatx_3 = "1000030cRR0R", + evlhhousplat_2 = "1000030dR2", + evlhhossplatx_3 = "1000030eRR0R", + evlhhossplat_2 = "1000030fR2", + evlwwsplatx_3 = "10000318RR0R", + evlwwsplat_2 = "10000319R4", + evlwhsplatx_3 = "1000031cRR0R", + evlwhsplat_2 = "1000031dR4", + evaddusiaaw_2 = "100004c0RR", + evaddssiaaw_2 = "100004c1RR", + evsubfusiaaw_2 = "100004c2RR", + evsubfssiaaw_2 = "100004c3RR", + evaddumiaaw_2 = "100004c8RR", + evaddsmiaaw_2 = "100004c9RR", + evsubfumiaaw_2 = "100004caRR", + evsubfsmiaaw_2 = "100004cbRR", + evmheusiaaw_3 = "10000500RRR", + evmhessiaaw_3 = "10000501RRR", + evmhessfaaw_3 = "10000503RRR", + evmhousiaaw_3 = "10000504RRR", + evmhossiaaw_3 = "10000505RRR", + evmhossfaaw_3 = "10000507RRR", + evmheumiaaw_3 = "10000508RRR", + evmhesmiaaw_3 = "10000509RRR", + evmhesmfaaw_3 = "1000050bRRR", + evmhoumiaaw_3 = "1000050cRRR", + evmhosmiaaw_3 = "1000050dRRR", + evmhosmfaaw_3 = "1000050fRRR", + evmhegumiaa_3 = "10000528RRR", + evmhegsmiaa_3 = "10000529RRR", + evmhegsmfaa_3 = "1000052bRRR", + evmhogumiaa_3 = "1000052cRRR", + evmhogsmiaa_3 = "1000052dRRR", + evmhogsmfaa_3 = "1000052fRRR", + evmwlusiaaw_3 = "10000540RRR", + evmwlssiaaw_3 = "10000541RRR", + evmwlumiaaw_3 = "10000548RRR", + evmwlsmiaaw_3 = "10000549RRR", + evmheusianw_3 = "10000580RRR", + evmhessianw_3 = "10000581RRR", + evmhessfanw_3 = "10000583RRR", + evmhousianw_3 = "10000584RRR", + evmhossianw_3 = "10000585RRR", + evmhossfanw_3 = "10000587RRR", + evmheumianw_3 = "10000588RRR", + evmhesmianw_3 = "10000589RRR", + evmhesmfanw_3 = "1000058bRRR", + evmhoumianw_3 = "1000058cRRR", + evmhosmianw_3 = "1000058dRRR", + evmhosmfanw_3 = "1000058fRRR", + evmhegumian_3 = "100005a8RRR", + evmhegsmian_3 = "100005a9RRR", + evmhegsmfan_3 = "100005abRRR", + evmhogumian_3 = "100005acRRR", + evmhogsmian_3 = "100005adRRR", + evmhogsmfan_3 = "100005afRRR", + evmwlusianw_3 = "100005c0RRR", + evmwlssianw_3 = "100005c1RRR", + evmwlumianw_3 = "100005c8RRR", + evmwlsmianw_3 = "100005c9RRR", + + -- NYI: Book E instructions. +} + +-- Add mnemonics for "." variants. +do + local t = {} + for k,v in pairs(map_op) do + if type(v) == "string" and sub(v, -1) == "." then + local v2 = sub(v, 1, 7)..char(byte(v, 8)+1)..sub(v, 9, -2) + t[sub(k, 1, -3).."."..sub(k, -2)] = v2 + end + end + for k,v in pairs(t) do + map_op[k] = v + end +end + +-- Add more branch mnemonics. +for cond,c in pairs(map_cond) do + local b1 = "b"..cond + local c1 = shl(band(c, 3), 16) + (c < 4 and 0x01000000 or 0) + -- bX[l] + map_op[b1.."_1"] = tohex(0x40800000 + c1).."K" + map_op[b1.."y_1"] = tohex(0x40a00000 + c1).."K" + map_op[b1.."l_1"] = tohex(0x40800001 + c1).."K" + map_op[b1.."_2"] = tohex(0x40800000 + c1).."-XK" + map_op[b1.."y_2"] = tohex(0x40a00000 + c1).."-XK" + map_op[b1.."l_2"] = tohex(0x40800001 + c1).."-XK" + -- bXlr[l] + map_op[b1.."lr_0"] = tohex(0x4c800020 + c1) + map_op[b1.."lrl_0"] = tohex(0x4c800021 + c1) + map_op[b1.."ctr_0"] = tohex(0x4c800420 + c1) + map_op[b1.."ctrl_0"] = tohex(0x4c800421 + c1) + -- bXctr[l] + map_op[b1.."lr_1"] = tohex(0x4c800020 + c1).."-X" + map_op[b1.."lrl_1"] = tohex(0x4c800021 + c1).."-X" + map_op[b1.."ctr_1"] = tohex(0x4c800420 + c1).."-X" + map_op[b1.."ctrl_1"] = tohex(0x4c800421 + c1).."-X" +end + +------------------------------------------------------------------------------ + +local function parse_gpr(expr) + local tname, ovreg = match(expr, "^([%w_]+):(r[1-3]?[0-9])$") + local tp = map_type[tname or expr] + if tp then + local reg = ovreg or tp.reg + if not reg then + werror("type `"..(tname or expr).."' needs a register override") + end + expr = reg + end + local r = match(expr, "^r([1-3]?[0-9])$") + if r then + r = tonumber(r) + if r <= 31 then return r, tp end + end + werror("bad register name `"..expr.."'") +end + +local function parse_fpr(expr) + local r = match(expr, "^f([1-3]?[0-9])$") + if r then + r = tonumber(r) + if r <= 31 then return r end + end + werror("bad register name `"..expr.."'") +end + +local function parse_vr(expr) + local r = match(expr, "^v([1-3]?[0-9])$") + if r then + r = tonumber(r) + if r <= 31 then return r end + end + werror("bad register name `"..expr.."'") +end + +local function parse_vs(expr) + local r = match(expr, "^vs([1-6]?[0-9])$") + if r then + r = tonumber(r) + if r <= 63 then return r end + end + werror("bad register name `"..expr.."'") +end + +local function parse_cr(expr) + local r = match(expr, "^cr([0-7])$") + if r then return tonumber(r) end + werror("bad condition register name `"..expr.."'") +end + +local function parse_cond(expr) + local r, cond = match(expr, "^4%*cr([0-7])%+(%w%w)$") + if r then + r = tonumber(r) + local c = map_cond[cond] + if c and c < 4 then return r*4+c end + end + werror("bad condition bit name `"..expr.."'") +end + +local parse_ctx = {} + +local loadenv = setfenv and function(s) + local code = loadstring(s, "") + if code then setfenv(code, parse_ctx) end + return code +end or function(s) + return load(s, "", nil, parse_ctx) +end + +-- Try to parse simple arithmetic, too, since some basic ops are aliases. +local function parse_number(n) + local x = tonumber(n) + if x then return x end + local code = loadenv("return "..n) + if code then + local ok, y = pcall(code) + if ok then return y end + end + return nil +end + +local function parse_imm(imm, bits, shift, scale, signed) + local n = parse_number(imm) + if n then + local m = sar(n, scale) + if shl(m, scale) == n then + if signed then + local s = sar(m, bits-1) + if s == 0 then return shl(m, shift) + elseif s == -1 then return shl(m + shl(1, bits), shift) end + else + if sar(m, bits) == 0 then return shl(m, shift) end + end + end + werror("out of range immediate `"..imm.."'") + elseif match(imm, "^[rfv]([1-3]?[0-9])$") or + match(imm, "^vs([1-6]?[0-9])$") or + match(imm, "^([%w_]+):(r[1-3]?[0-9])$") then + werror("expected immediate operand, got register") + else + waction("IMM", (signed and 32768 or 0)+scale*1024+bits*32+shift, imm) + return 0 + end +end + +local function parse_shiftmask(imm, isshift) + local n = parse_number(imm) + if n then + if shr(n, 6) == 0 then + local lsb = band(n, 31) + local msb = n - lsb + return isshift and (shl(lsb, 11)+shr(msb, 4)) or (shl(lsb, 6)+msb) + end + werror("out of range immediate `"..imm.."'") + elseif match(imm, "^r([1-3]?[0-9])$") or + match(imm, "^([%w_]+):(r[1-3]?[0-9])$") then + werror("expected immediate operand, got register") + else + waction("IMMSH", isshift and 1 or 0, imm) + return 0; + end +end + +local function parse_disp(disp) + local imm, reg = match(disp, "^(.*)%(([%w_:]+)%)$") + if imm then + local r = parse_gpr(reg) + if r == 0 then werror("cannot use r0 in displacement") end + return shl(r, 16) + parse_imm(imm, 16, 0, 0, true) + end + local reg, tailr = match(disp, "^([%w_:]+)%s*(.*)$") + if reg and tailr ~= "" then + local r, tp = parse_gpr(reg) + if r == 0 then werror("cannot use r0 in displacement") end + if tp then + waction("IMM", 32768+16*32, format(tp.ctypefmt, tailr)) + return shl(r, 16) + end + end + werror("bad displacement `"..disp.."'") +end + +local function parse_u5disp(disp, scale) + local imm, reg = match(disp, "^(.*)%(([%w_:]+)%)$") + if imm then + local r = parse_gpr(reg) + if r == 0 then werror("cannot use r0 in displacement") end + return shl(r, 16) + parse_imm(imm, 5, 11, scale, false) + end + local reg, tailr = match(disp, "^([%w_:]+)%s*(.*)$") + if reg and tailr ~= "" then + local r, tp = parse_gpr(reg) + if r == 0 then werror("cannot use r0 in displacement") end + if tp then + waction("IMM", scale*1024+5*32+11, format(tp.ctypefmt, tailr)) + return shl(r, 16) + end + end + werror("bad displacement `"..disp.."'") +end + +local function parse_label(label, def) + local prefix = sub(label, 1, 2) + -- =>label (pc label reference) + if prefix == "=>" then + return "PC", 0, sub(label, 3) + end + -- ->name (global label reference) + if prefix == "->" then + return "LG", map_global[sub(label, 3)] + end + if def then + -- [1-9] (local label definition) + if match(label, "^[1-9]$") then + return "LG", 10+tonumber(label) + end + else + -- [<>][1-9] (local label reference) + local dir, lnum = match(label, "^([<>])([1-9])$") + if dir then -- Fwd: 1-9, Bkwd: 11-19. + return "LG", lnum + (dir == ">" and 0 or 10) + end + -- extern label (extern label reference) + local extname = match(label, "^extern%s+(%S+)$") + if extname then + return "EXT", map_extern[extname] + end + end + werror("bad label `"..label.."'") +end + +------------------------------------------------------------------------------ + +-- Handle opcodes defined with template strings. +op_template = function(params, template, nparams) + if not params then return sub(template, 9) end + local op = tonumber(sub(template, 1, 8), 16) + local n, rs = 1, 26 + + -- Limit number of section buffer positions used by a single dasm_put(). + -- A single opcode needs a maximum of 3 positions (rlwinm). + if secpos+3 > maxsecpos then wflush() end + local pos = wpos() + + -- Process each character. + for p in gmatch(sub(template, 9), ".") do + if p == "R" then + rs = rs - 5; op = op + shl(parse_gpr(params[n]), rs); n = n + 1 + elseif p == "F" then + rs = rs - 5; op = op + shl(parse_fpr(params[n]), rs); n = n + 1 + elseif p == "V" then + rs = rs - 5; op = op + shl(parse_vr(params[n]), rs); n = n + 1 + elseif p == "Q" then + local vs = parse_vs(params[n]); n = n + 1; rs = rs - 5 + local sh = rs == 6 and 2 or 3 + band(shr(rs, 1), 3) + op = op + shl(band(vs, 31), rs) + shr(band(vs, 32), sh) + elseif p == "q" then + local vs = parse_vs(params[n]); n = n + 1 + op = op + shl(band(vs, 31), 21) + shr(band(vs, 32), 5) + elseif p == "A" then + rs = rs - 5; op = op + parse_imm(params[n], 5, rs, 0, false); n = n + 1 + elseif p == "S" then + rs = rs - 5; op = op + parse_imm(params[n], 5, rs, 0, true); n = n + 1 + elseif p == "I" then + op = op + parse_imm(params[n], 16, 0, 0, true); n = n + 1 + elseif p == "U" then + op = op + parse_imm(params[n], 16, 0, 0, false); n = n + 1 + elseif p == "D" then + op = op + parse_disp(params[n]); n = n + 1 + elseif p == "2" then + op = op + parse_u5disp(params[n], 1); n = n + 1 + elseif p == "4" then + op = op + parse_u5disp(params[n], 2); n = n + 1 + elseif p == "8" then + op = op + parse_u5disp(params[n], 3); n = n + 1 + elseif p == "C" then + rs = rs - 5; op = op + shl(parse_cond(params[n]), rs); n = n + 1 + elseif p == "X" then + rs = rs - 5; op = op + shl(parse_cr(params[n]), rs+2); n = n + 1 + elseif p == "1" then + rs = rs - 5; op = op + parse_imm(params[n], 1, rs, 0, false); n = n + 1 + elseif p == "g" then + rs = rs - 5; op = op + parse_imm(params[n], 2, rs, 0, false); n = n + 1 + elseif p == "3" then + rs = rs - 5; op = op + parse_imm(params[n], 3, rs, 0, false); n = n + 1 + elseif p == "P" then + rs = rs - 5; op = op + parse_imm(params[n], 4, rs, 0, false); n = n + 1 + elseif p == "p" then + op = op + parse_imm(params[n], 4, rs, 0, false); n = n + 1 + elseif p == "6" then + rs = rs - 6; op = op + parse_imm(params[n], 6, rs, 0, false); n = n + 1 + elseif p == "Y" then + rs = rs - 5; op = op + parse_imm(params[n], 1, rs+4, 0, false); n = n + 1 + elseif p == "y" then + rs = rs - 5; op = op + parse_imm(params[n], 1, rs+3, 0, false); n = n + 1 + elseif p == "Z" then + rs = rs - 5; op = op + parse_imm(params[n], 2, rs+3, 0, false); n = n + 1 + elseif p == "z" then + rs = rs - 5; op = op + parse_imm(params[n], 2, rs+2, 0, false); n = n + 1 + elseif p == "W" then + op = op + parse_cr(params[n]); n = n + 1 + elseif p == "G" then + op = op + parse_imm(params[n], 8, 12, 0, false); n = n + 1 + elseif p == "H" then + op = op + parse_shiftmask(params[n], true); n = n + 1 + elseif p == "M" then + op = op + parse_shiftmask(params[n], false); n = n + 1 + elseif p == "J" or p == "K" then + local mode, n, s = parse_label(params[n], false) + if p == "K" then n = n + 2048 end + waction("REL_"..mode, n, s, 1) + n = n + 1 + elseif p == "0" then + if band(shr(op, rs), 31) == 0 then werror("cannot use r0") end + elseif p == "=" or p == "%" then + local t = band(shr(op, p == "%" and rs+5 or rs), 31) + rs = rs - 5 + op = op + shl(t, rs) + elseif p == "~" then + local mm = shl(31, rs) + local lo = band(op, mm) + local hi = band(op, shl(mm, 5)) + op = op - lo - hi + shl(lo, 5) + shr(hi, 5) + elseif p == ":" then + if band(shr(op, rs), 1) ~= 0 then werror("register pair expected") end + elseif p == "-" then + rs = rs - 5 + elseif p == "." then + -- Ignored. + else + assert(false) + end + end + wputpos(pos, op) +end + +map_op[".template__"] = op_template + +------------------------------------------------------------------------------ + +-- Pseudo-opcode to mark the position where the action list is to be emitted. +map_op[".actionlist_1"] = function(params) + if not params then return "cvar" end + local name = params[1] -- No syntax check. You get to keep the pieces. + wline(function(out) writeactions(out, name) end) +end + +-- Pseudo-opcode to mark the position where the global enum is to be emitted. +map_op[".globals_1"] = function(params) + if not params then return "prefix" end + local prefix = params[1] -- No syntax check. You get to keep the pieces. + wline(function(out) writeglobals(out, prefix) end) +end + +-- Pseudo-opcode to mark the position where the global names are to be emitted. +map_op[".globalnames_1"] = function(params) + if not params then return "cvar" end + local name = params[1] -- No syntax check. You get to keep the pieces. + wline(function(out) writeglobalnames(out, name) end) +end + +-- Pseudo-opcode to mark the position where the extern names are to be emitted. +map_op[".externnames_1"] = function(params) + if not params then return "cvar" end + local name = params[1] -- No syntax check. You get to keep the pieces. + wline(function(out) writeexternnames(out, name) end) +end + +------------------------------------------------------------------------------ + +-- Label pseudo-opcode (converted from trailing colon form). +map_op[".label_1"] = function(params) + if not params then return "[1-9] | ->global | =>pcexpr" end + if secpos+1 > maxsecpos then wflush() end + local mode, n, s = parse_label(params[1], true) + if mode == "EXT" then werror("bad label definition") end + waction("LABEL_"..mode, n, s, 1) +end + +------------------------------------------------------------------------------ + +-- Pseudo-opcodes for data storage. +map_op[".long_*"] = function(params) + if not params then return "imm..." end + for _,p in ipairs(params) do + local n = tonumber(p) + if not n then werror("bad immediate `"..p.."'") end + if n < 0 then n = n + 2^32 end + wputw(n) + if secpos+2 > maxsecpos then wflush() end + end +end + +-- Alignment pseudo-opcode. +map_op[".align_1"] = function(params) + if not params then return "numpow2" end + if secpos+1 > maxsecpos then wflush() end + local align = tonumber(params[1]) + if align then + local x = align + -- Must be a power of 2 in the range (2 ... 256). + for i=1,8 do + x = x / 2 + if x == 1 then + waction("ALIGN", align-1, nil, 1) -- Action byte is 2**n-1. + return + end + end + end + werror("bad alignment") +end + +------------------------------------------------------------------------------ + +-- Pseudo-opcode for (primitive) type definitions (map to C types). +map_op[".type_3"] = function(params, nparams) + if not params then + return nparams == 2 and "name, ctype" or "name, ctype, reg" + end + local name, ctype, reg = params[1], params[2], params[3] + if not match(name, "^[%a_][%w_]*$") then + werror("bad type name `"..name.."'") + end + local tp = map_type[name] + if tp then + werror("duplicate type `"..name.."'") + end + -- Add #type to defines. A bit unclean to put it in map_archdef. + map_archdef["#"..name] = "sizeof("..ctype..")" + -- Add new type and emit shortcut define. + local num = ctypenum + 1 + map_type[name] = { + ctype = ctype, + ctypefmt = format("Dt%X(%%s)", num), + reg = reg, + } + wline(format("#define Dt%X(_V) (int)(ptrdiff_t)&(((%s *)0)_V)", num, ctype)) + ctypenum = num +end +map_op[".type_2"] = map_op[".type_3"] + +-- Dump type definitions. +local function dumptypes(out, lvl) + local t = {} + for name in pairs(map_type) do t[#t+1] = name end + sort(t) + out:write("Type definitions:\n") + for _,name in ipairs(t) do + local tp = map_type[name] + local reg = tp.reg or "" + out:write(format(" %-20s %-20s %s\n", name, tp.ctype, reg)) + end + out:write("\n") +end + +------------------------------------------------------------------------------ + +-- Set the current section. +function _M.section(num) + waction("SECTION", num) + wflush(true) -- SECTION is a terminal action. +end + +------------------------------------------------------------------------------ + +-- Dump architecture description. +function _M.dumparch(out) + out:write(format("DynASM %s version %s, released %s\n\n", + _info.arch, _info.version, _info.release)) + dumpactions(out) +end + +-- Dump all user defined elements. +function _M.dumpdef(out, lvl) + dumptypes(out, lvl) + dumpglobals(out, lvl) + dumpexterns(out, lvl) +end + +------------------------------------------------------------------------------ + +-- Pass callbacks from/to the DynASM core. +function _M.passcb(wl, we, wf, ww) + wline, werror, wfatal, wwarn = wl, we, wf, ww + return wflush +end + +-- Setup the arch-specific module. +function _M.setup(arch, opt) + g_arch, g_opt = arch, opt +end + +-- Merge the core maps and the arch-specific maps. +function _M.mergemaps(map_coreop, map_def) + setmetatable(map_op, { __index = map_coreop }) + setmetatable(map_def, { __index = map_archdef }) + return map_op, map_def +end + +return _M + +------------------------------------------------------------------------------ + diff --git a/ext/opcache/dynasm/dasm_proto.h b/ext/opcache/dynasm/dasm_proto.h new file mode 100644 index 0000000000000..22a654cbfd926 --- /dev/null +++ b/ext/opcache/dynasm/dasm_proto.h @@ -0,0 +1,83 @@ +/* +** DynASM encoding engine prototypes. +** Copyright (C) 2005-2016 Mike Pall. All rights reserved. +** Released under the MIT license. See dynasm.lua for full copyright notice. +*/ + +#ifndef _DASM_PROTO_H +#define _DASM_PROTO_H + +#include +#include + +#define DASM_IDENT "DynASM 1.4.0" +#define DASM_VERSION 10400 /* 1.4.0 */ + +#ifndef Dst_DECL +#define Dst_DECL dasm_State **Dst +#endif + +#ifndef Dst_REF +#define Dst_REF (*Dst) +#endif + +#ifndef DASM_FDEF +#define DASM_FDEF extern +#endif + +#ifndef DASM_M_GROW +#define DASM_M_GROW(ctx, t, p, sz, need) \ + do { \ + size_t _sz = (sz), _need = (need); \ + if (_sz < _need) { \ + if (_sz < 16) _sz = 16; \ + while (_sz < _need) _sz += _sz; \ + (p) = (t *)realloc((p), _sz); \ + if ((p) == NULL) exit(1); \ + (sz) = _sz; \ + } \ + } while(0) +#endif + +#ifndef DASM_M_FREE +#define DASM_M_FREE(ctx, p, sz) free(p) +#endif + +/* Internal DynASM encoder state. */ +typedef struct dasm_State dasm_State; + + +/* Initialize and free DynASM state. */ +DASM_FDEF void dasm_init(Dst_DECL, int maxsection); +DASM_FDEF void dasm_free(Dst_DECL); + +/* Setup global array. Must be called before dasm_setup(). */ +DASM_FDEF void dasm_setupglobal(Dst_DECL, void **gl, unsigned int maxgl); + +/* Grow PC label array. Can be called after dasm_setup(), too. */ +DASM_FDEF void dasm_growpc(Dst_DECL, unsigned int maxpc); + +/* Setup encoder. */ +DASM_FDEF void dasm_setup(Dst_DECL, const void *actionlist); + +/* Feed encoder with actions. Calls are generated by pre-processor. */ +DASM_FDEF void dasm_put(Dst_DECL, int start, ...); + +/* Link sections and return the resulting size. */ +DASM_FDEF int dasm_link(Dst_DECL, size_t *szp); + +/* Encode sections into buffer. */ +DASM_FDEF int dasm_encode(Dst_DECL, void *buffer); + +/* Get PC label offset. */ +DASM_FDEF int dasm_getpclabel(Dst_DECL, unsigned int pc); + +#ifdef DASM_CHECKS +/* Optional sanity checker to call between isolated encoding steps. */ +DASM_FDEF int dasm_checkstep(Dst_DECL, int secmatch); +#else +#define dasm_checkstep(a, b) 0 +#endif + + +#endif /* _DASM_PROTO_H */ diff --git a/ext/opcache/dynasm/dasm_x64.lua b/ext/opcache/dynasm/dasm_x64.lua new file mode 100644 index 0000000000000..2133355644fcb --- /dev/null +++ b/ext/opcache/dynasm/dasm_x64.lua @@ -0,0 +1,12 @@ +------------------------------------------------------------------------------ +-- DynASM x64 module. +-- +-- Copyright (C) 2005-2016 Mike Pall. All rights reserved. +-- See dynasm.lua for full copyright notice. +------------------------------------------------------------------------------ +-- This module just sets 64 bit mode for the combined x86/x64 module. +-- All the interesting stuff is there. +------------------------------------------------------------------------------ + +x64 = true -- Using a global is an ugly, but effective solution. +return require("dasm_x86") diff --git a/ext/opcache/dynasm/dasm_x86.h b/ext/opcache/dynasm/dasm_x86.h new file mode 100644 index 0000000000000..bf5b729633b2c --- /dev/null +++ b/ext/opcache/dynasm/dasm_x86.h @@ -0,0 +1,498 @@ +/* +** DynASM x86 encoding engine. +** Copyright (C) 2005-2016 Mike Pall. All rights reserved. +** Released under the MIT license. See dynasm.lua for full copyright notice. +*/ + +#include +#include +#include +#include + +#define DASM_ARCH "x86" + +#ifndef DASM_EXTERN +#define DASM_EXTERN(a,b,c,d) 0 +#endif + +/* Action definitions. DASM_STOP must be 255. */ +enum { + DASM_DISP = 233, + DASM_IMM_S, DASM_IMM_B, DASM_IMM_W, DASM_IMM_D, DASM_IMM_WB, DASM_IMM_DB, + DASM_VREG, DASM_SPACE, DASM_SETLABEL, DASM_REL_A, DASM_REL_LG, DASM_REL_PC, + DASM_IMM_LG, DASM_IMM_PC, DASM_LABEL_LG, DASM_LABEL_PC, DASM_ALIGN, + DASM_EXTERN, DASM_ESC, DASM_MARK, DASM_SECTION, DASM_STOP +}; + +/* Maximum number of section buffer positions for a single dasm_put() call. */ +#define DASM_MAXSECPOS 25 + +/* DynASM encoder status codes. Action list offset or number are or'ed in. */ +#define DASM_S_OK 0x00000000 +#define DASM_S_NOMEM 0x01000000 +#define DASM_S_PHASE 0x02000000 +#define DASM_S_MATCH_SEC 0x03000000 +#define DASM_S_RANGE_I 0x11000000 +#define DASM_S_RANGE_SEC 0x12000000 +#define DASM_S_RANGE_LG 0x13000000 +#define DASM_S_RANGE_PC 0x14000000 +#define DASM_S_RANGE_VREG 0x15000000 +#define DASM_S_UNDEF_L 0x21000000 +#define DASM_S_UNDEF_PC 0x22000000 + +/* Macros to convert positions (8 bit section + 24 bit index). */ +#define DASM_POS2IDX(pos) ((pos)&0x00ffffff) +#define DASM_POS2BIAS(pos) ((pos)&0xff000000) +#define DASM_SEC2POS(sec) ((sec)<<24) +#define DASM_POS2SEC(pos) ((pos)>>24) +#define DASM_POS2PTR(D, pos) (D->sections[DASM_POS2SEC(pos)].rbuf + (pos)) + +/* Action list type. */ +typedef const unsigned char *dasm_ActList; + +/* Per-section structure. */ +typedef struct dasm_Section { + int *rbuf; /* Biased buffer pointer (negative section bias). */ + int *buf; /* True buffer pointer. */ + size_t bsize; /* Buffer size in bytes. */ + int pos; /* Biased buffer position. */ + int epos; /* End of biased buffer position - max single put. */ + int ofs; /* Byte offset into section. */ +} dasm_Section; + +/* Core structure holding the DynASM encoding state. */ +struct dasm_State { + size_t psize; /* Allocated size of this structure. */ + dasm_ActList actionlist; /* Current actionlist pointer. */ + int *lglabels; /* Local/global chain/pos ptrs. */ + size_t lgsize; + int *pclabels; /* PC label chains/pos ptrs. */ + size_t pcsize; + void **globals; /* Array of globals (bias -10). */ + dasm_Section *section; /* Pointer to active section. */ + size_t codesize; /* Total size of all code sections. */ + int maxsection; /* 0 <= sectionidx < maxsection. */ + int status; /* Status code. */ + dasm_Section sections[1]; /* All sections. Alloc-extended. */ +}; + +/* The size of the core structure depends on the max. number of sections. */ +#define DASM_PSZ(ms) (sizeof(dasm_State)+(ms-1)*sizeof(dasm_Section)) + + +/* Initialize DynASM state. */ +void dasm_init(Dst_DECL, int maxsection) +{ + dasm_State *D; + size_t psz = 0; + int i; + Dst_REF = NULL; + DASM_M_GROW(Dst, struct dasm_State, Dst_REF, psz, DASM_PSZ(maxsection)); + D = Dst_REF; + D->psize = psz; + D->lglabels = NULL; + D->lgsize = 0; + D->pclabels = NULL; + D->pcsize = 0; + D->globals = NULL; + D->maxsection = maxsection; + for (i = 0; i < maxsection; i++) { + D->sections[i].buf = NULL; /* Need this for pass3. */ + D->sections[i].rbuf = D->sections[i].buf - DASM_SEC2POS(i); + D->sections[i].bsize = 0; + D->sections[i].epos = 0; /* Wrong, but is recalculated after resize. */ + } +} + +/* Free DynASM state. */ +void dasm_free(Dst_DECL) +{ + dasm_State *D = Dst_REF; + int i; + for (i = 0; i < D->maxsection; i++) + if (D->sections[i].buf) + DASM_M_FREE(Dst, D->sections[i].buf, D->sections[i].bsize); + if (D->pclabels) DASM_M_FREE(Dst, D->pclabels, D->pcsize); + if (D->lglabels) DASM_M_FREE(Dst, D->lglabels, D->lgsize); + DASM_M_FREE(Dst, D, D->psize); +} + +/* Setup global label array. Must be called before dasm_setup(). */ +void dasm_setupglobal(Dst_DECL, void **gl, unsigned int maxgl) +{ + dasm_State *D = Dst_REF; + D->globals = gl - 10; /* Negative bias to compensate for locals. */ + DASM_M_GROW(Dst, int, D->lglabels, D->lgsize, (10+maxgl)*sizeof(int)); +} + +/* Grow PC label array. Can be called after dasm_setup(), too. */ +void dasm_growpc(Dst_DECL, unsigned int maxpc) +{ + dasm_State *D = Dst_REF; + size_t osz = D->pcsize; + DASM_M_GROW(Dst, int, D->pclabels, D->pcsize, maxpc*sizeof(int)); + memset((void *)(((unsigned char *)D->pclabels)+osz), 0, D->pcsize-osz); +} + +/* Setup encoder. */ +void dasm_setup(Dst_DECL, const void *actionlist) +{ + dasm_State *D = Dst_REF; + int i; + D->actionlist = (dasm_ActList)actionlist; + D->status = DASM_S_OK; + D->section = &D->sections[0]; + memset((void *)D->lglabels, 0, D->lgsize); + if (D->pclabels) memset((void *)D->pclabels, 0, D->pcsize); + for (i = 0; i < D->maxsection; i++) { + D->sections[i].pos = DASM_SEC2POS(i); + D->sections[i].ofs = 0; + } +} + + +#ifdef DASM_CHECKS +#define CK(x, st) \ + do { if (!(x)) { \ + D->status = DASM_S_##st|(int)(p-D->actionlist-1); return; } } while (0) +#define CKPL(kind, st) \ + do { if ((size_t)((char *)pl-(char *)D->kind##labels) >= D->kind##size) { \ + D->status=DASM_S_RANGE_##st|(int)(p-D->actionlist-1); return; } } while (0) +#else +#define CK(x, st) ((void)0) +#define CKPL(kind, st) ((void)0) +#endif + +/* Pass 1: Store actions and args, link branches/labels, estimate offsets. */ +void dasm_put(Dst_DECL, int start, ...) +{ + va_list ap; + dasm_State *D = Dst_REF; + dasm_ActList p = D->actionlist + start; + dasm_Section *sec = D->section; + int pos = sec->pos, ofs = sec->ofs, mrm = -1; + int *b; + + if (pos >= sec->epos) { + DASM_M_GROW(Dst, int, sec->buf, sec->bsize, + sec->bsize + 2*DASM_MAXSECPOS*sizeof(int)); + sec->rbuf = sec->buf - DASM_POS2BIAS(pos); + sec->epos = (int)sec->bsize/sizeof(int) - DASM_MAXSECPOS+DASM_POS2BIAS(pos); + } + + b = sec->rbuf; + b[pos++] = start; + + va_start(ap, start); + while (1) { + int action = *p++; + if (action < DASM_DISP) { + ofs++; + } else if (action <= DASM_REL_A) { + int n = va_arg(ap, int); + b[pos++] = n; + switch (action) { + case DASM_DISP: + if (n == 0) { if (mrm < 0) mrm = p[-2]; if ((mrm&7) != 5) break; } + case DASM_IMM_DB: if (((n+128)&-256) == 0) goto ob; + case DASM_REL_A: /* Assumes ptrdiff_t is int. !x64 */ + case DASM_IMM_D: ofs += 4; break; + case DASM_IMM_S: CK(((n+128)&-256) == 0, RANGE_I); goto ob; + case DASM_IMM_B: CK((n&-256) == 0, RANGE_I); ob: ofs++; break; + case DASM_IMM_WB: if (((n+128)&-256) == 0) goto ob; + case DASM_IMM_W: CK((n&-65536) == 0, RANGE_I); ofs += 2; break; + case DASM_SPACE: p++; ofs += n; break; + case DASM_SETLABEL: b[pos-2] = -0x40000000; break; /* Neg. label ofs. */ + case DASM_VREG: CK((n&-16) == 0 && (n != 4 || (*p>>5) != 2), RANGE_VREG); + if (*p < 0x40 && p[1] == DASM_DISP) mrm = n; + if (*p < 0x20 && (n&7) == 4) ofs++; + switch ((*p++ >> 3) & 3) { + case 3: n |= b[pos-3]; + case 2: n |= b[pos-2]; + case 1: if (n <= 7) { b[pos-1] |= 0x10; ofs--; } + } + continue; + } + mrm = -1; + } else { + int *pl, n; + switch (action) { + case DASM_REL_LG: + case DASM_IMM_LG: + n = *p++; pl = D->lglabels + n; + /* Bkwd rel or global. */ + if (n <= 246) { CK(n>=10||*pl<0, RANGE_LG); CKPL(lg, LG); goto putrel; } + pl -= 246; n = *pl; + if (n < 0) n = 0; /* Start new chain for fwd rel if label exists. */ + goto linkrel; + case DASM_REL_PC: + case DASM_IMM_PC: pl = D->pclabels + va_arg(ap, int); CKPL(pc, PC); + putrel: + n = *pl; + if (n < 0) { /* Label exists. Get label pos and store it. */ + b[pos] = -n; + } else { + linkrel: + b[pos] = n; /* Else link to rel chain, anchored at label. */ + *pl = pos; + } + pos++; + ofs += 4; /* Maximum offset needed. */ + if (action == DASM_REL_LG || action == DASM_REL_PC) + b[pos++] = ofs; /* Store pass1 offset estimate. */ + break; + case DASM_LABEL_LG: pl = D->lglabels + *p++; CKPL(lg, LG); goto putlabel; + case DASM_LABEL_PC: pl = D->pclabels + va_arg(ap, int); CKPL(pc, PC); + putlabel: + n = *pl; /* n > 0: Collapse rel chain and replace with label pos. */ + while (n > 0) { int *pb = DASM_POS2PTR(D, n); n = *pb; *pb = pos; } + *pl = -pos; /* Label exists now. */ + b[pos++] = ofs; /* Store pass1 offset estimate. */ + break; + case DASM_ALIGN: + ofs += *p++; /* Maximum alignment needed (arg is 2**n-1). */ + b[pos++] = ofs; /* Store pass1 offset estimate. */ + break; + case DASM_EXTERN: p += 2; ofs += 4; break; + case DASM_ESC: p++; ofs++; break; + case DASM_MARK: mrm = p[-2]; break; + case DASM_SECTION: + n = *p; CK(n < D->maxsection, RANGE_SEC); D->section = &D->sections[n]; + case DASM_STOP: goto stop; + } + } + } +stop: + va_end(ap); + sec->pos = pos; + sec->ofs = ofs; +} +#undef CK + +/* Pass 2: Link sections, shrink branches/aligns, fix label offsets. */ +int dasm_link(Dst_DECL, size_t *szp) +{ + dasm_State *D = Dst_REF; + int secnum; + int ofs = 0; + +#ifdef DASM_CHECKS + *szp = 0; + if (D->status != DASM_S_OK) return D->status; + { + int pc; + for (pc = 0; pc*sizeof(int) < D->pcsize; pc++) + if (D->pclabels[pc] > 0) return DASM_S_UNDEF_PC|pc; + } +#endif + + { /* Handle globals not defined in this translation unit. */ + int idx; + for (idx = 10; idx*sizeof(int) < D->lgsize; idx++) { + int n = D->lglabels[idx]; + /* Undefined label: Collapse rel chain and replace with marker (< 0). */ + while (n > 0) { int *pb = DASM_POS2PTR(D, n); n = *pb; *pb = -idx; } + } + } + + /* Combine all code sections. No support for data sections (yet). */ + for (secnum = 0; secnum < D->maxsection; secnum++) { + dasm_Section *sec = D->sections + secnum; + int *b = sec->rbuf; + int pos = DASM_SEC2POS(secnum); + int lastpos = sec->pos; + + while (pos != lastpos) { + dasm_ActList p = D->actionlist + b[pos++]; + while (1) { + int op, action = *p++; + switch (action) { + case DASM_REL_LG: p++; op = p[-3]; goto rel_pc; + case DASM_REL_PC: op = p[-2]; rel_pc: { + int shrink = op == 0xe9 ? 3 : ((op&0xf0) == 0x80 ? 4 : 0); + if (shrink) { /* Shrinkable branch opcode? */ + int lofs, lpos = b[pos]; + if (lpos < 0) goto noshrink; /* Ext global? */ + lofs = *DASM_POS2PTR(D, lpos); + if (lpos > pos) { /* Fwd label: add cumulative section offsets. */ + int i; + for (i = secnum; i < DASM_POS2SEC(lpos); i++) + lofs += D->sections[i].ofs; + } else { + lofs -= ofs; /* Bkwd label: unfix offset. */ + } + lofs -= b[pos+1]; /* Short branch ok? */ + if (lofs >= -128-shrink && lofs <= 127) ofs -= shrink; /* Yes. */ + else { noshrink: shrink = 0; } /* No, cannot shrink op. */ + } + b[pos+1] = shrink; + pos += 2; + break; + } + case DASM_SPACE: case DASM_IMM_LG: case DASM_VREG: p++; + case DASM_DISP: case DASM_IMM_S: case DASM_IMM_B: case DASM_IMM_W: + case DASM_IMM_D: case DASM_IMM_WB: case DASM_IMM_DB: + case DASM_SETLABEL: case DASM_REL_A: case DASM_IMM_PC: pos++; break; + case DASM_LABEL_LG: p++; + case DASM_LABEL_PC: b[pos++] += ofs; break; /* Fix label offset. */ + case DASM_ALIGN: ofs -= (b[pos++]+ofs)&*p++; break; /* Adjust ofs. */ + case DASM_EXTERN: p += 2; break; + case DASM_ESC: p++; break; + case DASM_MARK: break; + case DASM_SECTION: case DASM_STOP: goto stop; + } + } + stop: (void)0; + } + ofs += sec->ofs; /* Next section starts right after current section. */ + } + + D->codesize = ofs; /* Total size of all code sections */ + *szp = ofs; + return DASM_S_OK; +} + +#define dasmb(x) *cp++ = (unsigned char)(x) +#ifndef DASM_ALIGNED_WRITES +#define dasmw(x) \ + do { *((unsigned short *)cp) = (unsigned short)(x); cp+=2; } while (0) +#define dasmd(x) \ + do { *((unsigned int *)cp) = (unsigned int)(x); cp+=4; } while (0) +#else +#define dasmw(x) do { dasmb(x); dasmb((x)>>8); } while (0) +#define dasmd(x) do { dasmw(x); dasmw((x)>>16); } while (0) +#endif + +/* Pass 3: Encode sections. */ +int dasm_encode(Dst_DECL, void *buffer) +{ + dasm_State *D = Dst_REF; + unsigned char *base = (unsigned char *)buffer; + unsigned char *cp = base; + int secnum; + + /* Encode all code sections. No support for data sections (yet). */ + for (secnum = 0; secnum < D->maxsection; secnum++) { + dasm_Section *sec = D->sections + secnum; + int *b = sec->buf; + int *endb = sec->rbuf + sec->pos; + + while (b != endb) { + dasm_ActList p = D->actionlist + *b++; + unsigned char *mark = NULL; + while (1) { + int action = *p++; + int n = (action >= DASM_DISP && action <= DASM_ALIGN) ? *b++ : 0; + switch (action) { + case DASM_DISP: if (!mark) mark = cp; { + unsigned char *mm = mark; + if (*p != DASM_IMM_DB && *p != DASM_IMM_WB) mark = NULL; + if (n == 0) { int mrm = mm[-1]&7; if (mrm == 4) mrm = mm[0]&7; + if (mrm != 5) { mm[-1] -= 0x80; break; } } + if (((n+128) & -256) != 0) goto wd; else mm[-1] -= 0x40; + } + case DASM_IMM_S: case DASM_IMM_B: wb: dasmb(n); break; + case DASM_IMM_DB: if (((n+128)&-256) == 0) { + db: if (!mark) mark = cp; mark[-2] += 2; mark = NULL; goto wb; + } else mark = NULL; + case DASM_IMM_D: wd: dasmd(n); break; + case DASM_IMM_WB: if (((n+128)&-256) == 0) goto db; else mark = NULL; + case DASM_IMM_W: dasmw(n); break; + case DASM_VREG: { + int t = *p++; + unsigned char *ex = cp - (t&7); + if ((n & 8) && t < 0xa0) { + if (*ex & 0x80) ex[1] ^= 0x20 << (t>>6); else *ex ^= 1 << (t>>6); + n &= 7; + } else if (n & 0x10) { + if (*ex & 0x80) { + *ex = 0xc5; ex[1] = (ex[1] & 0x80) | ex[2]; ex += 2; + } + while (++ex < cp) ex[-1] = *ex; + if (mark) mark--; + cp--; + n &= 7; + } + if (t >= 0xc0) n <<= 4; + else if (t >= 0x40) n <<= 3; + else if (n == 4 && t < 0x20) { cp[-1] ^= n; *cp++ = 0x20; } + cp[-1] ^= n; + break; + } + case DASM_REL_LG: p++; if (n >= 0) goto rel_pc; + b++; n = (int)(ptrdiff_t)D->globals[-n]; + case DASM_REL_A: rel_a: n -= (int)(ptrdiff_t)(cp+4); goto wd; /* !x64 */ + case DASM_REL_PC: rel_pc: { + int shrink = *b++; + int *pb = DASM_POS2PTR(D, n); if (*pb < 0) { n = pb[1]; goto rel_a; } + n = *pb - ((int)(cp-base) + 4-shrink); + if (shrink == 0) goto wd; + if (shrink == 4) { cp--; cp[-1] = *cp-0x10; } else cp[-1] = 0xeb; + goto wb; + } + case DASM_IMM_LG: + p++; if (n < 0) { n = (int)(ptrdiff_t)D->globals[-n]; goto wd; } + case DASM_IMM_PC: { + int *pb = DASM_POS2PTR(D, n); + n = *pb < 0 ? pb[1] : (*pb + (int)(ptrdiff_t)base); + goto wd; + } + case DASM_LABEL_LG: { + int idx = *p++; + if (idx >= 10) + D->globals[idx] = (void *)(base + (*p == DASM_SETLABEL ? *b : n)); + break; + } + case DASM_LABEL_PC: case DASM_SETLABEL: break; + case DASM_SPACE: { int fill = *p++; while (n--) *cp++ = fill; break; } + case DASM_ALIGN: + n = *p++; + while (((cp-base) & n)) *cp++ = 0x90; /* nop */ + break; + case DASM_EXTERN: n = DASM_EXTERN(Dst, cp, p[1], *p); p += 2; goto wd; + case DASM_MARK: mark = cp; break; + case DASM_ESC: action = *p++; + default: *cp++ = action; break; + case DASM_SECTION: case DASM_STOP: goto stop; + } + } + stop: (void)0; + } + } + + if (base + D->codesize != cp) /* Check for phase errors. */ + return DASM_S_PHASE; + return DASM_S_OK; +} + +/* Get PC label offset. */ +int dasm_getpclabel(Dst_DECL, unsigned int pc) +{ + dasm_State *D = Dst_REF; + if (pc*sizeof(int) < D->pcsize) { + int pos = D->pclabels[pc]; + if (pos < 0) return *DASM_POS2PTR(D, -pos); + if (pos > 0) return -1; /* Undefined. */ + } + return -2; /* Unused or out of range. */ +} + +#ifdef DASM_CHECKS +/* Optional sanity checker to call between isolated encoding steps. */ +int dasm_checkstep(Dst_DECL, int secmatch) +{ + dasm_State *D = Dst_REF; + if (D->status == DASM_S_OK) { + int i; + for (i = 1; i <= 9; i++) { + if (D->lglabels[i] > 0) { D->status = DASM_S_UNDEF_L|i; break; } + D->lglabels[i] = 0; + } + } + if (D->status == DASM_S_OK && secmatch >= 0 && + D->section != &D->sections[secmatch]) + D->status = DASM_S_MATCH_SEC|(int)(D->section-D->sections); + return D->status; +} +#endif + diff --git a/ext/opcache/dynasm/dasm_x86.lua b/ext/opcache/dynasm/dasm_x86.lua new file mode 100644 index 0000000000000..a5efd98fb5f39 --- /dev/null +++ b/ext/opcache/dynasm/dasm_x86.lua @@ -0,0 +1,2274 @@ +------------------------------------------------------------------------------ +-- DynASM x86/x64 module. +-- +-- Copyright (C) 2005-2016 Mike Pall. All rights reserved. +-- See dynasm.lua for full copyright notice. +------------------------------------------------------------------------------ + +local x64 = x64 + +-- Module information: +local _info = { + arch = x64 and "x64" or "x86", + description = "DynASM x86/x64 module", + version = "1.4.0", + vernum = 10400, + release = "2015-10-18", + author = "Mike Pall", + license = "MIT", +} + +-- Exported glue functions for the arch-specific module. +local _M = { _info = _info } + +-- Cache library functions. +local type, tonumber, pairs, ipairs = type, tonumber, pairs, ipairs +local assert, unpack, setmetatable = assert, unpack or table.unpack, setmetatable +local _s = string +local sub, format, byte, char = _s.sub, _s.format, _s.byte, _s.char +local find, match, gmatch, gsub = _s.find, _s.match, _s.gmatch, _s.gsub +local concat, sort, remove = table.concat, table.sort, table.remove +local bit = bit or require("bit") +local band, bxor, shl, shr = bit.band, bit.bxor, bit.lshift, bit.rshift + +-- Inherited tables and callbacks. +local g_opt, g_arch +local wline, werror, wfatal, wwarn + +-- Action name list. +-- CHECK: Keep this in sync with the C code! +local action_names = { + -- int arg, 1 buffer pos: + "DISP", "IMM_S", "IMM_B", "IMM_W", "IMM_D", "IMM_WB", "IMM_DB", + -- action arg (1 byte), int arg, 1 buffer pos (reg/num): + "VREG", "SPACE", + -- ptrdiff_t arg, 1 buffer pos (address): !x64 + "SETLABEL", "REL_A", + -- action arg (1 byte) or int arg, 2 buffer pos (link, offset): + "REL_LG", "REL_PC", + -- action arg (1 byte) or int arg, 1 buffer pos (link): + "IMM_LG", "IMM_PC", + -- action arg (1 byte) or int arg, 1 buffer pos (offset): + "LABEL_LG", "LABEL_PC", + -- action arg (1 byte), 1 buffer pos (offset): + "ALIGN", + -- action args (2 bytes), no buffer pos. + "EXTERN", + -- action arg (1 byte), no buffer pos. + "ESC", + -- no action arg, no buffer pos. + "MARK", + -- action arg (1 byte), no buffer pos, terminal action: + "SECTION", + -- no args, no buffer pos, terminal action: + "STOP" +} + +-- Maximum number of section buffer positions for dasm_put(). +-- CHECK: Keep this in sync with the C code! +local maxsecpos = 25 -- Keep this low, to avoid excessively long C lines. + +-- Action name -> action number (dynamically generated below). +local map_action = {} +-- First action number. Everything below does not need to be escaped. +local actfirst = 256-#action_names + +-- Action list buffer and string (only used to remove dupes). +local actlist = {} +local actstr = "" + +-- Argument list for next dasm_put(). Start with offset 0 into action list. +local actargs = { 0 } + +-- Current number of section buffer positions for dasm_put(). +local secpos = 1 + +-- VREG kind encodings, pre-shifted by 5 bits. +local map_vreg = { + ["modrm.rm.m"] = 0x00, + ["modrm.rm.r"] = 0x20, + ["opcode"] = 0x20, + ["sib.base"] = 0x20, + ["sib.index"] = 0x40, + ["modrm.reg"] = 0x80, + ["vex.v"] = 0xa0, + ["imm.hi"] = 0xc0, +} + +-- Current number of VREG actions contributing to REX/VEX shrinkage. +local vreg_shrink_count = 0 + +------------------------------------------------------------------------------ + +-- Compute action numbers for action names. +for n,name in ipairs(action_names) do + local num = actfirst + n - 1 + map_action[name] = num +end + +-- Dump action names and numbers. +local function dumpactions(out) + out:write("DynASM encoding engine action codes:\n") + for n,name in ipairs(action_names) do + local num = map_action[name] + out:write(format(" %-10s %02X %d\n", name, num, num)) + end + out:write("\n") +end + +-- Write action list buffer as a huge static C array. +local function writeactions(out, name) + local nn = #actlist + local last = actlist[nn] or 255 + actlist[nn] = nil -- Remove last byte. + if nn == 0 then nn = 1 end + out:write("static const unsigned char ", name, "[", nn, "] = {\n") + local s = " " + for n,b in ipairs(actlist) do + s = s..b.."," + if #s >= 75 then + assert(out:write(s, "\n")) + s = " " + end + end + out:write(s, last, "\n};\n\n") -- Add last byte back. +end + +------------------------------------------------------------------------------ + +-- Add byte to action list. +local function wputxb(n) + assert(n >= 0 and n <= 255 and n % 1 == 0, "byte out of range") + actlist[#actlist+1] = n +end + +-- Add action to list with optional arg. Advance buffer pos, too. +local function waction(action, a, num) + wputxb(assert(map_action[action], "bad action name `"..action.."'")) + if a then actargs[#actargs+1] = a end + if a or num then secpos = secpos + (num or 1) end +end + +-- Optionally add a VREG action. +local function wvreg(kind, vreg, psz, sk, defer) + if not vreg then return end + waction("VREG", vreg) + local b = assert(map_vreg[kind], "bad vreg kind `"..vreg.."'") + if b < (sk or 0) then + vreg_shrink_count = vreg_shrink_count + 1 + end + if not defer then + b = b + vreg_shrink_count * 8 + vreg_shrink_count = 0 + end + wputxb(b + (psz or 0)) +end + +-- Add call to embedded DynASM C code. +local function wcall(func, args) + wline(format("dasm_%s(Dst, %s);", func, concat(args, ", ")), true) +end + +-- Delete duplicate action list chunks. A tad slow, but so what. +local function dedupechunk(offset) + local al, as = actlist, actstr + local chunk = char(unpack(al, offset+1, #al)) + local orig = find(as, chunk, 1, true) + if orig then + actargs[1] = orig-1 -- Replace with original offset. + for i=offset+1,#al do al[i] = nil end -- Kill dupe. + else + actstr = as..chunk + end +end + +-- Flush action list (intervening C code or buffer pos overflow). +local function wflush(term) + local offset = actargs[1] + if #actlist == offset then return end -- Nothing to flush. + if not term then waction("STOP") end -- Terminate action list. + dedupechunk(offset) + wcall("put", actargs) -- Add call to dasm_put(). + actargs = { #actlist } -- Actionlist offset is 1st arg to next dasm_put(). + secpos = 1 -- The actionlist offset occupies a buffer position, too. +end + +-- Put escaped byte. +local function wputb(n) + if n >= actfirst then waction("ESC") end -- Need to escape byte. + wputxb(n) +end + +------------------------------------------------------------------------------ + +-- Global label name -> global label number. With auto assignment on 1st use. +local next_global = 10 +local map_global = setmetatable({}, { __index = function(t, name) + if not match(name, "^[%a_][%w_@]*$") then werror("bad global label") end + local n = next_global + if n > 246 then werror("too many global labels") end + next_global = n + 1 + t[name] = n + return n +end}) + +-- Dump global labels. +local function dumpglobals(out, lvl) + local t = {} + for name, n in pairs(map_global) do t[n] = name end + out:write("Global labels:\n") + for i=10,next_global-1 do + out:write(format(" %s\n", t[i])) + end + out:write("\n") +end + +-- Write global label enum. +local function writeglobals(out, prefix) + local t = {} + for name, n in pairs(map_global) do t[n] = name end + out:write("enum {\n") + for i=10,next_global-1 do + out:write(" ", prefix, gsub(t[i], "@.*", ""), ",\n") + end + out:write(" ", prefix, "_MAX\n};\n") +end + +-- Write global label names. +local function writeglobalnames(out, name) + local t = {} + for name, n in pairs(map_global) do t[n] = name end + out:write("static const char *const ", name, "[] = {\n") + for i=10,next_global-1 do + out:write(" \"", t[i], "\",\n") + end + out:write(" (const char *)0\n};\n") +end + +------------------------------------------------------------------------------ + +-- Extern label name -> extern label number. With auto assignment on 1st use. +local next_extern = -1 +local map_extern = setmetatable({}, { __index = function(t, name) + -- No restrictions on the name for now. + local n = next_extern + if n < -256 then werror("too many extern labels") end + next_extern = n - 1 + t[name] = n + return n +end}) + +-- Dump extern labels. +local function dumpexterns(out, lvl) + local t = {} + for name, n in pairs(map_extern) do t[-n] = name end + out:write("Extern labels:\n") + for i=1,-next_extern-1 do + out:write(format(" %s\n", t[i])) + end + out:write("\n") +end + +-- Write extern label names. +local function writeexternnames(out, name) + local t = {} + for name, n in pairs(map_extern) do t[-n] = name end + out:write("static const char *const ", name, "[] = {\n") + for i=1,-next_extern-1 do + out:write(" \"", t[i], "\",\n") + end + out:write(" (const char *)0\n};\n") +end + +------------------------------------------------------------------------------ + +-- Arch-specific maps. +local map_archdef = {} -- Ext. register name -> int. name. +local map_reg_rev = {} -- Int. register name -> ext. name. +local map_reg_num = {} -- Int. register name -> register number. +local map_reg_opsize = {} -- Int. register name -> operand size. +local map_reg_valid_base = {} -- Int. register name -> valid base register? +local map_reg_valid_index = {} -- Int. register name -> valid index register? +local map_reg_needrex = {} -- Int. register name -> need rex vs. no rex. +local reg_list = {} -- Canonical list of int. register names. + +local map_type = {} -- Type name -> { ctype, reg } +local ctypenum = 0 -- Type number (for _PTx macros). + +local addrsize = x64 and "q" or "d" -- Size for address operands. + +-- Helper functions to fill register maps. +local function mkrmap(sz, cl, names) + local cname = format("@%s", sz) + reg_list[#reg_list+1] = cname + map_archdef[cl] = cname + map_reg_rev[cname] = cl + map_reg_num[cname] = -1 + map_reg_opsize[cname] = sz + if sz == addrsize or sz == "d" then + map_reg_valid_base[cname] = true + map_reg_valid_index[cname] = true + end + if names then + for n,name in ipairs(names) do + local iname = format("@%s%x", sz, n-1) + reg_list[#reg_list+1] = iname + map_archdef[name] = iname + map_reg_rev[iname] = name + map_reg_num[iname] = n-1 + map_reg_opsize[iname] = sz + if sz == "b" and n > 4 then map_reg_needrex[iname] = false end + if sz == addrsize or sz == "d" then + map_reg_valid_base[iname] = true + map_reg_valid_index[iname] = true + end + end + end + for i=0,(x64 and sz ~= "f") and 15 or 7 do + local needrex = sz == "b" and i > 3 + local iname = format("@%s%x%s", sz, i, needrex and "R" or "") + if needrex then map_reg_needrex[iname] = true end + local name + if sz == "o" or sz == "y" then name = format("%s%d", cl, i) + elseif sz == "f" then name = format("st%d", i) + else name = format("r%d%s", i, sz == addrsize and "" or sz) end + map_archdef[name] = iname + if not map_reg_rev[iname] then + reg_list[#reg_list+1] = iname + map_reg_rev[iname] = name + map_reg_num[iname] = i + map_reg_opsize[iname] = sz + if sz == addrsize or sz == "d" then + map_reg_valid_base[iname] = true + map_reg_valid_index[iname] = true + end + end + end + reg_list[#reg_list+1] = "" +end + +-- Integer registers (qword, dword, word and byte sized). +if x64 then + mkrmap("q", "Rq", {"rax", "rcx", "rdx", "rbx", "rsp", "rbp", "rsi", "rdi"}) +end +mkrmap("d", "Rd", {"eax", "ecx", "edx", "ebx", "esp", "ebp", "esi", "edi"}) +mkrmap("w", "Rw", {"ax", "cx", "dx", "bx", "sp", "bp", "si", "di"}) +mkrmap("b", "Rb", {"al", "cl", "dl", "bl", "ah", "ch", "dh", "bh"}) +map_reg_valid_index[map_archdef.esp] = false +if x64 then map_reg_valid_index[map_archdef.rsp] = false end +if x64 then map_reg_needrex[map_archdef.Rb] = true end +map_archdef["Ra"] = "@"..addrsize + +-- FP registers (internally tword sized, but use "f" as operand size). +mkrmap("f", "Rf") + +-- SSE registers (oword sized, but qword and dword accessible). +mkrmap("o", "xmm") + +-- AVX registers (yword sized, but oword, qword and dword accessible). +mkrmap("y", "ymm") + +-- Operand size prefixes to codes. +local map_opsize = { + byte = "b", word = "w", dword = "d", qword = "q", oword = "o", yword = "y", + tword = "t", aword = addrsize, +} + +-- Operand size code to number. +local map_opsizenum = { + b = 1, w = 2, d = 4, q = 8, o = 16, y = 32, t = 10, +} + +-- Operand size code to name. +local map_opsizename = { + b = "byte", w = "word", d = "dword", q = "qword", o = "oword", y = "yword", + t = "tword", f = "fpword", +} + +-- Valid index register scale factors. +local map_xsc = { + ["1"] = 0, ["2"] = 1, ["4"] = 2, ["8"] = 3, +} + +-- Condition codes. +local map_cc = { + o = 0, no = 1, b = 2, nb = 3, e = 4, ne = 5, be = 6, nbe = 7, + s = 8, ns = 9, p = 10, np = 11, l = 12, nl = 13, le = 14, nle = 15, + c = 2, nae = 2, nc = 3, ae = 3, z = 4, nz = 5, na = 6, a = 7, + pe = 10, po = 11, nge = 12, ge = 13, ng = 14, g = 15, +} + + +-- Reverse defines for registers. +function _M.revdef(s) + return gsub(s, "@%w+", map_reg_rev) +end + +-- Dump register names and numbers +local function dumpregs(out) + out:write("Register names, sizes and internal numbers:\n") + for _,reg in ipairs(reg_list) do + if reg == "" then + out:write("\n") + else + local name = map_reg_rev[reg] + local num = map_reg_num[reg] + local opsize = map_opsizename[map_reg_opsize[reg]] + out:write(format(" %-5s %-8s %s\n", name, opsize, + num < 0 and "(variable)" or num)) + end + end +end + +------------------------------------------------------------------------------ + +-- Put action for label arg (IMM_LG, IMM_PC, REL_LG, REL_PC). +local function wputlabel(aprefix, imm, num) + if type(imm) == "number" then + if imm < 0 then + waction("EXTERN") + wputxb(aprefix == "IMM_" and 0 or 1) + imm = -imm-1 + else + waction(aprefix.."LG", nil, num); + end + wputxb(imm) + else + waction(aprefix.."PC", imm, num) + end +end + +-- Put signed byte or arg. +local function wputsbarg(n) + if type(n) == "number" then + if n < -128 or n > 127 then + werror("signed immediate byte out of range") + end + if n < 0 then n = n + 256 end + wputb(n) + else waction("IMM_S", n) end +end + +-- Put unsigned byte or arg. +local function wputbarg(n) + if type(n) == "number" then + if n < 0 or n > 255 then + werror("unsigned immediate byte out of range") + end + wputb(n) + else waction("IMM_B", n) end +end + +-- Put unsigned word or arg. +local function wputwarg(n) + if type(n) == "number" then + if shr(n, 16) ~= 0 then + werror("unsigned immediate word out of range") + end + wputb(band(n, 255)); wputb(shr(n, 8)); + else waction("IMM_W", n) end +end + +-- Put signed or unsigned dword or arg. +local function wputdarg(n) + local tn = type(n) + if tn == "number" then + wputb(band(n, 255)) + wputb(band(shr(n, 8), 255)) + wputb(band(shr(n, 16), 255)) + wputb(shr(n, 24)) + elseif tn == "table" then + wputlabel("IMM_", n[1], 1) + else + waction("IMM_D", n) + end +end + +-- Put operand-size dependent number or arg (defaults to dword). +local function wputszarg(sz, n) + if not sz or sz == "d" or sz == "q" then wputdarg(n) + elseif sz == "w" then wputwarg(n) + elseif sz == "b" then wputbarg(n) + elseif sz == "s" then wputsbarg(n) + else werror("bad operand size") end +end + +-- Put multi-byte opcode with operand-size dependent modifications. +local function wputop(sz, op, rex, vex, vregr, vregxb) + local psz, sk = 0, nil + if vex then + local tail + if vex.m == 1 and band(rex, 11) == 0 then + if x64 and vregxb then + sk = map_vreg["modrm.reg"] + else + wputb(0xc5) + tail = shl(bxor(band(rex, 4), 4), 5) + psz = 3 + end + end + if not tail then + wputb(0xc4) + wputb(shl(bxor(band(rex, 7), 7), 5) + vex.m) + tail = shl(band(rex, 8), 4) + psz = 4 + end + local reg, vreg = 0, nil + if vex.v then + reg = vex.v.reg + if not reg then werror("bad vex operand") end + if reg < 0 then reg = 0; vreg = vex.v.vreg end + end + if sz == "y" or vex.l then tail = tail + 4 end + wputb(tail + shl(bxor(reg, 15), 3) + vex.p) + wvreg("vex.v", vreg) + rex = 0 + if op >= 256 then werror("bad vex opcode") end + else + if rex ~= 0 then + if not x64 then werror("bad operand size") end + elseif (vregr or vregxb) and x64 then + rex = 0x10 + sk = map_vreg["vex.v"] + end + end + local r + if sz == "w" then wputb(102) end + -- Needs >32 bit numbers, but only for crc32 eax, word [ebx] + if op >= 4294967296 then r = op%4294967296 wputb((op-r)/4294967296) op = r end + if op >= 16777216 then wputb(shr(op, 24)); op = band(op, 0xffffff) end + if op >= 65536 then + if rex ~= 0 then + local opc3 = band(op, 0xffff00) + if opc3 == 0x0f3a00 or opc3 == 0x0f3800 then + wputb(64 + band(rex, 15)); rex = 0; psz = 2 + end + end + wputb(shr(op, 16)); op = band(op, 0xffff); psz = psz + 1 + end + if op >= 256 then + local b = shr(op, 8) + if b == 15 and rex ~= 0 then wputb(64 + band(rex, 15)); rex = 0; psz = 2 end + wputb(b); op = band(op, 255); psz = psz + 1 + end + if rex ~= 0 then wputb(64 + band(rex, 15)); psz = 2 end + if sz == "b" then op = op - 1 end + wputb(op) + return psz, sk +end + +-- Put ModRM or SIB formatted byte. +local function wputmodrm(m, s, rm, vs, vrm) + assert(m < 4 and s < 16 and rm < 16, "bad modrm operands") + wputb(shl(m, 6) + shl(band(s, 7), 3) + band(rm, 7)) +end + +-- Put ModRM/SIB plus optional displacement. +local function wputmrmsib(t, imark, s, vsreg, psz, sk) + local vreg, vxreg + local reg, xreg = t.reg, t.xreg + if reg and reg < 0 then reg = 0; vreg = t.vreg end + if xreg and xreg < 0 then xreg = 0; vxreg = t.vxreg end + if s < 0 then s = 0 end + + -- Register mode. + if sub(t.mode, 1, 1) == "r" then + wputmodrm(3, s, reg) + wvreg("modrm.reg", vsreg, psz+1, sk, vreg) + wvreg("modrm.rm.r", vreg, psz+1, sk) + return + end + + local disp = t.disp + local tdisp = type(disp) + -- No base register? + if not reg then + local riprel = false + if xreg then + -- Indexed mode with index register only. + -- [xreg*xsc+disp] -> (0, s, esp) (xsc, xreg, ebp) + wputmodrm(0, s, 4) + if imark == "I" then waction("MARK") end + wvreg("modrm.reg", vsreg, psz+1, sk, vxreg) + wputmodrm(t.xsc, xreg, 5) + wvreg("sib.index", vxreg, psz+2, sk) + else + -- Pure 32 bit displacement. + if x64 and tdisp ~= "table" then + wputmodrm(0, s, 4) -- [disp] -> (0, s, esp) (0, esp, ebp) + wvreg("modrm.reg", vsreg, psz+1, sk) + if imark == "I" then waction("MARK") end + wputmodrm(0, 4, 5) + else + riprel = x64 + wputmodrm(0, s, 5) -- [disp|rip-label] -> (0, s, ebp) + wvreg("modrm.reg", vsreg, psz+1, sk) + if imark == "I" then waction("MARK") end + end + end + if riprel then -- Emit rip-relative displacement. + if match("UWSiI", imark) then + werror("NYI: rip-relative displacement followed by immediate") + end + -- The previous byte in the action buffer cannot be 0xe9 or 0x80-0x8f. + wputlabel("REL_", disp[1], 2) + else + wputdarg(disp) + end + return + end + + local m + if tdisp == "number" then -- Check displacement size at assembly time. + if disp == 0 and band(reg, 7) ~= 5 then -- [ebp] -> [ebp+0] (in SIB, too) + if not vreg then m = 0 end -- Force DISP to allow [Rd(5)] -> [ebp+0] + elseif disp >= -128 and disp <= 127 then m = 1 + else m = 2 end + elseif tdisp == "table" then + m = 2 + end + + -- Index register present or esp as base register: need SIB encoding. + if xreg or band(reg, 7) == 4 then + wputmodrm(m or 2, s, 4) -- ModRM. + if m == nil or imark == "I" then waction("MARK") end + wvreg("modrm.reg", vsreg, psz+1, sk, vxreg or vreg) + wputmodrm(t.xsc or 0, xreg or 4, reg) -- SIB. + wvreg("sib.index", vxreg, psz+2, sk, vreg) + wvreg("sib.base", vreg, psz+2, sk) + else + wputmodrm(m or 2, s, reg) -- ModRM. + if (imark == "I" and (m == 1 or m == 2)) or + (m == nil and (vsreg or vreg)) then waction("MARK") end + wvreg("modrm.reg", vsreg, psz+1, sk, vreg) + wvreg("modrm.rm.m", vreg, psz+1, sk) + end + + -- Put displacement. + if m == 1 then wputsbarg(disp) + elseif m == 2 then wputdarg(disp) + elseif m == nil then waction("DISP", disp) end +end + +------------------------------------------------------------------------------ + +-- Return human-readable operand mode string. +local function opmodestr(op, args) + local m = {} + for i=1,#args do + local a = args[i] + m[#m+1] = sub(a.mode, 1, 1)..(a.opsize or "?") + end + return op.." "..concat(m, ",") +end + +-- Convert number to valid integer or nil. +local function toint(expr) + local n = tonumber(expr) + if n then + if n % 1 ~= 0 or n < -2147483648 or n > 4294967295 then + werror("bad integer number `"..expr.."'") + end + return n + end +end + +-- Parse immediate expression. +local function immexpr(expr) + -- &expr (pointer) + if sub(expr, 1, 1) == "&" then + return "iPJ", format("(ptrdiff_t)(%s)", sub(expr,2)) + end + + local prefix = sub(expr, 1, 2) + -- =>expr (pc label reference) + if prefix == "=>" then + return "iJ", sub(expr, 3) + end + -- ->name (global label reference) + if prefix == "->" then + return "iJ", map_global[sub(expr, 3)] + end + + -- [<>][1-9] (local label reference) + local dir, lnum = match(expr, "^([<>])([1-9])$") + if dir then -- Fwd: 247-255, Bkwd: 1-9. + return "iJ", lnum + (dir == ">" and 246 or 0) + end + + local extname = match(expr, "^extern%s+(%S+)$") + if extname then + return "iJ", map_extern[extname] + end + + -- expr (interpreted as immediate) + return "iI", expr +end + +-- Parse displacement expression: +-num, +-expr, +-opsize*num +local function dispexpr(expr) + local disp = expr == "" and 0 or toint(expr) + if disp then return disp end + local c, dispt = match(expr, "^([+-])%s*(.+)$") + if c == "+" then + expr = dispt + elseif not c then + werror("bad displacement expression `"..expr.."'") + end + local opsize, tailops = match(dispt, "^(%w+)%s*%*%s*(.+)$") + local ops, imm = map_opsize[opsize], toint(tailops) + if ops and imm then + if c == "-" then imm = -imm end + return imm*map_opsizenum[ops] + end + local mode, iexpr = immexpr(dispt) + if mode == "iJ" then + if c == "-" then werror("cannot invert label reference") end + return { iexpr } + end + return expr -- Need to return original signed expression. +end + +-- Parse register or type expression. +local function rtexpr(expr) + if not expr then return end + local tname, ovreg = match(expr, "^([%w_]+):(@[%w_]+)$") + local tp = map_type[tname or expr] + if tp then + local reg = ovreg or tp.reg + local rnum = map_reg_num[reg] + if not rnum then + werror("type `"..(tname or expr).."' needs a register override") + end + if not map_reg_valid_base[reg] then + werror("bad base register override `"..(map_reg_rev[reg] or reg).."'") + end + return reg, rnum, tp + end + return expr, map_reg_num[expr] +end + +-- Parse operand and return { mode, opsize, reg, xreg, xsc, disp, imm }. +local function parseoperand(param) + local t = {} + + local expr = param + local opsize, tailops = match(param, "^(%w+)%s*(.+)$") + if opsize then + t.opsize = map_opsize[opsize] + if t.opsize then expr = tailops end + end + + local br = match(expr, "^%[%s*(.-)%s*%]$") + repeat + if br then + t.mode = "xm" + + -- [disp] + t.disp = toint(br) + if t.disp then + t.mode = x64 and "xm" or "xmO" + break + end + + -- [reg...] + local tp + local reg, tailr = match(br, "^([@%w_:]+)%s*(.*)$") + reg, t.reg, tp = rtexpr(reg) + if not t.reg then + -- [expr] + t.mode = x64 and "xm" or "xmO" + t.disp = dispexpr("+"..br) + break + end + + if t.reg == -1 then + t.vreg, tailr = match(tailr, "^(%b())(.*)$") + if not t.vreg then werror("bad variable register expression") end + end + + -- [xreg*xsc] or [xreg*xsc+-disp] or [xreg*xsc+-expr] + local xsc, tailsc = match(tailr, "^%*%s*([1248])%s*(.*)$") + if xsc then + if not map_reg_valid_index[reg] then + werror("bad index register `"..map_reg_rev[reg].."'") + end + t.xsc = map_xsc[xsc] + t.xreg = t.reg + t.vxreg = t.vreg + t.reg = nil + t.vreg = nil + t.disp = dispexpr(tailsc) + break + end + if not map_reg_valid_base[reg] then + werror("bad base register `"..map_reg_rev[reg].."'") + end + + -- [reg] or [reg+-disp] + t.disp = toint(tailr) or (tailr == "" and 0) + if t.disp then break end + + -- [reg+xreg...] + local xreg, tailx = match(tailr, "^+%s*([@%w_:]+)%s*(.*)$") + xreg, t.xreg, tp = rtexpr(xreg) + if not t.xreg then + -- [reg+-expr] + t.disp = dispexpr(tailr) + break + end + if not map_reg_valid_index[xreg] then + werror("bad index register `"..map_reg_rev[xreg].."'") + end + + if t.xreg == -1 then + t.vxreg, tailx = match(tailx, "^(%b())(.*)$") + if not t.vxreg then werror("bad variable register expression") end + end + + -- [reg+xreg*xsc...] + local xsc, tailsc = match(tailx, "^%*%s*([1248])%s*(.*)$") + if xsc then + t.xsc = map_xsc[xsc] + tailx = tailsc + end + + -- [...] or [...+-disp] or [...+-expr] + t.disp = dispexpr(tailx) + else + -- imm or opsize*imm + local imm = toint(expr) + if not imm and sub(expr, 1, 1) == "*" and t.opsize then + imm = toint(sub(expr, 2)) + if imm then + imm = imm * map_opsizenum[t.opsize] + t.opsize = nil + end + end + if imm then + if t.opsize then werror("bad operand size override") end + local m = "i" + if imm == 1 then m = m.."1" end + if imm >= 4294967168 and imm <= 4294967295 then imm = imm-4294967296 end + if imm >= -128 and imm <= 127 then m = m.."S" end + t.imm = imm + t.mode = m + break + end + + local tp + local reg, tailr = match(expr, "^([@%w_:]+)%s*(.*)$") + reg, t.reg, tp = rtexpr(reg) + if t.reg then + if t.reg == -1 then + t.vreg, tailr = match(tailr, "^(%b())(.*)$") + if not t.vreg then werror("bad variable register expression") end + end + -- reg + if tailr == "" then + if t.opsize then werror("bad operand size override") end + t.opsize = map_reg_opsize[reg] + if t.opsize == "f" then + t.mode = t.reg == 0 and "fF" or "f" + else + if reg == "@w4" or (x64 and reg == "@d4") then + wwarn("bad idea, try again with `"..(x64 and "rsp'" or "esp'")) + end + t.mode = t.reg == 0 and "rmR" or (reg == "@b1" and "rmC" or "rm") + end + t.needrex = map_reg_needrex[reg] + break + end + + -- type[idx], type[idx].field, type->field -> [reg+offset_expr] + if not tp then werror("bad operand `"..param.."'") end + t.mode = "xm" + t.disp = format(tp.ctypefmt, tailr) + else + t.mode, t.imm = immexpr(expr) + if sub(t.mode, -1) == "J" then + if t.opsize and t.opsize ~= addrsize then + werror("bad operand size override") + end + t.opsize = addrsize + end + end + end + until true + return t +end + +------------------------------------------------------------------------------ +-- x86 Template String Description +-- =============================== +-- +-- Each template string is a list of [match:]pattern pairs, +-- separated by "|". The first match wins. No match means a +-- bad or unsupported combination of operand modes or sizes. +-- +-- The match part and the ":" is omitted if the operation has +-- no operands. Otherwise the first N characters are matched +-- against the mode strings of each of the N operands. +-- +-- The mode string for each operand type is (see parseoperand()): +-- Integer register: "rm", +"R" for eax, ax, al, +"C" for cl +-- FP register: "f", +"F" for st0 +-- Index operand: "xm", +"O" for [disp] (pure offset) +-- Immediate: "i", +"S" for signed 8 bit, +"1" for 1, +-- +"I" for arg, +"P" for pointer +-- Any: +"J" for valid jump targets +-- +-- So a match character "m" (mixed) matches both an integer register +-- and an index operand (to be encoded with the ModRM/SIB scheme). +-- But "r" matches only a register and "x" only an index operand +-- (e.g. for FP memory access operations). +-- +-- The operand size match string starts right after the mode match +-- characters and ends before the ":". "dwb" or "qdwb" is assumed, if empty. +-- The effective data size of the operation is matched against this list. +-- +-- If only the regular "b", "w", "d", "q", "t" operand sizes are +-- present, then all operands must be the same size. Unspecified sizes +-- are ignored, but at least one operand must have a size or the pattern +-- won't match (use the "byte", "word", "dword", "qword", "tword" +-- operand size overrides. E.g.: mov dword [eax], 1). +-- +-- If the list has a "1" or "2" prefix, the operand size is taken +-- from the respective operand and any other operand sizes are ignored. +-- If the list contains only ".", all operand sizes are ignored. +-- If the list has a "/" prefix, the concatenated (mixed) operand sizes +-- are compared to the match. +-- +-- E.g. "rrdw" matches for either two dword registers or two word +-- registers. "Fx2dq" matches an st0 operand plus an index operand +-- pointing to a dword (float) or qword (double). +-- +-- Every character after the ":" is part of the pattern string: +-- Hex chars are accumulated to form the opcode (left to right). +-- "n" disables the standard opcode mods +-- (otherwise: -1 for "b", o16 prefix for "w", rex.w for "q") +-- "X" Force REX.W. +-- "r"/"R" adds the reg. number from the 1st/2nd operand to the opcode. +-- "m"/"M" generates ModRM/SIB from the 1st/2nd operand. +-- The spare 3 bits are either filled with the last hex digit or +-- the result from a previous "r"/"R". The opcode is restored. +-- "u" Use VEX encoding, vvvv unused. +-- "v"/"V" Use VEX encoding, vvvv from 1st/2nd operand (the operand is +-- removed from the list used by future characters). +-- "L" Force VEX.L +-- +-- All of the following characters force a flush of the opcode: +-- "o"/"O" stores a pure 32 bit disp (offset) from the 1st/2nd operand. +-- "s" stores a 4 bit immediate from the last register operand, +-- followed by 4 zero bits. +-- "S" stores a signed 8 bit immediate from the last operand. +-- "U" stores an unsigned 8 bit immediate from the last operand. +-- "W" stores an unsigned 16 bit immediate from the last operand. +-- "i" stores an operand sized immediate from the last operand. +-- "I" dito, but generates an action code to optionally modify +-- the opcode (+2) for a signed 8 bit immediate. +-- "J" generates one of the REL action codes from the last operand. +-- +------------------------------------------------------------------------------ + +-- Template strings for x86 instructions. Ordered by first opcode byte. +-- Unimplemented opcodes (deliberate omissions) are marked with *. +local map_op = { + -- 00-05: add... + -- 06: *push es + -- 07: *pop es + -- 08-0D: or... + -- 0E: *push cs + -- 0F: two byte opcode prefix + -- 10-15: adc... + -- 16: *push ss + -- 17: *pop ss + -- 18-1D: sbb... + -- 1E: *push ds + -- 1F: *pop ds + -- 20-25: and... + es_0 = "26", + -- 27: *daa + -- 28-2D: sub... + cs_0 = "2E", + -- 2F: *das + -- 30-35: xor... + ss_0 = "36", + -- 37: *aaa + -- 38-3D: cmp... + ds_0 = "3E", + -- 3F: *aas + inc_1 = x64 and "m:FF0m" or "rdw:40r|m:FF0m", + dec_1 = x64 and "m:FF1m" or "rdw:48r|m:FF1m", + push_1 = (x64 and "rq:n50r|rw:50r|mq:nFF6m|mw:FF6m" or + "rdw:50r|mdw:FF6m").."|S.:6AS|ib:n6Ai|i.:68i", + pop_1 = x64 and "rq:n58r|rw:58r|mq:n8F0m|mw:8F0m" or "rdw:58r|mdw:8F0m", + -- 60: *pusha, *pushad, *pushaw + -- 61: *popa, *popad, *popaw + -- 62: *bound rdw,x + -- 63: x86: *arpl mw,rw + movsxd_2 = x64 and "rm/qd:63rM", + fs_0 = "64", + gs_0 = "65", + o16_0 = "66", + a16_0 = not x64 and "67" or nil, + a32_0 = x64 and "67", + -- 68: push idw + -- 69: imul rdw,mdw,idw + -- 6A: push ib + -- 6B: imul rdw,mdw,S + -- 6C: *insb + -- 6D: *insd, *insw + -- 6E: *outsb + -- 6F: *outsd, *outsw + -- 70-7F: jcc lb + -- 80: add... mb,i + -- 81: add... mdw,i + -- 82: *undefined + -- 83: add... mdw,S + test_2 = "mr:85Rm|rm:85rM|Ri:A9ri|mi:F70mi", + -- 86: xchg rb,mb + -- 87: xchg rdw,mdw + -- 88: mov mb,r + -- 89: mov mdw,r + -- 8A: mov r,mb + -- 8B: mov r,mdw + -- 8C: *mov mdw,seg + lea_2 = "rx1dq:8DrM", + -- 8E: *mov seg,mdw + -- 8F: pop mdw + nop_0 = "90", + xchg_2 = "Rrqdw:90R|rRqdw:90r|rm:87rM|mr:87Rm", + cbw_0 = "6698", + cwde_0 = "98", + cdqe_0 = "4898", + cwd_0 = "6699", + cdq_0 = "99", + cqo_0 = "4899", + -- 9A: *call iw:idw + wait_0 = "9B", + fwait_0 = "9B", + pushf_0 = "9C", + pushfd_0 = not x64 and "9C", + pushfq_0 = x64 and "9C", + popf_0 = "9D", + popfd_0 = not x64 and "9D", + popfq_0 = x64 and "9D", + sahf_0 = "9E", + lahf_0 = "9F", + mov_2 = "OR:A3o|RO:A1O|mr:89Rm|rm:8BrM|rib:nB0ri|ridw:B8ri|mi:C70mi", + movsb_0 = "A4", + movsw_0 = "66A5", + movsd_0 = "A5", + cmpsb_0 = "A6", + cmpsw_0 = "66A7", + cmpsd_0 = "A7", + -- A8: test Rb,i + -- A9: test Rdw,i + stosb_0 = "AA", + stosw_0 = "66AB", + stosd_0 = "AB", + lodsb_0 = "AC", + lodsw_0 = "66AD", + lodsd_0 = "AD", + scasb_0 = "AE", + scasw_0 = "66AF", + scasd_0 = "AF", + -- B0-B7: mov rb,i + -- B8-BF: mov rdw,i + -- C0: rol... mb,i + -- C1: rol... mdw,i + ret_1 = "i.:nC2W", + ret_0 = "C3", + -- C4: *les rdw,mq + -- C5: *lds rdw,mq + -- C6: mov mb,i + -- C7: mov mdw,i + -- C8: *enter iw,ib + leave_0 = "C9", + -- CA: *retf iw + -- CB: *retf + int3_0 = "CC", + int_1 = "i.:nCDU", + into_0 = "CE", + -- CF: *iret + -- D0: rol... mb,1 + -- D1: rol... mdw,1 + -- D2: rol... mb,cl + -- D3: rol... mb,cl + -- D4: *aam ib + -- D5: *aad ib + -- D6: *salc + -- D7: *xlat + -- D8-DF: floating point ops + -- E0: *loopne + -- E1: *loope + -- E2: *loop + -- E3: *jcxz, *jecxz + -- E4: *in Rb,ib + -- E5: *in Rdw,ib + -- E6: *out ib,Rb + -- E7: *out ib,Rdw + call_1 = x64 and "mq:nFF2m|J.:E8nJ" or "md:FF2m|J.:E8J", + jmp_1 = x64 and "mq:nFF4m|J.:E9nJ" or "md:FF4m|J.:E9J", -- short: EB + -- EA: *jmp iw:idw + -- EB: jmp ib + -- EC: *in Rb,dx + -- ED: *in Rdw,dx + -- EE: *out dx,Rb + -- EF: *out dx,Rdw + lock_0 = "F0", + int1_0 = "F1", + repne_0 = "F2", + repnz_0 = "F2", + rep_0 = "F3", + repe_0 = "F3", + repz_0 = "F3", + -- F4: *hlt + cmc_0 = "F5", + -- F6: test... mb,i; div... mb + -- F7: test... mdw,i; div... mdw + clc_0 = "F8", + stc_0 = "F9", + -- FA: *cli + cld_0 = "FC", + std_0 = "FD", + -- FE: inc... mb + -- FF: inc... mdw + + -- misc ops + not_1 = "m:F72m", + neg_1 = "m:F73m", + mul_1 = "m:F74m", + imul_1 = "m:F75m", + div_1 = "m:F76m", + idiv_1 = "m:F77m", + + imul_2 = "rmqdw:0FAFrM|rIqdw:69rmI|rSqdw:6BrmS|riqdw:69rmi", + imul_3 = "rmIqdw:69rMI|rmSqdw:6BrMS|rmiqdw:69rMi", + + movzx_2 = "rm/db:0FB6rM|rm/qb:|rm/wb:0FB6rM|rm/dw:0FB7rM|rm/qw:", + movsx_2 = "rm/db:0FBErM|rm/qb:|rm/wb:0FBErM|rm/dw:0FBFrM|rm/qw:", + + bswap_1 = "rqd:0FC8r", + bsf_2 = "rmqdw:0FBCrM", + bsr_2 = "rmqdw:0FBDrM", + bt_2 = "mrqdw:0FA3Rm|miqdw:0FBA4mU", + btc_2 = "mrqdw:0FBBRm|miqdw:0FBA7mU", + btr_2 = "mrqdw:0FB3Rm|miqdw:0FBA6mU", + bts_2 = "mrqdw:0FABRm|miqdw:0FBA5mU", + + shld_3 = "mriqdw:0FA4RmU|mrC/qq:0FA5Rm|mrC/dd:|mrC/ww:", + shrd_3 = "mriqdw:0FACRmU|mrC/qq:0FADRm|mrC/dd:|mrC/ww:", + + rdtsc_0 = "0F31", -- P1+ + rdpmc_0 = "0F33", -- P6+ + cpuid_0 = "0FA2", -- P1+ + + -- floating point ops + fst_1 = "ff:DDD0r|xd:D92m|xq:nDD2m", + fstp_1 = "ff:DDD8r|xd:D93m|xq:nDD3m|xt:DB7m", + fld_1 = "ff:D9C0r|xd:D90m|xq:nDD0m|xt:DB5m", + + fpop_0 = "DDD8", -- Alias for fstp st0. + + fist_1 = "xw:nDF2m|xd:DB2m", + fistp_1 = "xw:nDF3m|xd:DB3m|xq:nDF7m", + fild_1 = "xw:nDF0m|xd:DB0m|xq:nDF5m", + + fxch_0 = "D9C9", + fxch_1 = "ff:D9C8r", + fxch_2 = "fFf:D9C8r|Fff:D9C8R", + + fucom_1 = "ff:DDE0r", + fucom_2 = "Fff:DDE0R", + fucomp_1 = "ff:DDE8r", + fucomp_2 = "Fff:DDE8R", + fucomi_1 = "ff:DBE8r", -- P6+ + fucomi_2 = "Fff:DBE8R", -- P6+ + fucomip_1 = "ff:DFE8r", -- P6+ + fucomip_2 = "Fff:DFE8R", -- P6+ + fcomi_1 = "ff:DBF0r", -- P6+ + fcomi_2 = "Fff:DBF0R", -- P6+ + fcomip_1 = "ff:DFF0r", -- P6+ + fcomip_2 = "Fff:DFF0R", -- P6+ + fucompp_0 = "DAE9", + fcompp_0 = "DED9", + + fldenv_1 = "x.:D94m", + fnstenv_1 = "x.:D96m", + fstenv_1 = "x.:9BD96m", + fldcw_1 = "xw:nD95m", + fstcw_1 = "xw:n9BD97m", + fnstcw_1 = "xw:nD97m", + fstsw_1 = "Rw:n9BDFE0|xw:n9BDD7m", + fnstsw_1 = "Rw:nDFE0|xw:nDD7m", + fclex_0 = "9BDBE2", + fnclex_0 = "DBE2", + + fnop_0 = "D9D0", + -- D9D1-D9DF: unassigned + + fchs_0 = "D9E0", + fabs_0 = "D9E1", + -- D9E2: unassigned + -- D9E3: unassigned + ftst_0 = "D9E4", + fxam_0 = "D9E5", + -- D9E6: unassigned + -- D9E7: unassigned + fld1_0 = "D9E8", + fldl2t_0 = "D9E9", + fldl2e_0 = "D9EA", + fldpi_0 = "D9EB", + fldlg2_0 = "D9EC", + fldln2_0 = "D9ED", + fldz_0 = "D9EE", + -- D9EF: unassigned + + f2xm1_0 = "D9F0", + fyl2x_0 = "D9F1", + fptan_0 = "D9F2", + fpatan_0 = "D9F3", + fxtract_0 = "D9F4", + fprem1_0 = "D9F5", + fdecstp_0 = "D9F6", + fincstp_0 = "D9F7", + fprem_0 = "D9F8", + fyl2xp1_0 = "D9F9", + fsqrt_0 = "D9FA", + fsincos_0 = "D9FB", + frndint_0 = "D9FC", + fscale_0 = "D9FD", + fsin_0 = "D9FE", + fcos_0 = "D9FF", + + -- SSE, SSE2 + andnpd_2 = "rmo:660F55rM", + andnps_2 = "rmo:0F55rM", + andpd_2 = "rmo:660F54rM", + andps_2 = "rmo:0F54rM", + clflush_1 = "x.:0FAE7m", + cmppd_3 = "rmio:660FC2rMU", + cmpps_3 = "rmio:0FC2rMU", + cmpsd_3 = "rrio:F20FC2rMU|rxi/oq:", + cmpss_3 = "rrio:F30FC2rMU|rxi/od:", + comisd_2 = "rro:660F2FrM|rx/oq:", + comiss_2 = "rro:0F2FrM|rx/od:", + cvtdq2pd_2 = "rro:F30FE6rM|rx/oq:", + cvtdq2ps_2 = "rmo:0F5BrM", + cvtpd2dq_2 = "rmo:F20FE6rM", + cvtpd2ps_2 = "rmo:660F5ArM", + cvtpi2pd_2 = "rx/oq:660F2ArM", + cvtpi2ps_2 = "rx/oq:0F2ArM", + cvtps2dq_2 = "rmo:660F5BrM", + cvtps2pd_2 = "rro:0F5ArM|rx/oq:", + cvtsd2si_2 = "rr/do:F20F2DrM|rr/qo:|rx/dq:|rxq:", + cvtsd2ss_2 = "rro:F20F5ArM|rx/oq:", + cvtsi2sd_2 = "rm/od:F20F2ArM|rm/oq:F20F2ArXM", + cvtsi2ss_2 = "rm/od:F30F2ArM|rm/oq:F30F2ArXM", + cvtss2sd_2 = "rro:F30F5ArM|rx/od:", + cvtss2si_2 = "rr/do:F30F2DrM|rr/qo:|rxd:|rx/qd:", + cvttpd2dq_2 = "rmo:660FE6rM", + cvttps2dq_2 = "rmo:F30F5BrM", + cvttsd2si_2 = "rr/do:F20F2CrM|rr/qo:|rx/dq:|rxq:", + cvttss2si_2 = "rr/do:F30F2CrM|rr/qo:|rxd:|rx/qd:", + fxsave_1 = "x.:0FAE0m", + fxrstor_1 = "x.:0FAE1m", + ldmxcsr_1 = "xd:0FAE2m", + lfence_0 = "0FAEE8", + maskmovdqu_2 = "rro:660FF7rM", + mfence_0 = "0FAEF0", + movapd_2 = "rmo:660F28rM|mro:660F29Rm", + movaps_2 = "rmo:0F28rM|mro:0F29Rm", + movd_2 = "rm/od:660F6ErM|rm/oq:660F6ErXM|mr/do:660F7ERm|mr/qo:", + movdqa_2 = "rmo:660F6FrM|mro:660F7FRm", + movdqu_2 = "rmo:F30F6FrM|mro:F30F7FRm", + movhlps_2 = "rro:0F12rM", + movhpd_2 = "rx/oq:660F16rM|xr/qo:n660F17Rm", + movhps_2 = "rx/oq:0F16rM|xr/qo:n0F17Rm", + movlhps_2 = "rro:0F16rM", + movlpd_2 = "rx/oq:660F12rM|xr/qo:n660F13Rm", + movlps_2 = "rx/oq:0F12rM|xr/qo:n0F13Rm", + movmskpd_2 = "rr/do:660F50rM", + movmskps_2 = "rr/do:0F50rM", + movntdq_2 = "xro:660FE7Rm", + movnti_2 = "xrqd:0FC3Rm", + movntpd_2 = "xro:660F2BRm", + movntps_2 = "xro:0F2BRm", + movq_2 = "rro:F30F7ErM|rx/oq:|xr/qo:n660FD6Rm", + movsd_2 = "rro:F20F10rM|rx/oq:|xr/qo:nF20F11Rm", + movss_2 = "rro:F30F10rM|rx/od:|xr/do:F30F11Rm", + movupd_2 = "rmo:660F10rM|mro:660F11Rm", + movups_2 = "rmo:0F10rM|mro:0F11Rm", + orpd_2 = "rmo:660F56rM", + orps_2 = "rmo:0F56rM", + pause_0 = "F390", + pextrw_3 = "rri/do:660FC5rMU|xri/wo:660F3A15nRmU", -- Mem op: SSE4.1 only. + pinsrw_3 = "rri/od:660FC4rMU|rxi/ow:", + pmovmskb_2 = "rr/do:660FD7rM", + prefetchnta_1 = "xb:n0F180m", + prefetcht0_1 = "xb:n0F181m", + prefetcht1_1 = "xb:n0F182m", + prefetcht2_1 = "xb:n0F183m", + pshufd_3 = "rmio:660F70rMU", + pshufhw_3 = "rmio:F30F70rMU", + pshuflw_3 = "rmio:F20F70rMU", + pslld_2 = "rmo:660FF2rM|rio:660F726mU", + pslldq_2 = "rio:660F737mU", + psllq_2 = "rmo:660FF3rM|rio:660F736mU", + psllw_2 = "rmo:660FF1rM|rio:660F716mU", + psrad_2 = "rmo:660FE2rM|rio:660F724mU", + psraw_2 = "rmo:660FE1rM|rio:660F714mU", + psrld_2 = "rmo:660FD2rM|rio:660F722mU", + psrldq_2 = "rio:660F733mU", + psrlq_2 = "rmo:660FD3rM|rio:660F732mU", + psrlw_2 = "rmo:660FD1rM|rio:660F712mU", + rcpps_2 = "rmo:0F53rM", + rcpss_2 = "rro:F30F53rM|rx/od:", + rsqrtps_2 = "rmo:0F52rM", + rsqrtss_2 = "rmo:F30F52rM", + sfence_0 = "0FAEF8", + shufpd_3 = "rmio:660FC6rMU", + shufps_3 = "rmio:0FC6rMU", + stmxcsr_1 = "xd:0FAE3m", + ucomisd_2 = "rro:660F2ErM|rx/oq:", + ucomiss_2 = "rro:0F2ErM|rx/od:", + unpckhpd_2 = "rmo:660F15rM", + unpckhps_2 = "rmo:0F15rM", + unpcklpd_2 = "rmo:660F14rM", + unpcklps_2 = "rmo:0F14rM", + xorpd_2 = "rmo:660F57rM", + xorps_2 = "rmo:0F57rM", + + -- SSE3 ops + fisttp_1 = "xw:nDF1m|xd:DB1m|xq:nDD1m", + addsubpd_2 = "rmo:660FD0rM", + addsubps_2 = "rmo:F20FD0rM", + haddpd_2 = "rmo:660F7CrM", + haddps_2 = "rmo:F20F7CrM", + hsubpd_2 = "rmo:660F7DrM", + hsubps_2 = "rmo:F20F7DrM", + lddqu_2 = "rxo:F20FF0rM", + movddup_2 = "rmo:F20F12rM", + movshdup_2 = "rmo:F30F16rM", + movsldup_2 = "rmo:F30F12rM", + + -- SSSE3 ops + pabsb_2 = "rmo:660F381CrM", + pabsd_2 = "rmo:660F381ErM", + pabsw_2 = "rmo:660F381DrM", + palignr_3 = "rmio:660F3A0FrMU", + phaddd_2 = "rmo:660F3802rM", + phaddsw_2 = "rmo:660F3803rM", + phaddw_2 = "rmo:660F3801rM", + phsubd_2 = "rmo:660F3806rM", + phsubsw_2 = "rmo:660F3807rM", + phsubw_2 = "rmo:660F3805rM", + pmaddubsw_2 = "rmo:660F3804rM", + pmulhrsw_2 = "rmo:660F380BrM", + pshufb_2 = "rmo:660F3800rM", + psignb_2 = "rmo:660F3808rM", + psignd_2 = "rmo:660F380ArM", + psignw_2 = "rmo:660F3809rM", + + -- SSE4.1 ops + blendpd_3 = "rmio:660F3A0DrMU", + blendps_3 = "rmio:660F3A0CrMU", + blendvpd_3 = "rmRo:660F3815rM", + blendvps_3 = "rmRo:660F3814rM", + dppd_3 = "rmio:660F3A41rMU", + dpps_3 = "rmio:660F3A40rMU", + extractps_3 = "mri/do:660F3A17RmU|rri/qo:660F3A17RXmU", + insertps_3 = "rrio:660F3A41rMU|rxi/od:", + movntdqa_2 = "rxo:660F382ArM", + mpsadbw_3 = "rmio:660F3A42rMU", + packusdw_2 = "rmo:660F382BrM", + pblendvb_3 = "rmRo:660F3810rM", + pblendw_3 = "rmio:660F3A0ErMU", + pcmpeqq_2 = "rmo:660F3829rM", + pextrb_3 = "rri/do:660F3A14nRmU|rri/qo:|xri/bo:", + pextrd_3 = "mri/do:660F3A16RmU", + pextrq_3 = "mri/qo:660F3A16RmU", + -- pextrw is SSE2, mem operand is SSE4.1 only + phminposuw_2 = "rmo:660F3841rM", + pinsrb_3 = "rri/od:660F3A20nrMU|rxi/ob:", + pinsrd_3 = "rmi/od:660F3A22rMU", + pinsrq_3 = "rmi/oq:660F3A22rXMU", + pmaxsb_2 = "rmo:660F383CrM", + pmaxsd_2 = "rmo:660F383DrM", + pmaxud_2 = "rmo:660F383FrM", + pmaxuw_2 = "rmo:660F383ErM", + pminsb_2 = "rmo:660F3838rM", + pminsd_2 = "rmo:660F3839rM", + pminud_2 = "rmo:660F383BrM", + pminuw_2 = "rmo:660F383ArM", + pmovsxbd_2 = "rro:660F3821rM|rx/od:", + pmovsxbq_2 = "rro:660F3822rM|rx/ow:", + pmovsxbw_2 = "rro:660F3820rM|rx/oq:", + pmovsxdq_2 = "rro:660F3825rM|rx/oq:", + pmovsxwd_2 = "rro:660F3823rM|rx/oq:", + pmovsxwq_2 = "rro:660F3824rM|rx/od:", + pmovzxbd_2 = "rro:660F3831rM|rx/od:", + pmovzxbq_2 = "rro:660F3832rM|rx/ow:", + pmovzxbw_2 = "rro:660F3830rM|rx/oq:", + pmovzxdq_2 = "rro:660F3835rM|rx/oq:", + pmovzxwd_2 = "rro:660F3833rM|rx/oq:", + pmovzxwq_2 = "rro:660F3834rM|rx/od:", + pmuldq_2 = "rmo:660F3828rM", + pmulld_2 = "rmo:660F3840rM", + ptest_2 = "rmo:660F3817rM", + roundpd_3 = "rmio:660F3A09rMU", + roundps_3 = "rmio:660F3A08rMU", + roundsd_3 = "rrio:660F3A0BrMU|rxi/oq:", + roundss_3 = "rrio:660F3A0ArMU|rxi/od:", + + -- SSE4.2 ops + crc32_2 = "rmqd:F20F38F1rM|rm/dw:66F20F38F1rM|rm/db:F20F38F0rM|rm/qb:", + pcmpestri_3 = "rmio:660F3A61rMU", + pcmpestrm_3 = "rmio:660F3A60rMU", + pcmpgtq_2 = "rmo:660F3837rM", + pcmpistri_3 = "rmio:660F3A63rMU", + pcmpistrm_3 = "rmio:660F3A62rMU", + popcnt_2 = "rmqdw:F30FB8rM", + + -- SSE4a + extrq_2 = "rro:660F79rM", + extrq_3 = "riio:660F780mUU", + insertq_2 = "rro:F20F79rM", + insertq_4 = "rriio:F20F78rMUU", + lzcnt_2 = "rmqdw:F30FBDrM", + movntsd_2 = "xr/qo:nF20F2BRm", + movntss_2 = "xr/do:F30F2BRm", + -- popcnt is also in SSE4.2 + + -- AES-NI + aesdec_2 = "rmo:660F38DErM", + aesdeclast_2 = "rmo:660F38DFrM", + aesenc_2 = "rmo:660F38DCrM", + aesenclast_2 = "rmo:660F38DDrM", + aesimc_2 = "rmo:660F38DBrM", + aeskeygenassist_3 = "rmio:660F3ADFrMU", + pclmulqdq_3 = "rmio:660F3A44rMU", + + -- AVX FP ops + vaddsubpd_3 = "rrmoy:660FVD0rM", + vaddsubps_3 = "rrmoy:F20FVD0rM", + vandpd_3 = "rrmoy:660FV54rM", + vandps_3 = "rrmoy:0FV54rM", + vandnpd_3 = "rrmoy:660FV55rM", + vandnps_3 = "rrmoy:0FV55rM", + vblendpd_4 = "rrmioy:660F3AV0DrMU", + vblendps_4 = "rrmioy:660F3AV0CrMU", + vblendvpd_4 = "rrmroy:660F3AV4BrMs", + vblendvps_4 = "rrmroy:660F3AV4ArMs", + vbroadcastf128_2 = "rx/yo:660F38u1ArM", + vcmppd_4 = "rrmioy:660FVC2rMU", + vcmpps_4 = "rrmioy:0FVC2rMU", + vcmpsd_4 = "rrrio:F20FVC2rMU|rrxi/ooq:", + vcmpss_4 = "rrrio:F30FVC2rMU|rrxi/ood:", + vcomisd_2 = "rro:660Fu2FrM|rx/oq:", + vcomiss_2 = "rro:0Fu2FrM|rx/od:", + vcvtdq2pd_2 = "rro:F30FuE6rM|rx/oq:|rm/yo:", + vcvtdq2ps_2 = "rmoy:0Fu5BrM", + vcvtpd2dq_2 = "rmoy:F20FuE6rM", + vcvtpd2ps_2 = "rmoy:660Fu5ArM", + vcvtps2dq_2 = "rmoy:660Fu5BrM", + vcvtps2pd_2 = "rro:0Fu5ArM|rx/oq:|rm/yo:", + vcvtsd2si_2 = "rr/do:F20Fu2DrM|rx/dq:|rr/qo:|rxq:", + vcvtsd2ss_3 = "rrro:F20FV5ArM|rrx/ooq:", + vcvtsi2sd_3 = "rrm/ood:F20FV2ArM|rrm/ooq:F20FVX2ArM", + vcvtsi2ss_3 = "rrm/ood:F30FV2ArM|rrm/ooq:F30FVX2ArM", + vcvtss2sd_3 = "rrro:F30FV5ArM|rrx/ood:", + vcvtss2si_2 = "rr/do:F30Fu2DrM|rxd:|rr/qo:|rx/qd:", + vcvttpd2dq_2 = "rmo:660FuE6rM|rm/oy:660FuLE6rM", + vcvttps2dq_2 = "rmoy:F30Fu5BrM", + vcvttsd2si_2 = "rr/do:F20Fu2CrM|rx/dq:|rr/qo:|rxq:", + vcvttss2si_2 = "rr/do:F30Fu2CrM|rxd:|rr/qo:|rx/qd:", + vdppd_4 = "rrmio:660F3AV41rMU", + vdpps_4 = "rrmioy:660F3AV40rMU", + vextractf128_3 = "mri/oy:660F3AuL19RmU", + vextractps_3 = "mri/do:660F3Au17RmU", + vhaddpd_3 = "rrmoy:660FV7CrM", + vhaddps_3 = "rrmoy:F20FV7CrM", + vhsubpd_3 = "rrmoy:660FV7DrM", + vhsubps_3 = "rrmoy:F20FV7DrM", + vinsertf128_4 = "rrmi/yyo:660F3AV18rMU", + vinsertps_4 = "rrrio:660F3AV21rMU|rrxi/ood:", + vldmxcsr_1 = "xd:0FuAE2m", + vmaskmovps_3 = "rrxoy:660F38V2CrM|xrroy:660F38V2ERm", + vmaskmovpd_3 = "rrxoy:660F38V2DrM|xrroy:660F38V2FRm", + vmovapd_2 = "rmoy:660Fu28rM|mroy:660Fu29Rm", + vmovaps_2 = "rmoy:0Fu28rM|mroy:0Fu29Rm", + vmovd_2 = "rm/od:660Fu6ErM|rm/oq:660FuX6ErM|mr/do:660Fu7ERm|mr/qo:", + vmovq_2 = "rro:F30Fu7ErM|rx/oq:|xr/qo:660FuD6Rm", + vmovddup_2 = "rmy:F20Fu12rM|rro:|rx/oq:", + vmovhlps_3 = "rrro:0FV12rM", + vmovhpd_2 = "xr/qo:660Fu17Rm", + vmovhpd_3 = "rrx/ooq:660FV16rM", + vmovhps_2 = "xr/qo:0Fu17Rm", + vmovhps_3 = "rrx/ooq:0FV16rM", + vmovlhps_3 = "rrro:0FV16rM", + vmovlpd_2 = "xr/qo:660Fu13Rm", + vmovlpd_3 = "rrx/ooq:660FV12rM", + vmovlps_2 = "xr/qo:0Fu13Rm", + vmovlps_3 = "rrx/ooq:0FV12rM", + vmovmskpd_2 = "rr/do:660Fu50rM|rr/dy:660FuL50rM", + vmovmskps_2 = "rr/do:0Fu50rM|rr/dy:0FuL50rM", + vmovntpd_2 = "xroy:660Fu2BRm", + vmovntps_2 = "xroy:0Fu2BRm", + vmovsd_2 = "rx/oq:F20Fu10rM|xr/qo:F20Fu11Rm", + vmovsd_3 = "rrro:F20FV10rM", + vmovshdup_2 = "rmoy:F30Fu16rM", + vmovsldup_2 = "rmoy:F30Fu12rM", + vmovss_2 = "rx/od:F30Fu10rM|xr/do:F30Fu11Rm", + vmovss_3 = "rrro:F30FV10rM", + vmovupd_2 = "rmoy:660Fu10rM|mroy:660Fu11Rm", + vmovups_2 = "rmoy:0Fu10rM|mroy:0Fu11Rm", + vorpd_3 = "rrmoy:660FV56rM", + vorps_3 = "rrmoy:0FV56rM", + vpermilpd_3 = "rrmoy:660F38V0DrM|rmioy:660F3Au05rMU", + vpermilps_3 = "rrmoy:660F38V0CrM|rmioy:660F3Au04rMU", + vperm2f128_4 = "rrmiy:660F3AV06rMU", + vptestpd_2 = "rmoy:660F38u0FrM", + vptestps_2 = "rmoy:660F38u0ErM", + vrcpps_2 = "rmoy:0Fu53rM", + vrcpss_3 = "rrro:F30FV53rM|rrx/ood:", + vrsqrtps_2 = "rmoy:0Fu52rM", + vrsqrtss_3 = "rrro:F30FV52rM|rrx/ood:", + vroundpd_3 = "rmioy:660F3AV09rMU", + vroundps_3 = "rmioy:660F3AV08rMU", + vroundsd_4 = "rrrio:660F3AV0BrMU|rrxi/ooq:", + vroundss_4 = "rrrio:660F3AV0ArMU|rrxi/ood:", + vshufpd_4 = "rrmioy:660FVC6rMU", + vshufps_4 = "rrmioy:0FVC6rMU", + vsqrtps_2 = "rmoy:0Fu51rM", + vsqrtss_2 = "rro:F30Fu51rM|rx/od:", + vsqrtpd_2 = "rmoy:660Fu51rM", + vsqrtsd_2 = "rro:F20Fu51rM|rx/oq:", + vstmxcsr_1 = "xd:0FuAE3m", + vucomisd_2 = "rro:660Fu2ErM|rx/oq:", + vucomiss_2 = "rro:0Fu2ErM|rx/od:", + vunpckhpd_3 = "rrmoy:660FV15rM", + vunpckhps_3 = "rrmoy:0FV15rM", + vunpcklpd_3 = "rrmoy:660FV14rM", + vunpcklps_3 = "rrmoy:0FV14rM", + vxorpd_3 = "rrmoy:660FV57rM", + vxorps_3 = "rrmoy:0FV57rM", + vzeroall_0 = "0FuL77", + vzeroupper_0 = "0Fu77", + + -- AVX2 FP ops + vbroadcastss_2 = "rx/od:660F38u18rM|rx/yd:|rro:|rr/yo:", + vbroadcastsd_2 = "rx/yq:660F38u19rM|rr/yo:", + -- *vgather* (!vsib) + vpermpd_3 = "rmiy:660F3AuX01rMU", + vpermps_3 = "rrmy:660F38V16rM", + + -- AVX, AVX2 integer ops + -- In general, xmm requires AVX, ymm requires AVX2. + vaesdec_3 = "rrmo:660F38VDErM", + vaesdeclast_3 = "rrmo:660F38VDFrM", + vaesenc_3 = "rrmo:660F38VDCrM", + vaesenclast_3 = "rrmo:660F38VDDrM", + vaesimc_2 = "rmo:660F38uDBrM", + vaeskeygenassist_3 = "rmio:660F3AuDFrMU", + vlddqu_2 = "rxoy:F20FuF0rM", + vmaskmovdqu_2 = "rro:660FuF7rM", + vmovdqa_2 = "rmoy:660Fu6FrM|mroy:660Fu7FRm", + vmovdqu_2 = "rmoy:F30Fu6FrM|mroy:F30Fu7FRm", + vmovntdq_2 = "xroy:660FuE7Rm", + vmovntdqa_2 = "rxoy:660F38u2ArM", + vmpsadbw_4 = "rrmioy:660F3AV42rMU", + vpabsb_2 = "rmoy:660F38u1CrM", + vpabsd_2 = "rmoy:660F38u1ErM", + vpabsw_2 = "rmoy:660F38u1DrM", + vpackusdw_3 = "rrmoy:660F38V2BrM", + vpalignr_4 = "rrmioy:660F3AV0FrMU", + vpblendvb_4 = "rrmroy:660F3AV4CrMs", + vpblendw_4 = "rrmioy:660F3AV0ErMU", + vpclmulqdq_4 = "rrmio:660F3AV44rMU", + vpcmpeqq_3 = "rrmoy:660F38V29rM", + vpcmpestri_3 = "rmio:660F3Au61rMU", + vpcmpestrm_3 = "rmio:660F3Au60rMU", + vpcmpgtq_3 = "rrmoy:660F38V37rM", + vpcmpistri_3 = "rmio:660F3Au63rMU", + vpcmpistrm_3 = "rmio:660F3Au62rMU", + vpextrb_3 = "rri/do:660F3Au14nRmU|rri/qo:|xri/bo:", + vpextrw_3 = "rri/do:660FuC5rMU|xri/wo:660F3Au15nRmU", + vpextrd_3 = "mri/do:660F3Au16RmU", + vpextrq_3 = "mri/qo:660F3Au16RmU", + vphaddw_3 = "rrmoy:660F38V01rM", + vphaddd_3 = "rrmoy:660F38V02rM", + vphaddsw_3 = "rrmoy:660F38V03rM", + vphminposuw_2 = "rmo:660F38u41rM", + vphsubw_3 = "rrmoy:660F38V05rM", + vphsubd_3 = "rrmoy:660F38V06rM", + vphsubsw_3 = "rrmoy:660F38V07rM", + vpinsrb_4 = "rrri/ood:660F3AV20rMU|rrxi/oob:", + vpinsrw_4 = "rrri/ood:660FVC4rMU|rrxi/oow:", + vpinsrd_4 = "rrmi/ood:660F3AV22rMU", + vpinsrq_4 = "rrmi/ooq:660F3AVX22rMU", + vpmaddubsw_3 = "rrmoy:660F38V04rM", + vpmaxsb_3 = "rrmoy:660F38V3CrM", + vpmaxsd_3 = "rrmoy:660F38V3DrM", + vpmaxuw_3 = "rrmoy:660F38V3ErM", + vpmaxud_3 = "rrmoy:660F38V3FrM", + vpminsb_3 = "rrmoy:660F38V38rM", + vpminsd_3 = "rrmoy:660F38V39rM", + vpminuw_3 = "rrmoy:660F38V3ArM", + vpminud_3 = "rrmoy:660F38V3BrM", + vpmovmskb_2 = "rr/do:660FuD7rM|rr/dy:660FuLD7rM", + vpmovsxbw_2 = "rroy:660F38u20rM|rx/oq:|rx/yo:", + vpmovsxbd_2 = "rroy:660F38u21rM|rx/od:|rx/yq:", + vpmovsxbq_2 = "rroy:660F38u22rM|rx/ow:|rx/yd:", + vpmovsxwd_2 = "rroy:660F38u23rM|rx/oq:|rx/yo:", + vpmovsxwq_2 = "rroy:660F38u24rM|rx/od:|rx/yq:", + vpmovsxdq_2 = "rroy:660F38u25rM|rx/oq:|rx/yo:", + vpmovzxbw_2 = "rroy:660F38u30rM|rx/oq:|rx/yo:", + vpmovzxbd_2 = "rroy:660F38u31rM|rx/od:|rx/yq:", + vpmovzxbq_2 = "rroy:660F38u32rM|rx/ow:|rx/yd:", + vpmovzxwd_2 = "rroy:660F38u33rM|rx/oq:|rx/yo:", + vpmovzxwq_2 = "rroy:660F38u34rM|rx/od:|rx/yq:", + vpmovzxdq_2 = "rroy:660F38u35rM|rx/oq:|rx/yo:", + vpmuldq_3 = "rrmoy:660F38V28rM", + vpmulhrsw_3 = "rrmoy:660F38V0BrM", + vpmulld_3 = "rrmoy:660F38V40rM", + vpshufb_3 = "rrmoy:660F38V00rM", + vpshufd_3 = "rmioy:660Fu70rMU", + vpshufhw_3 = "rmioy:F30Fu70rMU", + vpshuflw_3 = "rmioy:F20Fu70rMU", + vpsignb_3 = "rrmoy:660F38V08rM", + vpsignw_3 = "rrmoy:660F38V09rM", + vpsignd_3 = "rrmoy:660F38V0ArM", + vpslldq_3 = "rrioy:660Fv737mU", + vpsllw_3 = "rrmoy:660FVF1rM|rrioy:660Fv716mU", + vpslld_3 = "rrmoy:660FVF2rM|rrioy:660Fv726mU", + vpsllq_3 = "rrmoy:660FVF3rM|rrioy:660Fv736mU", + vpsraw_3 = "rrmoy:660FVE1rM|rrioy:660Fv714mU", + vpsrad_3 = "rrmoy:660FVE2rM|rrioy:660Fv724mU", + vpsrldq_3 = "rrioy:660Fv733mU", + vpsrlw_3 = "rrmoy:660FVD1rM|rrioy:660Fv712mU", + vpsrld_3 = "rrmoy:660FVD2rM|rrioy:660Fv722mU", + vpsrlq_3 = "rrmoy:660FVD3rM|rrioy:660Fv732mU", + vptest_2 = "rmoy:660F38u17rM", + + -- AVX2 integer ops + vbroadcasti128_2 = "rx/yo:660F38u5ArM", + vinserti128_4 = "rrmi/yyo:660F3AV38rMU", + vextracti128_3 = "mri/oy:660F3AuL39RmU", + vpblendd_4 = "rrmioy:660F3AV02rMU", + vpbroadcastb_2 = "rro:660F38u78rM|rx/ob:|rr/yo:|rx/yb:", + vpbroadcastw_2 = "rro:660F38u79rM|rx/ow:|rr/yo:|rx/yw:", + vpbroadcastd_2 = "rro:660F38u58rM|rx/od:|rr/yo:|rx/yd:", + vpbroadcastq_2 = "rro:660F38u59rM|rx/oq:|rr/yo:|rx/yq:", + vpermd_3 = "rrmy:660F38V36rM", + vpermq_3 = "rmiy:660F3AuX00rMU", + -- *vpgather* (!vsib) + vperm2i128_4 = "rrmiy:660F3AV46rMU", + vpmaskmovd_3 = "rrxoy:660F38V8CrM|xrroy:660F38V8ERm", + vpmaskmovq_3 = "rrxoy:660F38VX8CrM|xrroy:660F38VX8ERm", + vpsllvd_3 = "rrmoy:660F38V47rM", + vpsllvq_3 = "rrmoy:660F38VX47rM", + vpsravd_3 = "rrmoy:660F38V46rM", + vpsrlvd_3 = "rrmoy:660F38V45rM", + vpsrlvq_3 = "rrmoy:660F38VX45rM", + + -- Intel ADX + adcx_2 = "rmqd:660F38F6rM", + adox_2 = "rmqd:F30F38F6rM", +} + +------------------------------------------------------------------------------ + +-- Arithmetic ops. +for name,n in pairs{ add = 0, ["or"] = 1, adc = 2, sbb = 3, + ["and"] = 4, sub = 5, xor = 6, cmp = 7 } do + local n8 = shl(n, 3) + map_op[name.."_2"] = format( + "mr:%02XRm|rm:%02XrM|mI1qdw:81%XmI|mS1qdw:83%XmS|Ri1qdwb:%02Xri|mi1qdwb:81%Xmi", + 1+n8, 3+n8, n, n, 5+n8, n) +end + +-- Shift ops. +for name,n in pairs{ rol = 0, ror = 1, rcl = 2, rcr = 3, + shl = 4, shr = 5, sar = 7, sal = 4 } do + map_op[name.."_2"] = format("m1:D1%Xm|mC1qdwb:D3%Xm|mi:C1%XmU", n, n, n) +end + +-- Conditional ops. +for cc,n in pairs(map_cc) do + map_op["j"..cc.."_1"] = format("J.:n0F8%XJ", n) -- short: 7%X + map_op["set"..cc.."_1"] = format("mb:n0F9%X2m", n) + map_op["cmov"..cc.."_2"] = format("rmqdw:0F4%XrM", n) -- P6+ +end + +-- FP arithmetic ops. +for name,n in pairs{ add = 0, mul = 1, com = 2, comp = 3, + sub = 4, subr = 5, div = 6, divr = 7 } do + local nc = 0xc0 + shl(n, 3) + local nr = nc + (n < 4 and 0 or (n % 2 == 0 and 8 or -8)) + local fn = "f"..name + map_op[fn.."_1"] = format("ff:D8%02Xr|xd:D8%Xm|xq:nDC%Xm", nc, n, n) + if n == 2 or n == 3 then + map_op[fn.."_2"] = format("Fff:D8%02XR|Fx2d:D8%XM|Fx2q:nDC%XM", nc, n, n) + else + map_op[fn.."_2"] = format("Fff:D8%02XR|fFf:DC%02Xr|Fx2d:D8%XM|Fx2q:nDC%XM", nc, nr, n, n) + map_op[fn.."p_1"] = format("ff:DE%02Xr", nr) + map_op[fn.."p_2"] = format("fFf:DE%02Xr", nr) + end + map_op["fi"..name.."_1"] = format("xd:DA%Xm|xw:nDE%Xm", n, n) +end + +-- FP conditional moves. +for cc,n in pairs{ b=0, e=1, be=2, u=3, nb=4, ne=5, nbe=6, nu=7 } do + local nc = 0xdac0 + shl(band(n, 3), 3) + shl(band(n, 4), 6) + map_op["fcmov"..cc.."_1"] = format("ff:%04Xr", nc) -- P6+ + map_op["fcmov"..cc.."_2"] = format("Fff:%04XR", nc) -- P6+ +end + +-- SSE / AVX FP arithmetic ops. +for name,n in pairs{ sqrt = 1, add = 8, mul = 9, + sub = 12, min = 13, div = 14, max = 15 } do + map_op[name.."ps_2"] = format("rmo:0F5%XrM", n) + map_op[name.."ss_2"] = format("rro:F30F5%XrM|rx/od:", n) + map_op[name.."pd_2"] = format("rmo:660F5%XrM", n) + map_op[name.."sd_2"] = format("rro:F20F5%XrM|rx/oq:", n) + if n ~= 1 then + map_op["v"..name.."ps_3"] = format("rrmoy:0FV5%XrM", n) + map_op["v"..name.."ss_3"] = format("rrro:F30FV5%XrM|rrx/ood:", n) + map_op["v"..name.."pd_3"] = format("rrmoy:660FV5%XrM", n) + map_op["v"..name.."sd_3"] = format("rrro:F20FV5%XrM|rrx/ooq:", n) + end +end + +-- SSE2 / AVX / AVX2 integer arithmetic ops (66 0F leaf). +for name,n in pairs{ + paddb = 0xFC, paddw = 0xFD, paddd = 0xFE, paddq = 0xD4, + paddsb = 0xEC, paddsw = 0xED, packssdw = 0x6B, + packsswb = 0x63, packuswb = 0x67, paddusb = 0xDC, + paddusw = 0xDD, pand = 0xDB, pandn = 0xDF, pavgb = 0xE0, + pavgw = 0xE3, pcmpeqb = 0x74, pcmpeqd = 0x76, + pcmpeqw = 0x75, pcmpgtb = 0x64, pcmpgtd = 0x66, + pcmpgtw = 0x65, pmaddwd = 0xF5, pmaxsw = 0xEE, + pmaxub = 0xDE, pminsw = 0xEA, pminub = 0xDA, + pmulhuw = 0xE4, pmulhw = 0xE5, pmullw = 0xD5, + pmuludq = 0xF4, por = 0xEB, psadbw = 0xF6, psubb = 0xF8, + psubw = 0xF9, psubd = 0xFA, psubq = 0xFB, psubsb = 0xE8, + psubsw = 0xE9, psubusb = 0xD8, psubusw = 0xD9, + punpckhbw = 0x68, punpckhwd = 0x69, punpckhdq = 0x6A, + punpckhqdq = 0x6D, punpcklbw = 0x60, punpcklwd = 0x61, + punpckldq = 0x62, punpcklqdq = 0x6C, pxor = 0xEF +} do + map_op[name.."_2"] = format("rmo:660F%02XrM", n) + map_op["v"..name.."_3"] = format("rrmoy:660FV%02XrM", n) +end + +------------------------------------------------------------------------------ + +local map_vexarg = { u = false, v = 1, V = 2 } + +-- Process pattern string. +local function dopattern(pat, args, sz, op, needrex) + local digit, addin, vex + local opcode = 0 + local szov = sz + local narg = 1 + local rex = 0 + + -- Limit number of section buffer positions used by a single dasm_put(). + -- A single opcode needs a maximum of 6 positions. + if secpos+6 > maxsecpos then wflush() end + + -- Process each character. + for c in gmatch(pat.."|", ".") do + if match(c, "%x") then -- Hex digit. + digit = byte(c) - 48 + if digit > 48 then digit = digit - 39 + elseif digit > 16 then digit = digit - 7 end + opcode = opcode*16 + digit + addin = nil + elseif c == "n" then -- Disable operand size mods for opcode. + szov = nil + elseif c == "X" then -- Force REX.W. + rex = 8 + elseif c == "L" then -- Force VEX.L. + vex.l = true + elseif c == "r" then -- Merge 1st operand regno. into opcode. + addin = args[1]; opcode = opcode + (addin.reg % 8) + if narg < 2 then narg = 2 end + elseif c == "R" then -- Merge 2nd operand regno. into opcode. + addin = args[2]; opcode = opcode + (addin.reg % 8) + narg = 3 + elseif c == "m" or c == "M" then -- Encode ModRM/SIB. + local s + if addin then + s = addin.reg + opcode = opcode - band(s, 7) -- Undo regno opcode merge. + else + s = band(opcode, 15) -- Undo last digit. + opcode = shr(opcode, 4) + end + local nn = c == "m" and 1 or 2 + local t = args[nn] + if narg <= nn then narg = nn + 1 end + if szov == "q" and rex == 0 then rex = rex + 8 end + if t.reg and t.reg > 7 then rex = rex + 1 end + if t.xreg and t.xreg > 7 then rex = rex + 2 end + if s > 7 then rex = rex + 4 end + if needrex then rex = rex + 16 end + local psz, sk = wputop(szov, opcode, rex, vex, s < 0, t.vreg or t.vxreg) + opcode = nil + local imark = sub(pat, -1) -- Force a mark (ugly). + -- Put ModRM/SIB with regno/last digit as spare. + wputmrmsib(t, imark, s, addin and addin.vreg, psz, sk) + addin = nil + elseif map_vexarg[c] ~= nil then -- Encode using VEX prefix + local b = band(opcode, 255); opcode = shr(opcode, 8) + local m = 1 + if b == 0x38 then m = 2 + elseif b == 0x3a then m = 3 end + if m ~= 1 then b = band(opcode, 255); opcode = shr(opcode, 8) end + if b ~= 0x0f then + werror("expected `0F', `0F38', or `0F3A' to precede `"..c.. + "' in pattern `"..pat.."' for `"..op.."'") + end + local v = map_vexarg[c] + if v then v = remove(args, v) end + b = band(opcode, 255) + local p = 0 + if b == 0x66 then p = 1 + elseif b == 0xf3 then p = 2 + elseif b == 0xf2 then p = 3 end + if p ~= 0 then opcode = shr(opcode, 8) end + if opcode ~= 0 then wputop(nil, opcode, 0); opcode = 0 end + vex = { m = m, p = p, v = v } + else + if opcode then -- Flush opcode. + if szov == "q" and rex == 0 then rex = rex + 8 end + if needrex then rex = rex + 16 end + if addin and addin.reg == -1 then + local psz, sk = wputop(szov, opcode - 7, rex, vex, true) + wvreg("opcode", addin.vreg, psz, sk) + else + if addin and addin.reg > 7 then rex = rex + 1 end + wputop(szov, opcode, rex, vex) + end + opcode = nil + end + if c == "|" then break end + if c == "o" then -- Offset (pure 32 bit displacement). + wputdarg(args[1].disp); if narg < 2 then narg = 2 end + elseif c == "O" then + wputdarg(args[2].disp); narg = 3 + else + -- Anything else is an immediate operand. + local a = args[narg] + narg = narg + 1 + local mode, imm = a.mode, a.imm + if mode == "iJ" and not match("iIJ", c) then + werror("bad operand size for label") + end + if c == "S" then + wputsbarg(imm) + elseif c == "U" then + wputbarg(imm) + elseif c == "W" then + wputwarg(imm) + elseif c == "i" or c == "I" then + if mode == "iJ" then + wputlabel("IMM_", imm, 1) + elseif mode == "iI" and c == "I" then + waction(sz == "w" and "IMM_WB" or "IMM_DB", imm) + else + wputszarg(sz, imm) + end + elseif c == "J" then + if mode == "iPJ" then + waction("REL_A", imm) -- !x64 (secpos) + else + wputlabel("REL_", imm, 2) + end + elseif c == "s" then + local reg = a.reg + if reg < 0 then + wputb(0) + wvreg("imm.hi", a.vreg) + else + wputb(shl(reg, 4)) + end + else + werror("bad char `"..c.."' in pattern `"..pat.."' for `"..op.."'") + end + end + end + end +end + +------------------------------------------------------------------------------ + +-- Mapping of operand modes to short names. Suppress output with '#'. +local map_modename = { + r = "reg", R = "eax", C = "cl", x = "mem", m = "mrm", i = "imm", + f = "stx", F = "st0", J = "lbl", ["1"] = "1", + I = "#", S = "#", O = "#", +} + +-- Return a table/string showing all possible operand modes. +local function templatehelp(template, nparams) + if nparams == 0 then return "" end + local t = {} + for tm in gmatch(template, "[^%|]+") do + local s = map_modename[sub(tm, 1, 1)] + s = s..gsub(sub(tm, 2, nparams), ".", function(c) + return ", "..map_modename[c] + end) + if not match(s, "#") then t[#t+1] = s end + end + return t +end + +-- Match operand modes against mode match part of template. +local function matchtm(tm, args) + for i=1,#args do + if not match(args[i].mode, sub(tm, i, i)) then return end + end + return true +end + +-- Handle opcodes defined with template strings. +map_op[".template__"] = function(params, template, nparams) + if not params then return templatehelp(template, nparams) end + local args = {} + + -- Zero-operand opcodes have no match part. + if #params == 0 then + dopattern(template, args, "d", params.op, nil) + return + end + + -- Determine common operand size (coerce undefined size) or flag as mixed. + local sz, szmix, needrex + for i,p in ipairs(params) do + args[i] = parseoperand(p) + local nsz = args[i].opsize + if nsz then + if sz and sz ~= nsz then szmix = true else sz = nsz end + end + local nrex = args[i].needrex + if nrex ~= nil then + if needrex == nil then + needrex = nrex + elseif needrex ~= nrex then + werror("bad mix of byte-addressable registers") + end + end + end + + -- Try all match:pattern pairs (separated by '|'). + local gotmatch, lastpat + for tm in gmatch(template, "[^%|]+") do + -- Split off size match (starts after mode match) and pattern string. + local szm, pat = match(tm, "^(.-):(.*)$", #args+1) + if pat == "" then pat = lastpat else lastpat = pat end + if matchtm(tm, args) then + local prefix = sub(szm, 1, 1) + if prefix == "/" then -- Exactly match leading operand sizes. + for i = #szm,1,-1 do + if i == 1 then + dopattern(pat, args, sz, params.op, needrex) -- Process pattern. + return + elseif args[i-1].opsize ~= sub(szm, i, i) then + break + end + end + else -- Match common operand size. + local szp = sz + if szm == "" then szm = x64 and "qdwb" or "dwb" end -- Default sizes. + if prefix == "1" then szp = args[1].opsize; szmix = nil + elseif prefix == "2" then szp = args[2].opsize; szmix = nil end + if not szmix and (prefix == "." or match(szm, szp or "#")) then + dopattern(pat, args, szp, params.op, needrex) -- Process pattern. + return + end + end + gotmatch = true + end + end + + local msg = "bad operand mode" + if gotmatch then + if szmix then + msg = "mixed operand size" + else + msg = sz and "bad operand size" or "missing operand size" + end + end + + werror(msg.." in `"..opmodestr(params.op, args).."'") +end + +------------------------------------------------------------------------------ + +-- x64-specific opcode for 64 bit immediates and displacements. +if x64 then + function map_op.mov64_2(params) + if not params then return { "reg, imm", "reg, [disp]", "[disp], reg" } end + if secpos+2 > maxsecpos then wflush() end + local opcode, op64, sz, rex, vreg + local op64 = match(params[1], "^%[%s*(.-)%s*%]$") + if op64 then + local a = parseoperand(params[2]) + if a.mode ~= "rmR" then werror("bad operand mode") end + sz = a.opsize + rex = sz == "q" and 8 or 0 + opcode = 0xa3 + else + op64 = match(params[2], "^%[%s*(.-)%s*%]$") + local a = parseoperand(params[1]) + if op64 then + if a.mode ~= "rmR" then werror("bad operand mode") end + sz = a.opsize + rex = sz == "q" and 8 or 0 + opcode = 0xa1 + else + if sub(a.mode, 1, 1) ~= "r" or a.opsize ~= "q" then + werror("bad operand mode") + end + op64 = params[2] + if a.reg == -1 then + vreg = a.vreg + opcode = 0xb8 + else + opcode = 0xb8 + band(a.reg, 7) + end + rex = a.reg > 7 and 9 or 8 + end + end + local psz, sk = wputop(sz, opcode, rex, nil, vreg) + wvreg("opcode", vreg, psz, sk) + waction("IMM_D", format("(unsigned int)(%s)", op64)) + waction("IMM_D", format("(unsigned int)((%s)>>32)", op64)) + end +end + +------------------------------------------------------------------------------ + +-- Pseudo-opcodes for data storage. +local function op_data(params) + if not params then return "imm..." end + local sz = sub(params.op, 2, 2) + if sz == "a" then sz = addrsize end + for _,p in ipairs(params) do + local a = parseoperand(p) + if sub(a.mode, 1, 1) ~= "i" or (a.opsize and a.opsize ~= sz) then + werror("bad mode or size in `"..p.."'") + end + if a.mode == "iJ" then + wputlabel("IMM_", a.imm, 1) + else + wputszarg(sz, a.imm) + end + if secpos+2 > maxsecpos then wflush() end + end +end + +map_op[".byte_*"] = op_data +map_op[".sbyte_*"] = op_data +map_op[".word_*"] = op_data +map_op[".dword_*"] = op_data +map_op[".aword_*"] = op_data + +------------------------------------------------------------------------------ + +-- Pseudo-opcode to mark the position where the action list is to be emitted. +map_op[".actionlist_1"] = function(params) + if not params then return "cvar" end + local name = params[1] -- No syntax check. You get to keep the pieces. + wline(function(out) writeactions(out, name) end) +end + +-- Pseudo-opcode to mark the position where the global enum is to be emitted. +map_op[".globals_1"] = function(params) + if not params then return "prefix" end + local prefix = params[1] -- No syntax check. You get to keep the pieces. + wline(function(out) writeglobals(out, prefix) end) +end + +-- Pseudo-opcode to mark the position where the global names are to be emitted. +map_op[".globalnames_1"] = function(params) + if not params then return "cvar" end + local name = params[1] -- No syntax check. You get to keep the pieces. + wline(function(out) writeglobalnames(out, name) end) +end + +-- Pseudo-opcode to mark the position where the extern names are to be emitted. +map_op[".externnames_1"] = function(params) + if not params then return "cvar" end + local name = params[1] -- No syntax check. You get to keep the pieces. + wline(function(out) writeexternnames(out, name) end) +end + +------------------------------------------------------------------------------ + +-- Label pseudo-opcode (converted from trailing colon form). +map_op[".label_2"] = function(params) + if not params then return "[1-9] | ->global | =>pcexpr [, addr]" end + if secpos+2 > maxsecpos then wflush() end + local a = parseoperand(params[1]) + local mode, imm = a.mode, a.imm + if type(imm) == "number" and (mode == "iJ" or (imm >= 1 and imm <= 9)) then + -- Local label (1: ... 9:) or global label (->global:). + waction("LABEL_LG", nil, 1) + wputxb(imm) + elseif mode == "iJ" then + -- PC label (=>pcexpr:). + waction("LABEL_PC", imm) + else + werror("bad label definition") + end + -- SETLABEL must immediately follow LABEL_LG/LABEL_PC. + local addr = params[2] + if addr then + local a = parseoperand(addr) + if a.mode == "iPJ" then + waction("SETLABEL", a.imm) + else + werror("bad label assignment") + end + end +end +map_op[".label_1"] = map_op[".label_2"] + +------------------------------------------------------------------------------ + +-- Alignment pseudo-opcode. +map_op[".align_1"] = function(params) + if not params then return "numpow2" end + if secpos+1 > maxsecpos then wflush() end + local align = tonumber(params[1]) or map_opsizenum[map_opsize[params[1]]] + if align then + local x = align + -- Must be a power of 2 in the range (2 ... 256). + for i=1,8 do + x = x / 2 + if x == 1 then + waction("ALIGN", nil, 1) + wputxb(align-1) -- Action byte is 2**n-1. + return + end + end + end + werror("bad alignment") +end + +-- Spacing pseudo-opcode. +map_op[".space_2"] = function(params) + if not params then return "num [, filler]" end + if secpos+1 > maxsecpos then wflush() end + waction("SPACE", params[1]) + local fill = params[2] + if fill then + fill = tonumber(fill) + if not fill or fill < 0 or fill > 255 then werror("bad filler") end + end + wputxb(fill or 0) +end +map_op[".space_1"] = map_op[".space_2"] + +------------------------------------------------------------------------------ + +-- Pseudo-opcode for (primitive) type definitions (map to C types). +map_op[".type_3"] = function(params, nparams) + if not params then + return nparams == 2 and "name, ctype" or "name, ctype, reg" + end + local name, ctype, reg = params[1], params[2], params[3] + if not match(name, "^[%a_][%w_]*$") then + werror("bad type name `"..name.."'") + end + local tp = map_type[name] + if tp then + werror("duplicate type `"..name.."'") + end + if reg and not map_reg_valid_base[reg] then + werror("bad base register `"..(map_reg_rev[reg] or reg).."'") + end + -- Add #type to defines. A bit unclean to put it in map_archdef. + map_archdef["#"..name] = "sizeof("..ctype..")" + -- Add new type and emit shortcut define. + local num = ctypenum + 1 + map_type[name] = { + ctype = ctype, + ctypefmt = format("Dt%X(%%s)", num), + reg = reg, + } + wline(format("#define Dt%X(_V) (int)(ptrdiff_t)&(((%s *)0)_V)", num, ctype)) + ctypenum = num +end +map_op[".type_2"] = map_op[".type_3"] + +-- Dump type definitions. +local function dumptypes(out, lvl) + local t = {} + for name in pairs(map_type) do t[#t+1] = name end + sort(t) + out:write("Type definitions:\n") + for _,name in ipairs(t) do + local tp = map_type[name] + local reg = tp.reg and map_reg_rev[tp.reg] or "" + out:write(format(" %-20s %-20s %s\n", name, tp.ctype, reg)) + end + out:write("\n") +end + +------------------------------------------------------------------------------ + +-- Set the current section. +function _M.section(num) + waction("SECTION") + wputxb(num) + wflush(true) -- SECTION is a terminal action. +end + +------------------------------------------------------------------------------ + +-- Dump architecture description. +function _M.dumparch(out) + out:write(format("DynASM %s version %s, released %s\n\n", + _info.arch, _info.version, _info.release)) + dumpregs(out) + dumpactions(out) +end + +-- Dump all user defined elements. +function _M.dumpdef(out, lvl) + dumptypes(out, lvl) + dumpglobals(out, lvl) + dumpexterns(out, lvl) +end + +------------------------------------------------------------------------------ + +-- Pass callbacks from/to the DynASM core. +function _M.passcb(wl, we, wf, ww) + wline, werror, wfatal, wwarn = wl, we, wf, ww + return wflush +end + +-- Setup the arch-specific module. +function _M.setup(arch, opt) + g_arch, g_opt = arch, opt +end + +-- Merge the core maps and the arch-specific maps. +function _M.mergemaps(map_coreop, map_def) + setmetatable(map_op, { __index = map_coreop }) + setmetatable(map_def, { __index = map_archdef }) + return map_op, map_def +end + +return _M + +------------------------------------------------------------------------------ + diff --git a/ext/opcache/dynasm/dynasm.lua b/ext/opcache/dynasm/dynasm.lua new file mode 100644 index 0000000000000..1e5899a4bf093 --- /dev/null +++ b/ext/opcache/dynasm/dynasm.lua @@ -0,0 +1,1094 @@ +------------------------------------------------------------------------------ +-- DynASM. A dynamic assembler for code generation engines. +-- Originally designed and implemented for LuaJIT. +-- +-- Copyright (C) 2005-2016 Mike Pall. All rights reserved. +-- See below for full copyright notice. +------------------------------------------------------------------------------ + +-- Application information. +local _info = { + name = "DynASM", + description = "A dynamic assembler for code generation engines", + version = "1.4.0", + vernum = 10400, + release = "2015-10-18", + author = "Mike Pall", + url = "http://luajit.org/dynasm.html", + license = "MIT", + copyright = [[ +Copyright (C) 2005-2016 Mike Pall. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +[ MIT license: http://www.opensource.org/licenses/mit-license.php ] +]], +} + +-- Cache library functions. +local type, pairs, ipairs = type, pairs, ipairs +local pcall, error, assert = pcall, error, assert +local _s = string +local sub, match, gmatch, gsub = _s.sub, _s.match, _s.gmatch, _s.gsub +local format, rep, upper = _s.format, _s.rep, _s.upper +local _t = table +local insert, remove, concat, sort = _t.insert, _t.remove, _t.concat, _t.sort +local exit = os.exit +local io = io +local stdin, stdout, stderr = io.stdin, io.stdout, io.stderr + +------------------------------------------------------------------------------ + +-- Program options. +local g_opt = {} + +-- Global state for current file. +local g_fname, g_curline, g_indent, g_lineno, g_synclineno, g_arch +local g_errcount = 0 + +-- Write buffer for output file. +local g_wbuffer, g_capbuffer + +------------------------------------------------------------------------------ + +-- Write an output line (or callback function) to the buffer. +local function wline(line, needindent) + local buf = g_capbuffer or g_wbuffer + buf[#buf+1] = needindent and g_indent..line or line + g_synclineno = g_synclineno + 1 +end + +-- Write assembler line as a comment, if requestd. +local function wcomment(aline) + if g_opt.comment then + wline(g_opt.comment..aline..g_opt.endcomment, true) + end +end + +-- Resync CPP line numbers. +local function wsync() + if g_synclineno ~= g_lineno and g_opt.cpp then + wline("#line "..g_lineno..' "'..g_fname..'"') + g_synclineno = g_lineno + end +end + +-- Dummy action flush function. Replaced with arch-specific function later. +local function wflush(term) +end + +-- Dump all buffered output lines. +local function wdumplines(out, buf) + for _,line in ipairs(buf) do + if type(line) == "string" then + assert(out:write(line, "\n")) + else + -- Special callback to dynamically insert lines after end of processing. + line(out) + end + end +end + +------------------------------------------------------------------------------ + +-- Emit an error. Processing continues with next statement. +local function werror(msg) + error(format("%s:%s: error: %s:\n%s", g_fname, g_lineno, msg, g_curline), 0) +end + +-- Emit a fatal error. Processing stops. +local function wfatal(msg) + g_errcount = "fatal" + werror(msg) +end + +-- Print a warning. Processing continues. +local function wwarn(msg) + stderr:write(format("%s:%s: warning: %s:\n%s\n", + g_fname, g_lineno, msg, g_curline)) +end + +-- Print caught error message. But suppress excessive errors. +local function wprinterr(...) + if type(g_errcount) == "number" then + -- Regular error. + g_errcount = g_errcount + 1 + if g_errcount < 21 then -- Seems to be a reasonable limit. + stderr:write(...) + elseif g_errcount == 21 then + stderr:write(g_fname, + ":*: warning: too many errors (suppressed further messages).\n") + end + else + -- Fatal error. + stderr:write(...) + return true -- Stop processing. + end +end + +------------------------------------------------------------------------------ + +-- Map holding all option handlers. +local opt_map = {} +local opt_current + +-- Print error and exit with error status. +local function opterror(...) + stderr:write("dynasm.lua: ERROR: ", ...) + stderr:write("\n") + exit(1) +end + +-- Get option parameter. +local function optparam(args) + local argn = args.argn + local p = args[argn] + if not p then + opterror("missing parameter for option `", opt_current, "'.") + end + args.argn = argn + 1 + return p +end + +------------------------------------------------------------------------------ + +-- Core pseudo-opcodes. +local map_coreop = {} +-- Dummy opcode map. Replaced by arch-specific map. +local map_op = {} + +-- Forward declarations. +local dostmt +local readfile + +------------------------------------------------------------------------------ + +-- Map for defines (initially empty, chains to arch-specific map). +local map_def = {} + +-- Pseudo-opcode to define a substitution. +map_coreop[".define_2"] = function(params, nparams) + if not params then return nparams == 1 and "name" or "name, subst" end + local name, def = params[1], params[2] or "1" + if not match(name, "^[%a_][%w_]*$") then werror("bad or duplicate define") end + map_def[name] = def +end +map_coreop[".define_1"] = map_coreop[".define_2"] + +-- Define a substitution on the command line. +function opt_map.D(args) + local namesubst = optparam(args) + local name, subst = match(namesubst, "^([%a_][%w_]*)=(.*)$") + if name then + map_def[name] = subst + elseif match(namesubst, "^[%a_][%w_]*$") then + map_def[namesubst] = "1" + else + opterror("bad define") + end +end + +-- Undefine a substitution on the command line. +function opt_map.U(args) + local name = optparam(args) + if match(name, "^[%a_][%w_]*$") then + map_def[name] = nil + else + opterror("bad define") + end +end + +-- Helper for definesubst. +local gotsubst + +local function definesubst_one(word) + local subst = map_def[word] + if subst then gotsubst = word; return subst else return word end +end + +-- Iteratively substitute defines. +local function definesubst(stmt) + -- Limit number of iterations. + for i=1,100 do + gotsubst = false + stmt = gsub(stmt, "#?[%w_]+", definesubst_one) + if not gotsubst then break end + end + if gotsubst then wfatal("recursive define involving `"..gotsubst.."'") end + return stmt +end + +-- Dump all defines. +local function dumpdefines(out, lvl) + local t = {} + for name in pairs(map_def) do + t[#t+1] = name + end + sort(t) + out:write("Defines:\n") + for _,name in ipairs(t) do + local subst = map_def[name] + if g_arch then subst = g_arch.revdef(subst) end + out:write(format(" %-20s %s\n", name, subst)) + end + out:write("\n") +end + +------------------------------------------------------------------------------ + +-- Support variables for conditional assembly. +local condlevel = 0 +local condstack = {} + +-- Evaluate condition with a Lua expression. Substitutions already performed. +local function cond_eval(cond) + local func, err + if setfenv then + func, err = loadstring("return "..cond, "=expr") + else + -- No globals. All unknown identifiers evaluate to nil. + func, err = load("return "..cond, "=expr", "t", {}) + end + if func then + if setfenv then + setfenv(func, {}) -- No globals. All unknown identifiers evaluate to nil. + end + local ok, res = pcall(func) + if ok then + if res == 0 then return false end -- Oh well. + return not not res + end + err = res + end + wfatal("bad condition: "..err) +end + +-- Skip statements until next conditional pseudo-opcode at the same level. +local function stmtskip() + local dostmt_save = dostmt + local lvl = 0 + dostmt = function(stmt) + local op = match(stmt, "^%s*(%S+)") + if op == ".if" then + lvl = lvl + 1 + elseif lvl ~= 0 then + if op == ".endif" then lvl = lvl - 1 end + elseif op == ".elif" or op == ".else" or op == ".endif" then + dostmt = dostmt_save + dostmt(stmt) + end + end +end + +-- Pseudo-opcodes for conditional assembly. +map_coreop[".if_1"] = function(params) + if not params then return "condition" end + local lvl = condlevel + 1 + local res = cond_eval(params[1]) + condlevel = lvl + condstack[lvl] = res + if not res then stmtskip() end +end + +map_coreop[".elif_1"] = function(params) + if not params then return "condition" end + if condlevel == 0 then wfatal(".elif without .if") end + local lvl = condlevel + local res = condstack[lvl] + if res then + if res == "else" then wfatal(".elif after .else") end + else + res = cond_eval(params[1]) + if res then + condstack[lvl] = res + return + end + end + stmtskip() +end + +map_coreop[".else_0"] = function(params) + if condlevel == 0 then wfatal(".else without .if") end + local lvl = condlevel + local res = condstack[lvl] + condstack[lvl] = "else" + if res then + if res == "else" then wfatal(".else after .else") end + stmtskip() + end +end + +map_coreop[".endif_0"] = function(params) + local lvl = condlevel + if lvl == 0 then wfatal(".endif without .if") end + condlevel = lvl - 1 +end + +-- Check for unfinished conditionals. +local function checkconds() + if g_errcount ~= "fatal" and condlevel ~= 0 then + wprinterr(g_fname, ":*: error: unbalanced conditional\n") + end +end + +------------------------------------------------------------------------------ + +-- Search for a file in the given path and open it for reading. +local function pathopen(path, name) + local dirsep = package and match(package.path, "\\") and "\\" or "/" + for _,p in ipairs(path) do + local fullname = p == "" and name or p..dirsep..name + local fin = io.open(fullname, "r") + if fin then + g_fname = fullname + return fin + end + end +end + +-- Include a file. +map_coreop[".include_1"] = function(params) + if not params then return "filename" end + local name = params[1] + -- Save state. Ugly, I know. but upvalues are fast. + local gf, gl, gcl, gi = g_fname, g_lineno, g_curline, g_indent + -- Read the included file. + local fatal = readfile(pathopen(g_opt.include, name) or + wfatal("include file `"..name.."' not found")) + -- Restore state. + g_synclineno = -1 + g_fname, g_lineno, g_curline, g_indent = gf, gl, gcl, gi + if fatal then wfatal("in include file") end +end + +-- Make .include and conditionals initially available, too. +map_op[".include_1"] = map_coreop[".include_1"] +map_op[".if_1"] = map_coreop[".if_1"] +map_op[".elif_1"] = map_coreop[".elif_1"] +map_op[".else_0"] = map_coreop[".else_0"] +map_op[".endif_0"] = map_coreop[".endif_0"] + +------------------------------------------------------------------------------ + +-- Support variables for macros. +local mac_capture, mac_lineno, mac_name +local mac_active = {} +local mac_list = {} + +-- Pseudo-opcode to define a macro. +map_coreop[".macro_*"] = function(mparams) + if not mparams then return "name [, params...]" end + -- Split off and validate macro name. + local name = remove(mparams, 1) + if not name then werror("missing macro name") end + if not (match(name, "^[%a_][%w_%.]*$") or match(name, "^%.[%w_%.]*$")) then + wfatal("bad macro name `"..name.."'") + end + -- Validate macro parameter names. + local mdup = {} + for _,mp in ipairs(mparams) do + if not match(mp, "^[%a_][%w_]*$") then + wfatal("bad macro parameter name `"..mp.."'") + end + if mdup[mp] then wfatal("duplicate macro parameter name `"..mp.."'") end + mdup[mp] = true + end + -- Check for duplicate or recursive macro definitions. + local opname = name.."_"..#mparams + if map_op[opname] or map_op[name.."_*"] then + wfatal("duplicate macro `"..name.."' ("..#mparams.." parameters)") + end + if mac_capture then wfatal("recursive macro definition") end + + -- Enable statement capture. + local lines = {} + mac_lineno = g_lineno + mac_name = name + mac_capture = function(stmt) -- Statement capture function. + -- Stop macro definition with .endmacro pseudo-opcode. + if not match(stmt, "^%s*.endmacro%s*$") then + lines[#lines+1] = stmt + return + end + mac_capture = nil + mac_lineno = nil + mac_name = nil + mac_list[#mac_list+1] = opname + -- Add macro-op definition. + map_op[opname] = function(params) + if not params then return mparams, lines end + -- Protect against recursive macro invocation. + if mac_active[opname] then wfatal("recursive macro invocation") end + mac_active[opname] = true + -- Setup substitution map. + local subst = {} + for i,mp in ipairs(mparams) do subst[mp] = params[i] end + local mcom + if g_opt.maccomment and g_opt.comment then + mcom = " MACRO "..name.." ("..#mparams..")" + wcomment("{"..mcom) + end + -- Loop through all captured statements + for _,stmt in ipairs(lines) do + -- Substitute macro parameters. + local st = gsub(stmt, "[%w_]+", subst) + st = definesubst(st) + st = gsub(st, "%s*%.%.%s*", "") -- Token paste a..b. + if mcom and sub(st, 1, 1) ~= "|" then wcomment(st) end + -- Emit statement. Use a protected call for better diagnostics. + local ok, err = pcall(dostmt, st) + if not ok then + -- Add the captured statement to the error. + wprinterr(err, "\n", g_indent, "| ", stmt, + "\t[MACRO ", name, " (", #mparams, ")]\n") + end + end + if mcom then wcomment("}"..mcom) end + mac_active[opname] = nil + end + end +end + +-- An .endmacro pseudo-opcode outside of a macro definition is an error. +map_coreop[".endmacro_0"] = function(params) + wfatal(".endmacro without .macro") +end + +-- Dump all macros and their contents (with -PP only). +local function dumpmacros(out, lvl) + sort(mac_list) + out:write("Macros:\n") + for _,opname in ipairs(mac_list) do + local name = sub(opname, 1, -3) + local params, lines = map_op[opname]() + out:write(format(" %-20s %s\n", name, concat(params, ", "))) + if lvl > 1 then + for _,line in ipairs(lines) do + out:write(" |", line, "\n") + end + out:write("\n") + end + end + out:write("\n") +end + +-- Check for unfinished macro definitions. +local function checkmacros() + if mac_capture then + wprinterr(g_fname, ":", mac_lineno, + ": error: unfinished .macro `", mac_name ,"'\n") + end +end + +------------------------------------------------------------------------------ + +-- Support variables for captures. +local cap_lineno, cap_name +local cap_buffers = {} +local cap_used = {} + +-- Start a capture. +map_coreop[".capture_1"] = function(params) + if not params then return "name" end + wflush() + local name = params[1] + if not match(name, "^[%a_][%w_]*$") then + wfatal("bad capture name `"..name.."'") + end + if cap_name then + wfatal("already capturing to `"..cap_name.."' since line "..cap_lineno) + end + cap_name = name + cap_lineno = g_lineno + -- Create or continue a capture buffer and start the output line capture. + local buf = cap_buffers[name] + if not buf then buf = {}; cap_buffers[name] = buf end + g_capbuffer = buf + g_synclineno = 0 +end + +-- Stop a capture. +map_coreop[".endcapture_0"] = function(params) + wflush() + if not cap_name then wfatal(".endcapture without a valid .capture") end + cap_name = nil + cap_lineno = nil + g_capbuffer = nil + g_synclineno = 0 +end + +-- Dump a capture buffer. +map_coreop[".dumpcapture_1"] = function(params) + if not params then return "name" end + wflush() + local name = params[1] + if not match(name, "^[%a_][%w_]*$") then + wfatal("bad capture name `"..name.."'") + end + cap_used[name] = true + wline(function(out) + local buf = cap_buffers[name] + if buf then wdumplines(out, buf) end + end) + g_synclineno = 0 +end + +-- Dump all captures and their buffers (with -PP only). +local function dumpcaptures(out, lvl) + out:write("Captures:\n") + for name,buf in pairs(cap_buffers) do + out:write(format(" %-20s %4s)\n", name, "("..#buf)) + if lvl > 1 then + local bar = rep("=", 76) + out:write(" ", bar, "\n") + for _,line in ipairs(buf) do + out:write(" ", line, "\n") + end + out:write(" ", bar, "\n\n") + end + end + out:write("\n") +end + +-- Check for unfinished or unused captures. +local function checkcaptures() + if cap_name then + wprinterr(g_fname, ":", cap_lineno, + ": error: unfinished .capture `", cap_name,"'\n") + return + end + for name in pairs(cap_buffers) do + if not cap_used[name] then + wprinterr(g_fname, ":*: error: missing .dumpcapture ", name ,"\n") + end + end +end + +------------------------------------------------------------------------------ + +-- Sections names. +local map_sections = {} + +-- Pseudo-opcode to define code sections. +-- TODO: Data sections, BSS sections. Needs extra C code and API. +map_coreop[".section_*"] = function(params) + if not params then return "name..." end + if #map_sections > 0 then werror("duplicate section definition") end + wflush() + for sn,name in ipairs(params) do + local opname = "."..name.."_0" + if not match(name, "^[%a][%w_]*$") or + map_op[opname] or map_op["."..name.."_*"] then + werror("bad section name `"..name.."'") + end + map_sections[#map_sections+1] = name + wline(format("#define DASM_SECTION_%s\t%d", upper(name), sn-1)) + map_op[opname] = function(params) g_arch.section(sn-1) end + end + wline(format("#define DASM_MAXSECTION\t\t%d", #map_sections)) +end + +-- Dump all sections. +local function dumpsections(out, lvl) + out:write("Sections:\n") + for _,name in ipairs(map_sections) do + out:write(format(" %s\n", name)) + end + out:write("\n") +end + +------------------------------------------------------------------------------ + +-- Replacement for customized Lua, which lacks the package library. +local prefix = "" +if not require then + function require(name) + local fp = assert(io.open(prefix..name..".lua")) + local s = fp:read("*a") + assert(fp:close()) + return assert(loadstring(s, "@"..name..".lua"))() + end +end + +-- Load architecture-specific module. +local function loadarch(arch) + if not match(arch, "^[%w_]+$") then return "bad arch name" end + local ok, m_arch = pcall(require, "dasm_"..arch) + if not ok then return "cannot load module: "..m_arch end + g_arch = m_arch + wflush = m_arch.passcb(wline, werror, wfatal, wwarn) + m_arch.setup(arch, g_opt) + map_op, map_def = m_arch.mergemaps(map_coreop, map_def) +end + +-- Dump architecture description. +function opt_map.dumparch(args) + local name = optparam(args) + if not g_arch then + local err = loadarch(name) + if err then opterror(err) end + end + + local t = {} + for name in pairs(map_coreop) do t[#t+1] = name end + for name in pairs(map_op) do t[#t+1] = name end + sort(t) + + local out = stdout + local _arch = g_arch._info + out:write(format("%s version %s, released %s, %s\n", + _info.name, _info.version, _info.release, _info.url)) + g_arch.dumparch(out) + + local pseudo = true + out:write("Pseudo-Opcodes:\n") + for _,sname in ipairs(t) do + local name, nparam = match(sname, "^(.+)_([0-9%*])$") + if name then + if pseudo and sub(name, 1, 1) ~= "." then + out:write("\nOpcodes:\n") + pseudo = false + end + local f = map_op[sname] + local s + if nparam ~= "*" then nparam = nparam + 0 end + if nparam == 0 then + s = "" + elseif type(f) == "string" then + s = map_op[".template__"](nil, f, nparam) + else + s = f(nil, nparam) + end + if type(s) == "table" then + for _,s2 in ipairs(s) do + out:write(format(" %-12s %s\n", name, s2)) + end + else + out:write(format(" %-12s %s\n", name, s)) + end + end + end + out:write("\n") + exit(0) +end + +-- Pseudo-opcode to set the architecture. +-- Only initially available (map_op is replaced when called). +map_op[".arch_1"] = function(params) + if not params then return "name" end + local err = loadarch(params[1]) + if err then wfatal(err) end + wline(format("#if DASM_VERSION != %d", _info.vernum)) + wline('#error "Version mismatch between DynASM and included encoding engine"') + wline("#endif") +end + +-- Dummy .arch pseudo-opcode to improve the error report. +map_coreop[".arch_1"] = function(params) + if not params then return "name" end + wfatal("duplicate .arch statement") +end + +------------------------------------------------------------------------------ + +-- Dummy pseudo-opcode. Don't confuse '.nop' with 'nop'. +map_coreop[".nop_*"] = function(params) + if not params then return "[ignored...]" end +end + +-- Pseudo-opcodes to raise errors. +map_coreop[".error_1"] = function(params) + if not params then return "message" end + werror(params[1]) +end + +map_coreop[".fatal_1"] = function(params) + if not params then return "message" end + wfatal(params[1]) +end + +-- Dump all user defined elements. +local function dumpdef(out) + local lvl = g_opt.dumpdef + if lvl == 0 then return end + dumpsections(out, lvl) + dumpdefines(out, lvl) + if g_arch then g_arch.dumpdef(out, lvl) end + dumpmacros(out, lvl) + dumpcaptures(out, lvl) +end + +------------------------------------------------------------------------------ + +-- Helper for splitstmt. +local splitlvl + +local function splitstmt_one(c) + if c == "(" then + splitlvl = ")"..splitlvl + elseif c == "[" then + splitlvl = "]"..splitlvl + elseif c == "{" then + splitlvl = "}"..splitlvl + elseif c == ")" or c == "]" or c == "}" then + if sub(splitlvl, 1, 1) ~= c then werror("unbalanced (), [] or {}") end + splitlvl = sub(splitlvl, 2) + elseif splitlvl == "" then + return " \0 " + end + return c +end + +-- Split statement into (pseudo-)opcode and params. +local function splitstmt(stmt) + -- Convert label with trailing-colon into .label statement. + local label = match(stmt, "^%s*(.+):%s*$") + if label then return ".label", {label} end + + -- Split at commas and equal signs, but obey parentheses and brackets. + splitlvl = "" + stmt = gsub(stmt, "[,%(%)%[%]{}]", splitstmt_one) + if splitlvl ~= "" then werror("unbalanced () or []") end + + -- Split off opcode. + local op, other = match(stmt, "^%s*([^%s%z]+)%s*(.*)$") + if not op then werror("bad statement syntax") end + + -- Split parameters. + local params = {} + for p in gmatch(other, "%s*(%Z+)%z?") do + params[#params+1] = gsub(p, "%s+$", "") + end + if #params > 16 then werror("too many parameters") end + + params.op = op + return op, params +end + +-- Process a single statement. +dostmt = function(stmt) + -- Ignore empty statements. + if match(stmt, "^%s*$") then return end + + -- Capture macro defs before substitution. + if mac_capture then return mac_capture(stmt) end + stmt = definesubst(stmt) + + -- Emit C code without parsing the line. + if sub(stmt, 1, 1) == "|" then + local tail = sub(stmt, 2) + wflush() + if sub(tail, 1, 2) == "//" then wcomment(tail) else wline(tail, true) end + return + end + + -- Split into (pseudo-)opcode and params. + local op, params = splitstmt(stmt) + + -- Get opcode handler (matching # of parameters or generic handler). + local f = map_op[op.."_"..#params] or map_op[op.."_*"] + if not f then + if not g_arch then wfatal("first statement must be .arch") end + -- Improve error report. + for i=0,9 do + if map_op[op.."_"..i] then + werror("wrong number of parameters for `"..op.."'") + end + end + werror("unknown statement `"..op.."'") + end + + -- Call opcode handler or special handler for template strings. + if type(f) == "string" then + map_op[".template__"](params, f) + else + f(params) + end +end + +-- Process a single line. +local function doline(line) + if g_opt.flushline then wflush() end + + -- Assembler line? + local indent, aline = match(line, "^(%s*)%|(.*)$") + if not aline then + -- No, plain C code line, need to flush first. + wflush() + wsync() + wline(line, false) + return + end + + g_indent = indent -- Remember current line indentation. + + -- Emit C code (even from macros). Avoids echo and line parsing. + if sub(aline, 1, 1) == "|" then + if not mac_capture then + wsync() + elseif g_opt.comment then + wsync() + wcomment(aline) + end + dostmt(aline) + return + end + + -- Echo assembler line as a comment. + if g_opt.comment then + wsync() + wcomment(aline) + end + + -- Strip assembler comments. + aline = gsub(aline, "//.*$", "") + + -- Split line into statements at semicolons. + if match(aline, ";") then + for stmt in gmatch(aline, "[^;]+") do dostmt(stmt) end + else + dostmt(aline) + end +end + +------------------------------------------------------------------------------ + +-- Write DynASM header. +local function dasmhead(out) + out:write(format([[ +/* +** This file has been pre-processed with DynASM. +** %s +** DynASM version %s, DynASM %s version %s +** DO NOT EDIT! The original file is in "%s". +*/ + +]], _info.url, + _info.version, g_arch._info.arch, g_arch._info.version, + g_fname)) +end + +-- Read input file. +readfile = function(fin) + g_indent = "" + g_lineno = 0 + g_synclineno = -1 + + -- Process all lines. + for line in fin:lines() do + g_lineno = g_lineno + 1 + g_curline = line + local ok, err = pcall(doline, line) + if not ok and wprinterr(err, "\n") then return true end + end + wflush() + + -- Close input file. + assert(fin == stdin or fin:close()) +end + +-- Write output file. +local function writefile(outfile) + local fout + + -- Open output file. + if outfile == nil or outfile == "-" then + fout = stdout + else + fout = assert(io.open(outfile, "w")) + end + + -- Write all buffered lines + wdumplines(fout, g_wbuffer) + + -- Close output file. + assert(fout == stdout or fout:close()) + + -- Optionally dump definitions. + dumpdef(fout == stdout and stderr or stdout) +end + +-- Translate an input file to an output file. +local function translate(infile, outfile) + g_wbuffer = {} + g_indent = "" + g_lineno = 0 + g_synclineno = -1 + + -- Put header. + wline(dasmhead) + + -- Read input file. + local fin + if infile == "-" then + g_fname = "(stdin)" + fin = stdin + else + g_fname = infile + fin = assert(io.open(infile, "r")) + end + readfile(fin) + + -- Check for errors. + if not g_arch then + wprinterr(g_fname, ":*: error: missing .arch directive\n") + end + checkconds() + checkmacros() + checkcaptures() + + if g_errcount ~= 0 then + stderr:write(g_fname, ":*: info: ", g_errcount, " error", + (type(g_errcount) == "number" and g_errcount > 1) and "s" or "", + " in input file -- no output file generated.\n") + dumpdef(stderr) + exit(1) + end + + -- Write output file. + writefile(outfile) +end + +------------------------------------------------------------------------------ + +-- Print help text. +function opt_map.help() + stdout:write("DynASM -- ", _info.description, ".\n") + stdout:write("DynASM ", _info.version, " ", _info.release, " ", _info.url, "\n") + stdout:write[[ + +Usage: dynasm [OPTION]... INFILE.dasc|- + + -h, --help Display this help text. + -V, --version Display version and copyright information. + + -o, --outfile FILE Output file name (default is stdout). + -I, --include DIR Add directory to the include search path. + + -c, --ccomment Use /* */ comments for assembler lines. + -C, --cppcomment Use // comments for assembler lines (default). + -N, --nocomment Suppress assembler lines in output. + -M, --maccomment Show macro expansions as comments (default off). + + -L, --nolineno Suppress CPP line number information in output. + -F, --flushline Flush action list for every line. + + -D NAME[=SUBST] Define a substitution. + -U NAME Undefine a substitution. + + -P, --dumpdef Dump defines, macros, etc. Repeat for more output. + -A, --dumparch ARCH Load architecture ARCH and dump description. +]] + exit(0) +end + +-- Print version information. +function opt_map.version() + stdout:write(format("%s version %s, released %s\n%s\n\n%s", + _info.name, _info.version, _info.release, _info.url, _info.copyright)) + exit(0) +end + +-- Misc. options. +function opt_map.outfile(args) g_opt.outfile = optparam(args) end +function opt_map.include(args) insert(g_opt.include, 1, optparam(args)) end +function opt_map.ccomment() g_opt.comment = "/*|"; g_opt.endcomment = " */" end +function opt_map.cppcomment() g_opt.comment = "//|"; g_opt.endcomment = "" end +function opt_map.nocomment() g_opt.comment = false end +function opt_map.maccomment() g_opt.maccomment = true end +function opt_map.nolineno() g_opt.cpp = false end +function opt_map.flushline() g_opt.flushline = true end +function opt_map.dumpdef() g_opt.dumpdef = g_opt.dumpdef + 1 end + +------------------------------------------------------------------------------ + +-- Short aliases for long options. +local opt_alias = { + h = "help", ["?"] = "help", V = "version", + o = "outfile", I = "include", + c = "ccomment", C = "cppcomment", N = "nocomment", M = "maccomment", + L = "nolineno", F = "flushline", + P = "dumpdef", A = "dumparch", +} + +-- Parse single option. +local function parseopt(opt, args) + opt_current = #opt == 1 and "-"..opt or "--"..opt + local f = opt_map[opt] or opt_map[opt_alias[opt]] + if not f then + opterror("unrecognized option `", opt_current, "'. Try `--help'.\n") + end + f(args) +end + +-- Parse arguments. +local function parseargs(args) + -- Default options. + g_opt.comment = "//|" + g_opt.endcomment = "" + g_opt.cpp = true + g_opt.dumpdef = 0 + g_opt.include = { "" } + + -- Process all option arguments. + args.argn = 1 + repeat + local a = args[args.argn] + if not a then break end + local lopt, opt = match(a, "^%-(%-?)(.+)") + if not opt then break end + args.argn = args.argn + 1 + if lopt == "" then + -- Loop through short options. + for o in gmatch(opt, ".") do parseopt(o, args) end + else + -- Long option. + parseopt(opt, args) + end + until false + + -- Check for proper number of arguments. + local nargs = #args - args.argn + 1 + if nargs ~= 1 then + if nargs == 0 then + if g_opt.dumpdef > 0 then return dumpdef(stdout) end + end + opt_map.help() + end + + -- Translate a single input file to a single output file + -- TODO: Handle multiple files? + translate(args[args.argn], g_opt.outfile) +end + +------------------------------------------------------------------------------ + +-- Add the directory dynasm.lua resides in to the Lua module search path. +local arg = arg +if arg and arg[0] then + prefix = match(arg[0], "^(.*[/\\])") + if package and prefix then package.path = prefix.."?.lua;"..package.path end +end + +-- Start DynASM. +parseargs{...} + +------------------------------------------------------------------------------ + diff --git a/ext/opcache/dynasm/minilua.c b/ext/opcache/dynasm/minilua.c new file mode 100644 index 0000000000000..79150286cfc3d --- /dev/null +++ b/ext/opcache/dynasm/minilua.c @@ -0,0 +1,7770 @@ +/* This is a heavily customized and minimized copy of Lua 5.1.5. */ +/* It's only used to build LuaJIT. It does NOT have all standard functions! */ +/****************************************************************************** +* Copyright (C) 1994-2012 Lua.org, PUC-Rio. All rights reserved. +* +* Permission is hereby granted, free of charge, to any person obtaining +* a copy of this software and associated documentation files (the +* "Software"), to deal in the Software without restriction, including +* without limitation the rights to use, copy, modify, merge, publish, +* distribute, sublicense, and/or sell copies of the Software, and to +* permit persons to whom the Software is furnished to do so, subject to +* the following conditions: +* +* The above copyright notice and this permission notice shall be +* included in all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +******************************************************************************/ +#ifdef _MSC_VER +typedef unsigned __int64 U64; +#else +typedef unsigned long long U64; +#endif +int _CRT_glob = 0; +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +typedef enum{ +TM_INDEX, +TM_NEWINDEX, +TM_GC, +TM_MODE, +TM_EQ, +TM_ADD, +TM_SUB, +TM_MUL, +TM_DIV, +TM_MOD, +TM_POW, +TM_UNM, +TM_LEN, +TM_LT, +TM_LE, +TM_CONCAT, +TM_CALL, +TM_N +}TMS; +enum OpMode{iABC,iABx,iAsBx}; +typedef enum{ +OP_MOVE, +OP_LOADK, +OP_LOADBOOL, +OP_LOADNIL, +OP_GETUPVAL, +OP_GETGLOBAL, +OP_GETTABLE, +OP_SETGLOBAL, +OP_SETUPVAL, +OP_SETTABLE, +OP_NEWTABLE, +OP_SELF, +OP_ADD, +OP_SUB, +OP_MUL, +OP_DIV, +OP_MOD, +OP_POW, +OP_UNM, +OP_NOT, +OP_LEN, +OP_CONCAT, +OP_JMP, +OP_EQ, +OP_LT, +OP_LE, +OP_TEST, +OP_TESTSET, +OP_CALL, +OP_TAILCALL, +OP_RETURN, +OP_FORLOOP, +OP_FORPREP, +OP_TFORLOOP, +OP_SETLIST, +OP_CLOSE, +OP_CLOSURE, +OP_VARARG +}OpCode; +enum OpArgMask{ +OpArgN, +OpArgU, +OpArgR, +OpArgK +}; +typedef enum{ +VVOID, +VNIL, +VTRUE, +VFALSE, +VK, +VKNUM, +VLOCAL, +VUPVAL, +VGLOBAL, +VINDEXED, +VJMP, +VRELOCABLE, +VNONRELOC, +VCALL, +VVARARG +}expkind; +enum RESERVED{ +TK_AND=257,TK_BREAK, +TK_DO,TK_ELSE,TK_ELSEIF,TK_END,TK_FALSE,TK_FOR,TK_FUNCTION, +TK_IF,TK_IN,TK_LOCAL,TK_NIL,TK_NOT,TK_OR,TK_REPEAT, +TK_RETURN,TK_THEN,TK_TRUE,TK_UNTIL,TK_WHILE, +TK_CONCAT,TK_DOTS,TK_EQ,TK_GE,TK_LE,TK_NE,TK_NUMBER, +TK_NAME,TK_STRING,TK_EOS +}; +typedef enum BinOpr{ +OPR_ADD,OPR_SUB,OPR_MUL,OPR_DIV,OPR_MOD,OPR_POW, +OPR_CONCAT, +OPR_NE,OPR_EQ, +OPR_LT,OPR_LE,OPR_GT,OPR_GE, +OPR_AND,OPR_OR, +OPR_NOBINOPR +}BinOpr; +typedef enum UnOpr{OPR_MINUS,OPR_NOT,OPR_LEN,OPR_NOUNOPR}UnOpr; +#define LUA_QL(x)"'"x"'" +#define luai_apicheck(L,o){(void)L;} +#define lua_number2str(s,n)sprintf((s),"%.14g",(n)) +#define lua_str2number(s,p)strtod((s),(p)) +#define luai_numadd(a,b)((a)+(b)) +#define luai_numsub(a,b)((a)-(b)) +#define luai_nummul(a,b)((a)*(b)) +#define luai_numdiv(a,b)((a)/(b)) +#define luai_nummod(a,b)((a)-floor((a)/(b))*(b)) +#define luai_numpow(a,b)(pow(a,b)) +#define luai_numunm(a)(-(a)) +#define luai_numeq(a,b)((a)==(b)) +#define luai_numlt(a,b)((a)<(b)) +#define luai_numle(a,b)((a)<=(b)) +#define luai_numisnan(a)(!luai_numeq((a),(a))) +#define lua_number2int(i,d)((i)=(int)(d)) +#define lua_number2integer(i,d)((i)=(lua_Integer)(d)) +#define LUAI_THROW(L,c)longjmp((c)->b,1) +#define LUAI_TRY(L,c,a)if(setjmp((c)->b)==0){a} +#define lua_pclose(L,file)((void)((void)L,file),0) +#define lua_upvalueindex(i)((-10002)-(i)) +typedef struct lua_State lua_State; +typedef int(*lua_CFunction)(lua_State*L); +typedef const char*(*lua_Reader)(lua_State*L,void*ud,size_t*sz); +typedef void*(*lua_Alloc)(void*ud,void*ptr,size_t osize,size_t nsize); +typedef double lua_Number; +typedef ptrdiff_t lua_Integer; +static void lua_settop(lua_State*L,int idx); +static int lua_type(lua_State*L,int idx); +static const char* lua_tolstring(lua_State*L,int idx,size_t*len); +static size_t lua_objlen(lua_State*L,int idx); +static void lua_pushlstring(lua_State*L,const char*s,size_t l); +static void lua_pushcclosure(lua_State*L,lua_CFunction fn,int n); +static void lua_createtable(lua_State*L,int narr,int nrec); +static void lua_setfield(lua_State*L,int idx,const char*k); +#define lua_pop(L,n)lua_settop(L,-(n)-1) +#define lua_newtable(L)lua_createtable(L,0,0) +#define lua_pushcfunction(L,f)lua_pushcclosure(L,(f),0) +#define lua_strlen(L,i)lua_objlen(L,(i)) +#define lua_isfunction(L,n)(lua_type(L,(n))==6) +#define lua_istable(L,n)(lua_type(L,(n))==5) +#define lua_isnil(L,n)(lua_type(L,(n))==0) +#define lua_isboolean(L,n)(lua_type(L,(n))==1) +#define lua_isnone(L,n)(lua_type(L,(n))==(-1)) +#define lua_isnoneornil(L,n)(lua_type(L,(n))<=0) +#define lua_pushliteral(L,s)lua_pushlstring(L,""s,(sizeof(s)/sizeof(char))-1) +#define lua_setglobal(L,s)lua_setfield(L,(-10002),(s)) +#define lua_tostring(L,i)lua_tolstring(L,(i),NULL) +typedef struct lua_Debug lua_Debug; +typedef void(*lua_Hook)(lua_State*L,lua_Debug*ar); +struct lua_Debug{ +int event; +const char*name; +const char*namewhat; +const char*what; +const char*source; +int currentline; +int nups; +int linedefined; +int lastlinedefined; +char short_src[60]; +int i_ci; +}; +typedef unsigned int lu_int32; +typedef size_t lu_mem; +typedef ptrdiff_t l_mem; +typedef unsigned char lu_byte; +#define IntPoint(p)((unsigned int)(lu_mem)(p)) +typedef union{double u;void*s;long l;}L_Umaxalign; +typedef double l_uacNumber; +#define check_exp(c,e)(e) +#define UNUSED(x)((void)(x)) +#define cast(t,exp)((t)(exp)) +#define cast_byte(i)cast(lu_byte,(i)) +#define cast_num(i)cast(lua_Number,(i)) +#define cast_int(i)cast(int,(i)) +typedef lu_int32 Instruction; +#define condhardstacktests(x)((void)0) +typedef union GCObject GCObject; +typedef struct GCheader{ +GCObject*next;lu_byte tt;lu_byte marked; +}GCheader; +typedef union{ +GCObject*gc; +void*p; +lua_Number n; +int b; +}Value; +typedef struct lua_TValue{ +Value value;int tt; +}TValue; +#define ttisnil(o)(ttype(o)==0) +#define ttisnumber(o)(ttype(o)==3) +#define ttisstring(o)(ttype(o)==4) +#define ttistable(o)(ttype(o)==5) +#define ttisfunction(o)(ttype(o)==6) +#define ttisboolean(o)(ttype(o)==1) +#define ttisuserdata(o)(ttype(o)==7) +#define ttisthread(o)(ttype(o)==8) +#define ttislightuserdata(o)(ttype(o)==2) +#define ttype(o)((o)->tt) +#define gcvalue(o)check_exp(iscollectable(o),(o)->value.gc) +#define pvalue(o)check_exp(ttislightuserdata(o),(o)->value.p) +#define nvalue(o)check_exp(ttisnumber(o),(o)->value.n) +#define rawtsvalue(o)check_exp(ttisstring(o),&(o)->value.gc->ts) +#define tsvalue(o)(&rawtsvalue(o)->tsv) +#define rawuvalue(o)check_exp(ttisuserdata(o),&(o)->value.gc->u) +#define uvalue(o)(&rawuvalue(o)->uv) +#define clvalue(o)check_exp(ttisfunction(o),&(o)->value.gc->cl) +#define hvalue(o)check_exp(ttistable(o),&(o)->value.gc->h) +#define bvalue(o)check_exp(ttisboolean(o),(o)->value.b) +#define thvalue(o)check_exp(ttisthread(o),&(o)->value.gc->th) +#define l_isfalse(o)(ttisnil(o)||(ttisboolean(o)&&bvalue(o)==0)) +#define checkconsistency(obj) +#define checkliveness(g,obj) +#define setnilvalue(obj)((obj)->tt=0) +#define setnvalue(obj,x){TValue*i_o=(obj);i_o->value.n=(x);i_o->tt=3;} +#define setbvalue(obj,x){TValue*i_o=(obj);i_o->value.b=(x);i_o->tt=1;} +#define setsvalue(L,obj,x){TValue*i_o=(obj);i_o->value.gc=cast(GCObject*,(x));i_o->tt=4;checkliveness(G(L),i_o);} +#define setuvalue(L,obj,x){TValue*i_o=(obj);i_o->value.gc=cast(GCObject*,(x));i_o->tt=7;checkliveness(G(L),i_o);} +#define setthvalue(L,obj,x){TValue*i_o=(obj);i_o->value.gc=cast(GCObject*,(x));i_o->tt=8;checkliveness(G(L),i_o);} +#define setclvalue(L,obj,x){TValue*i_o=(obj);i_o->value.gc=cast(GCObject*,(x));i_o->tt=6;checkliveness(G(L),i_o);} +#define sethvalue(L,obj,x){TValue*i_o=(obj);i_o->value.gc=cast(GCObject*,(x));i_o->tt=5;checkliveness(G(L),i_o);} +#define setptvalue(L,obj,x){TValue*i_o=(obj);i_o->value.gc=cast(GCObject*,(x));i_o->tt=(8+1);checkliveness(G(L),i_o);} +#define setobj(L,obj1,obj2){const TValue*o2=(obj2);TValue*o1=(obj1);o1->value=o2->value;o1->tt=o2->tt;checkliveness(G(L),o1);} +#define setttype(obj,tt)(ttype(obj)=(tt)) +#define iscollectable(o)(ttype(o)>=4) +typedef TValue*StkId; +typedef union TString{ +L_Umaxalign dummy; +struct{ +GCObject*next;lu_byte tt;lu_byte marked; +lu_byte reserved; +unsigned int hash; +size_t len; +}tsv; +}TString; +#define getstr(ts)cast(const char*,(ts)+1) +#define svalue(o)getstr(rawtsvalue(o)) +typedef union Udata{ +L_Umaxalign dummy; +struct{ +GCObject*next;lu_byte tt;lu_byte marked; +struct Table*metatable; +struct Table*env; +size_t len; +}uv; +}Udata; +typedef struct Proto{ +GCObject*next;lu_byte tt;lu_byte marked; +TValue*k; +Instruction*code; +struct Proto**p; +int*lineinfo; +struct LocVar*locvars; +TString**upvalues; +TString*source; +int sizeupvalues; +int sizek; +int sizecode; +int sizelineinfo; +int sizep; +int sizelocvars; +int linedefined; +int lastlinedefined; +GCObject*gclist; +lu_byte nups; +lu_byte numparams; +lu_byte is_vararg; +lu_byte maxstacksize; +}Proto; +typedef struct LocVar{ +TString*varname; +int startpc; +int endpc; +}LocVar; +typedef struct UpVal{ +GCObject*next;lu_byte tt;lu_byte marked; +TValue*v; +union{ +TValue value; +struct{ +struct UpVal*prev; +struct UpVal*next; +}l; +}u; +}UpVal; +typedef struct CClosure{ +GCObject*next;lu_byte tt;lu_byte marked;lu_byte isC;lu_byte nupvalues;GCObject*gclist;struct Table*env; +lua_CFunction f; +TValue upvalue[1]; +}CClosure; +typedef struct LClosure{ +GCObject*next;lu_byte tt;lu_byte marked;lu_byte isC;lu_byte nupvalues;GCObject*gclist;struct Table*env; +struct Proto*p; +UpVal*upvals[1]; +}LClosure; +typedef union Closure{ +CClosure c; +LClosure l; +}Closure; +#define iscfunction(o)(ttype(o)==6&&clvalue(o)->c.isC) +typedef union TKey{ +struct{ +Value value;int tt; +struct Node*next; +}nk; +TValue tvk; +}TKey; +typedef struct Node{ +TValue i_val; +TKey i_key; +}Node; +typedef struct Table{ +GCObject*next;lu_byte tt;lu_byte marked; +lu_byte flags; +lu_byte lsizenode; +struct Table*metatable; +TValue*array; +Node*node; +Node*lastfree; +GCObject*gclist; +int sizearray; +}Table; +#define lmod(s,size)(check_exp((size&(size-1))==0,(cast(int,(s)&((size)-1))))) +#define twoto(x)((size_t)1<<(x)) +#define sizenode(t)(twoto((t)->lsizenode)) +static const TValue luaO_nilobject_; +#define ceillog2(x)(luaO_log2((x)-1)+1) +static int luaO_log2(unsigned int x); +#define gfasttm(g,et,e)((et)==NULL?NULL:((et)->flags&(1u<<(e)))?NULL:luaT_gettm(et,e,(g)->tmname[e])) +#define fasttm(l,et,e)gfasttm(G(l),et,e) +static const TValue*luaT_gettm(Table*events,TMS event,TString*ename); +#define luaM_reallocv(L,b,on,n,e)((cast(size_t,(n)+1)<=((size_t)(~(size_t)0)-2)/(e))?luaM_realloc_(L,(b),(on)*(e),(n)*(e)):luaM_toobig(L)) +#define luaM_freemem(L,b,s)luaM_realloc_(L,(b),(s),0) +#define luaM_free(L,b)luaM_realloc_(L,(b),sizeof(*(b)),0) +#define luaM_freearray(L,b,n,t)luaM_reallocv(L,(b),n,0,sizeof(t)) +#define luaM_malloc(L,t)luaM_realloc_(L,NULL,0,(t)) +#define luaM_new(L,t)cast(t*,luaM_malloc(L,sizeof(t))) +#define luaM_newvector(L,n,t)cast(t*,luaM_reallocv(L,NULL,0,n,sizeof(t))) +#define luaM_growvector(L,v,nelems,size,t,limit,e)if((nelems)+1>(size))((v)=cast(t*,luaM_growaux_(L,v,&(size),sizeof(t),limit,e))) +#define luaM_reallocvector(L,v,oldn,n,t)((v)=cast(t*,luaM_reallocv(L,v,oldn,n,sizeof(t)))) +static void*luaM_realloc_(lua_State*L,void*block,size_t oldsize, +size_t size); +static void*luaM_toobig(lua_State*L); +static void*luaM_growaux_(lua_State*L,void*block,int*size, +size_t size_elem,int limit, +const char*errormsg); +typedef struct Zio ZIO; +#define char2int(c)cast(int,cast(unsigned char,(c))) +#define zgetc(z)(((z)->n--)>0?char2int(*(z)->p++):luaZ_fill(z)) +typedef struct Mbuffer{ +char*buffer; +size_t n; +size_t buffsize; +}Mbuffer; +#define luaZ_initbuffer(L,buff)((buff)->buffer=NULL,(buff)->buffsize=0) +#define luaZ_buffer(buff)((buff)->buffer) +#define luaZ_sizebuffer(buff)((buff)->buffsize) +#define luaZ_bufflen(buff)((buff)->n) +#define luaZ_resetbuffer(buff)((buff)->n=0) +#define luaZ_resizebuffer(L,buff,size)(luaM_reallocvector(L,(buff)->buffer,(buff)->buffsize,size,char),(buff)->buffsize=size) +#define luaZ_freebuffer(L,buff)luaZ_resizebuffer(L,buff,0) +struct Zio{ +size_t n; +const char*p; +lua_Reader reader; +void*data; +lua_State*L; +}; +static int luaZ_fill(ZIO*z); +struct lua_longjmp; +#define gt(L)(&L->l_gt) +#define registry(L)(&G(L)->l_registry) +typedef struct stringtable{ +GCObject**hash; +lu_int32 nuse; +int size; +}stringtable; +typedef struct CallInfo{ +StkId base; +StkId func; +StkId top; +const Instruction*savedpc; +int nresults; +int tailcalls; +}CallInfo; +#define curr_func(L)(clvalue(L->ci->func)) +#define ci_func(ci)(clvalue((ci)->func)) +#define f_isLua(ci)(!ci_func(ci)->c.isC) +#define isLua(ci)(ttisfunction((ci)->func)&&f_isLua(ci)) +typedef struct global_State{ +stringtable strt; +lua_Alloc frealloc; +void*ud; +lu_byte currentwhite; +lu_byte gcstate; +int sweepstrgc; +GCObject*rootgc; +GCObject**sweepgc; +GCObject*gray; +GCObject*grayagain; +GCObject*weak; +GCObject*tmudata; +Mbuffer buff; +lu_mem GCthreshold; +lu_mem totalbytes; +lu_mem estimate; +lu_mem gcdept; +int gcpause; +int gcstepmul; +lua_CFunction panic; +TValue l_registry; +struct lua_State*mainthread; +UpVal uvhead; +struct Table*mt[(8+1)]; +TString*tmname[TM_N]; +}global_State; +struct lua_State{ +GCObject*next;lu_byte tt;lu_byte marked; +lu_byte status; +StkId top; +StkId base; +global_State*l_G; +CallInfo*ci; +const Instruction*savedpc; +StkId stack_last; +StkId stack; +CallInfo*end_ci; +CallInfo*base_ci; +int stacksize; +int size_ci; +unsigned short nCcalls; +unsigned short baseCcalls; +lu_byte hookmask; +lu_byte allowhook; +int basehookcount; +int hookcount; +lua_Hook hook; +TValue l_gt; +TValue env; +GCObject*openupval; +GCObject*gclist; +struct lua_longjmp*errorJmp; +ptrdiff_t errfunc; +}; +#define G(L)(L->l_G) +union GCObject{ +GCheader gch; +union TString ts; +union Udata u; +union Closure cl; +struct Table h; +struct Proto p; +struct UpVal uv; +struct lua_State th; +}; +#define rawgco2ts(o)check_exp((o)->gch.tt==4,&((o)->ts)) +#define gco2ts(o)(&rawgco2ts(o)->tsv) +#define rawgco2u(o)check_exp((o)->gch.tt==7,&((o)->u)) +#define gco2u(o)(&rawgco2u(o)->uv) +#define gco2cl(o)check_exp((o)->gch.tt==6,&((o)->cl)) +#define gco2h(o)check_exp((o)->gch.tt==5,&((o)->h)) +#define gco2p(o)check_exp((o)->gch.tt==(8+1),&((o)->p)) +#define gco2uv(o)check_exp((o)->gch.tt==(8+2),&((o)->uv)) +#define ngcotouv(o)check_exp((o)==NULL||(o)->gch.tt==(8+2),&((o)->uv)) +#define gco2th(o)check_exp((o)->gch.tt==8,&((o)->th)) +#define obj2gco(v)(cast(GCObject*,(v))) +static void luaE_freethread(lua_State*L,lua_State*L1); +#define pcRel(pc,p)(cast(int,(pc)-(p)->code)-1) +#define getline_(f,pc)(((f)->lineinfo)?(f)->lineinfo[pc]:0) +#define resethookcount(L)(L->hookcount=L->basehookcount) +static void luaG_typeerror(lua_State*L,const TValue*o, +const char*opname); +static void luaG_runerror(lua_State*L,const char*fmt,...); +#define luaD_checkstack(L,n)if((char*)L->stack_last-(char*)L->top<=(n)*(int)sizeof(TValue))luaD_growstack(L,n);else condhardstacktests(luaD_reallocstack(L,L->stacksize-5-1)); +#define incr_top(L){luaD_checkstack(L,1);L->top++;} +#define savestack(L,p)((char*)(p)-(char*)L->stack) +#define restorestack(L,n)((TValue*)((char*)L->stack+(n))) +#define saveci(L,p)((char*)(p)-(char*)L->base_ci) +#define restoreci(L,n)((CallInfo*)((char*)L->base_ci+(n))) +typedef void(*Pfunc)(lua_State*L,void*ud); +static int luaD_poscall(lua_State*L,StkId firstResult); +static void luaD_reallocCI(lua_State*L,int newsize); +static void luaD_reallocstack(lua_State*L,int newsize); +static void luaD_growstack(lua_State*L,int n); +static void luaD_throw(lua_State*L,int errcode); +static void*luaM_growaux_(lua_State*L,void*block,int*size,size_t size_elems, +int limit,const char*errormsg){ +void*newblock; +int newsize; +if(*size>=limit/2){ +if(*size>=limit) +luaG_runerror(L,errormsg); +newsize=limit; +} +else{ +newsize=(*size)*2; +if(newsize<4) +newsize=4; +} +newblock=luaM_reallocv(L,block,*size,newsize,size_elems); +*size=newsize; +return newblock; +} +static void*luaM_toobig(lua_State*L){ +luaG_runerror(L,"memory allocation error: block too big"); +return NULL; +} +static void*luaM_realloc_(lua_State*L,void*block,size_t osize,size_t nsize){ +global_State*g=G(L); +block=(*g->frealloc)(g->ud,block,osize,nsize); +if(block==NULL&&nsize>0) +luaD_throw(L,4); +g->totalbytes=(g->totalbytes-osize)+nsize; +return block; +} +#define resetbits(x,m)((x)&=cast(lu_byte,~(m))) +#define setbits(x,m)((x)|=(m)) +#define testbits(x,m)((x)&(m)) +#define bitmask(b)(1<<(b)) +#define bit2mask(b1,b2)(bitmask(b1)|bitmask(b2)) +#define l_setbit(x,b)setbits(x,bitmask(b)) +#define resetbit(x,b)resetbits(x,bitmask(b)) +#define testbit(x,b)testbits(x,bitmask(b)) +#define set2bits(x,b1,b2)setbits(x,(bit2mask(b1,b2))) +#define reset2bits(x,b1,b2)resetbits(x,(bit2mask(b1,b2))) +#define test2bits(x,b1,b2)testbits(x,(bit2mask(b1,b2))) +#define iswhite(x)test2bits((x)->gch.marked,0,1) +#define isblack(x)testbit((x)->gch.marked,2) +#define isgray(x)(!isblack(x)&&!iswhite(x)) +#define otherwhite(g)(g->currentwhite^bit2mask(0,1)) +#define isdead(g,v)((v)->gch.marked&otherwhite(g)&bit2mask(0,1)) +#define changewhite(x)((x)->gch.marked^=bit2mask(0,1)) +#define gray2black(x)l_setbit((x)->gch.marked,2) +#define valiswhite(x)(iscollectable(x)&&iswhite(gcvalue(x))) +#define luaC_white(g)cast(lu_byte,(g)->currentwhite&bit2mask(0,1)) +#define luaC_checkGC(L){condhardstacktests(luaD_reallocstack(L,L->stacksize-5-1));if(G(L)->totalbytes>=G(L)->GCthreshold)luaC_step(L);} +#define luaC_barrier(L,p,v){if(valiswhite(v)&&isblack(obj2gco(p)))luaC_barrierf(L,obj2gco(p),gcvalue(v));} +#define luaC_barriert(L,t,v){if(valiswhite(v)&&isblack(obj2gco(t)))luaC_barrierback(L,t);} +#define luaC_objbarrier(L,p,o){if(iswhite(obj2gco(o))&&isblack(obj2gco(p)))luaC_barrierf(L,obj2gco(p),obj2gco(o));} +#define luaC_objbarriert(L,t,o){if(iswhite(obj2gco(o))&&isblack(obj2gco(t)))luaC_barrierback(L,t);} +static void luaC_step(lua_State*L); +static void luaC_link(lua_State*L,GCObject*o,lu_byte tt); +static void luaC_linkupval(lua_State*L,UpVal*uv); +static void luaC_barrierf(lua_State*L,GCObject*o,GCObject*v); +static void luaC_barrierback(lua_State*L,Table*t); +#define sizestring(s)(sizeof(union TString)+((s)->len+1)*sizeof(char)) +#define sizeudata(u)(sizeof(union Udata)+(u)->len) +#define luaS_new(L,s)(luaS_newlstr(L,s,strlen(s))) +#define luaS_newliteral(L,s)(luaS_newlstr(L,""s,(sizeof(s)/sizeof(char))-1)) +#define luaS_fix(s)l_setbit((s)->tsv.marked,5) +static TString*luaS_newlstr(lua_State*L,const char*str,size_t l); +#define tostring(L,o)((ttype(o)==4)||(luaV_tostring(L,o))) +#define tonumber(o,n)(ttype(o)==3||(((o)=luaV_tonumber(o,n))!=NULL)) +#define equalobj(L,o1,o2)(ttype(o1)==ttype(o2)&&luaV_equalval(L,o1,o2)) +static int luaV_equalval(lua_State*L,const TValue*t1,const TValue*t2); +static const TValue*luaV_tonumber(const TValue*obj,TValue*n); +static int luaV_tostring(lua_State*L,StkId obj); +static void luaV_execute(lua_State*L,int nexeccalls); +static void luaV_concat(lua_State*L,int total,int last); +static const TValue luaO_nilobject_={{NULL},0}; +static int luaO_int2fb(unsigned int x){ +int e=0; +while(x>=16){ +x=(x+1)>>1; +e++; +} +if(x<8)return x; +else return((e+1)<<3)|(cast_int(x)-8); +} +static int luaO_fb2int(int x){ +int e=(x>>3)&31; +if(e==0)return x; +else return((x&7)+8)<<(e-1); +} +static int luaO_log2(unsigned int x){ +static const lu_byte log_2[256]={ +0,1,2,2,3,3,3,3,4,4,4,4,4,4,4,4,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5, +6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6, +7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, +7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, +8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, +8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, +8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, +8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8 +}; +int l=-1; +while(x>=256){l+=8;x>>=8;} +return l+log_2[x]; +} +static int luaO_rawequalObj(const TValue*t1,const TValue*t2){ +if(ttype(t1)!=ttype(t2))return 0; +else switch(ttype(t1)){ +case 0: +return 1; +case 3: +return luai_numeq(nvalue(t1),nvalue(t2)); +case 1: +return bvalue(t1)==bvalue(t2); +case 2: +return pvalue(t1)==pvalue(t2); +default: +return gcvalue(t1)==gcvalue(t2); +} +} +static int luaO_str2d(const char*s,lua_Number*result){ +char*endptr; +*result=lua_str2number(s,&endptr); +if(endptr==s)return 0; +if(*endptr=='x'||*endptr=='X') +*result=cast_num(strtoul(s,&endptr,16)); +if(*endptr=='\0')return 1; +while(isspace(cast(unsigned char,*endptr)))endptr++; +if(*endptr!='\0')return 0; +return 1; +} +static void pushstr(lua_State*L,const char*str){ +setsvalue(L,L->top,luaS_new(L,str)); +incr_top(L); +} +static const char*luaO_pushvfstring(lua_State*L,const char*fmt,va_list argp){ +int n=1; +pushstr(L,""); +for(;;){ +const char*e=strchr(fmt,'%'); +if(e==NULL)break; +setsvalue(L,L->top,luaS_newlstr(L,fmt,e-fmt)); +incr_top(L); +switch(*(e+1)){ +case's':{ +const char*s=va_arg(argp,char*); +if(s==NULL)s="(null)"; +pushstr(L,s); +break; +} +case'c':{ +char buff[2]; +buff[0]=cast(char,va_arg(argp,int)); +buff[1]='\0'; +pushstr(L,buff); +break; +} +case'd':{ +setnvalue(L->top,cast_num(va_arg(argp,int))); +incr_top(L); +break; +} +case'f':{ +setnvalue(L->top,cast_num(va_arg(argp,l_uacNumber))); +incr_top(L); +break; +} +case'p':{ +char buff[4*sizeof(void*)+8]; +sprintf(buff,"%p",va_arg(argp,void*)); +pushstr(L,buff); +break; +} +case'%':{ +pushstr(L,"%"); +break; +} +default:{ +char buff[3]; +buff[0]='%'; +buff[1]=*(e+1); +buff[2]='\0'; +pushstr(L,buff); +break; +} +} +n+=2; +fmt=e+2; +} +pushstr(L,fmt); +luaV_concat(L,n+1,cast_int(L->top-L->base)-1); +L->top-=n; +return svalue(L->top-1); +} +static const char*luaO_pushfstring(lua_State*L,const char*fmt,...){ +const char*msg; +va_list argp; +va_start(argp,fmt); +msg=luaO_pushvfstring(L,fmt,argp); +va_end(argp); +return msg; +} +static void luaO_chunkid(char*out,const char*source,size_t bufflen){ +if(*source=='='){ +strncpy(out,source+1,bufflen); +out[bufflen-1]='\0'; +} +else{ +if(*source=='@'){ +size_t l; +source++; +bufflen-=sizeof(" '...' "); +l=strlen(source); +strcpy(out,""); +if(l>bufflen){ +source+=(l-bufflen); +strcat(out,"..."); +} +strcat(out,source); +} +else{ +size_t len=strcspn(source,"\n\r"); +bufflen-=sizeof(" [string \"...\"] "); +if(len>bufflen)len=bufflen; +strcpy(out,"[string \""); +if(source[len]!='\0'){ +strncat(out,source,len); +strcat(out,"..."); +} +else +strcat(out,source); +strcat(out,"\"]"); +} +} +} +#define gnode(t,i)(&(t)->node[i]) +#define gkey(n)(&(n)->i_key.nk) +#define gval(n)(&(n)->i_val) +#define gnext(n)((n)->i_key.nk.next) +#define key2tval(n)(&(n)->i_key.tvk) +static TValue*luaH_setnum(lua_State*L,Table*t,int key); +static const TValue*luaH_getstr(Table*t,TString*key); +static TValue*luaH_set(lua_State*L,Table*t,const TValue*key); +static const char*const luaT_typenames[]={ +"nil","boolean","userdata","number", +"string","table","function","userdata","thread", +"proto","upval" +}; +static void luaT_init(lua_State*L){ +static const char*const luaT_eventname[]={ +"__index","__newindex", +"__gc","__mode","__eq", +"__add","__sub","__mul","__div","__mod", +"__pow","__unm","__len","__lt","__le", +"__concat","__call" +}; +int i; +for(i=0;itmname[i]=luaS_new(L,luaT_eventname[i]); +luaS_fix(G(L)->tmname[i]); +} +} +static const TValue*luaT_gettm(Table*events,TMS event,TString*ename){ +const TValue*tm=luaH_getstr(events,ename); +if(ttisnil(tm)){ +events->flags|=cast_byte(1u<metatable; +break; +case 7: +mt=uvalue(o)->metatable; +break; +default: +mt=G(L)->mt[ttype(o)]; +} +return(mt?luaH_getstr(mt,G(L)->tmname[event]):(&luaO_nilobject_)); +} +#define sizeCclosure(n)(cast(int,sizeof(CClosure))+cast(int,sizeof(TValue)*((n)-1))) +#define sizeLclosure(n)(cast(int,sizeof(LClosure))+cast(int,sizeof(TValue*)*((n)-1))) +static Closure*luaF_newCclosure(lua_State*L,int nelems,Table*e){ +Closure*c=cast(Closure*,luaM_malloc(L,sizeCclosure(nelems))); +luaC_link(L,obj2gco(c),6); +c->c.isC=1; +c->c.env=e; +c->c.nupvalues=cast_byte(nelems); +return c; +} +static Closure*luaF_newLclosure(lua_State*L,int nelems,Table*e){ +Closure*c=cast(Closure*,luaM_malloc(L,sizeLclosure(nelems))); +luaC_link(L,obj2gco(c),6); +c->l.isC=0; +c->l.env=e; +c->l.nupvalues=cast_byte(nelems); +while(nelems--)c->l.upvals[nelems]=NULL; +return c; +} +static UpVal*luaF_newupval(lua_State*L){ +UpVal*uv=luaM_new(L,UpVal); +luaC_link(L,obj2gco(uv),(8+2)); +uv->v=&uv->u.value; +setnilvalue(uv->v); +return uv; +} +static UpVal*luaF_findupval(lua_State*L,StkId level){ +global_State*g=G(L); +GCObject**pp=&L->openupval; +UpVal*p; +UpVal*uv; +while(*pp!=NULL&&(p=ngcotouv(*pp))->v>=level){ +if(p->v==level){ +if(isdead(g,obj2gco(p))) +changewhite(obj2gco(p)); +return p; +} +pp=&p->next; +} +uv=luaM_new(L,UpVal); +uv->tt=(8+2); +uv->marked=luaC_white(g); +uv->v=level; +uv->next=*pp; +*pp=obj2gco(uv); +uv->u.l.prev=&g->uvhead; +uv->u.l.next=g->uvhead.u.l.next; +uv->u.l.next->u.l.prev=uv; +g->uvhead.u.l.next=uv; +return uv; +} +static void unlinkupval(UpVal*uv){ +uv->u.l.next->u.l.prev=uv->u.l.prev; +uv->u.l.prev->u.l.next=uv->u.l.next; +} +static void luaF_freeupval(lua_State*L,UpVal*uv){ +if(uv->v!=&uv->u.value) +unlinkupval(uv); +luaM_free(L,uv); +} +static void luaF_close(lua_State*L,StkId level){ +UpVal*uv; +global_State*g=G(L); +while(L->openupval!=NULL&&(uv=ngcotouv(L->openupval))->v>=level){ +GCObject*o=obj2gco(uv); +L->openupval=uv->next; +if(isdead(g,o)) +luaF_freeupval(L,uv); +else{ +unlinkupval(uv); +setobj(L,&uv->u.value,uv->v); +uv->v=&uv->u.value; +luaC_linkupval(L,uv); +} +} +} +static Proto*luaF_newproto(lua_State*L){ +Proto*f=luaM_new(L,Proto); +luaC_link(L,obj2gco(f),(8+1)); +f->k=NULL; +f->sizek=0; +f->p=NULL; +f->sizep=0; +f->code=NULL; +f->sizecode=0; +f->sizelineinfo=0; +f->sizeupvalues=0; +f->nups=0; +f->upvalues=NULL; +f->numparams=0; +f->is_vararg=0; +f->maxstacksize=0; +f->lineinfo=NULL; +f->sizelocvars=0; +f->locvars=NULL; +f->linedefined=0; +f->lastlinedefined=0; +f->source=NULL; +return f; +} +static void luaF_freeproto(lua_State*L,Proto*f){ +luaM_freearray(L,f->code,f->sizecode,Instruction); +luaM_freearray(L,f->p,f->sizep,Proto*); +luaM_freearray(L,f->k,f->sizek,TValue); +luaM_freearray(L,f->lineinfo,f->sizelineinfo,int); +luaM_freearray(L,f->locvars,f->sizelocvars,struct LocVar); +luaM_freearray(L,f->upvalues,f->sizeupvalues,TString*); +luaM_free(L,f); +} +static void luaF_freeclosure(lua_State*L,Closure*c){ +int size=(c->c.isC)?sizeCclosure(c->c.nupvalues): +sizeLclosure(c->l.nupvalues); +luaM_freemem(L,c,size); +} +#define MASK1(n,p)((~((~(Instruction)0)<>0)&MASK1(6,0))) +#define SET_OPCODE(i,o)((i)=(((i)&MASK0(6,0))|((cast(Instruction,o)<<0)&MASK1(6,0)))) +#define GETARG_A(i)(cast(int,((i)>>(0+6))&MASK1(8,0))) +#define SETARG_A(i,u)((i)=(((i)&MASK0(8,(0+6)))|((cast(Instruction,u)<<(0+6))&MASK1(8,(0+6))))) +#define GETARG_B(i)(cast(int,((i)>>(((0+6)+8)+9))&MASK1(9,0))) +#define SETARG_B(i,b)((i)=(((i)&MASK0(9,(((0+6)+8)+9)))|((cast(Instruction,b)<<(((0+6)+8)+9))&MASK1(9,(((0+6)+8)+9))))) +#define GETARG_C(i)(cast(int,((i)>>((0+6)+8))&MASK1(9,0))) +#define SETARG_C(i,b)((i)=(((i)&MASK0(9,((0+6)+8)))|((cast(Instruction,b)<<((0+6)+8))&MASK1(9,((0+6)+8))))) +#define GETARG_Bx(i)(cast(int,((i)>>((0+6)+8))&MASK1((9+9),0))) +#define SETARG_Bx(i,b)((i)=(((i)&MASK0((9+9),((0+6)+8)))|((cast(Instruction,b)<<((0+6)+8))&MASK1((9+9),((0+6)+8))))) +#define GETARG_sBx(i)(GETARG_Bx(i)-(((1<<(9+9))-1)>>1)) +#define SETARG_sBx(i,b)SETARG_Bx((i),cast(unsigned int,(b)+(((1<<(9+9))-1)>>1))) +#define CREATE_ABC(o,a,b,c)((cast(Instruction,o)<<0)|(cast(Instruction,a)<<(0+6))|(cast(Instruction,b)<<(((0+6)+8)+9))|(cast(Instruction,c)<<((0+6)+8))) +#define CREATE_ABx(o,a,bc)((cast(Instruction,o)<<0)|(cast(Instruction,a)<<(0+6))|(cast(Instruction,bc)<<((0+6)+8))) +#define ISK(x)((x)&(1<<(9-1))) +#define INDEXK(r)((int)(r)&~(1<<(9-1))) +#define RKASK(x)((x)|(1<<(9-1))) +static const lu_byte luaP_opmodes[(cast(int,OP_VARARG)+1)]; +#define getBMode(m)(cast(enum OpArgMask,(luaP_opmodes[m]>>4)&3)) +#define getCMode(m)(cast(enum OpArgMask,(luaP_opmodes[m]>>2)&3)) +#define testTMode(m)(luaP_opmodes[m]&(1<<7)) +typedef struct expdesc{ +expkind k; +union{ +struct{int info,aux;}s; +lua_Number nval; +}u; +int t; +int f; +}expdesc; +typedef struct upvaldesc{ +lu_byte k; +lu_byte info; +}upvaldesc; +struct BlockCnt; +typedef struct FuncState{ +Proto*f; +Table*h; +struct FuncState*prev; +struct LexState*ls; +struct lua_State*L; +struct BlockCnt*bl; +int pc; +int lasttarget; +int jpc; +int freereg; +int nk; +int np; +short nlocvars; +lu_byte nactvar; +upvaldesc upvalues[60]; +unsigned short actvar[200]; +}FuncState; +static Proto*luaY_parser(lua_State*L,ZIO*z,Mbuffer*buff, +const char*name); +struct lua_longjmp{ +struct lua_longjmp*previous; +jmp_buf b; +volatile int status; +}; +static void luaD_seterrorobj(lua_State*L,int errcode,StkId oldtop){ +switch(errcode){ +case 4:{ +setsvalue(L,oldtop,luaS_newliteral(L,"not enough memory")); +break; +} +case 5:{ +setsvalue(L,oldtop,luaS_newliteral(L,"error in error handling")); +break; +} +case 3: +case 2:{ +setobj(L,oldtop,L->top-1); +break; +} +} +L->top=oldtop+1; +} +static void restore_stack_limit(lua_State*L){ +if(L->size_ci>20000){ +int inuse=cast_int(L->ci-L->base_ci); +if(inuse+1<20000) +luaD_reallocCI(L,20000); +} +} +static void resetstack(lua_State*L,int status){ +L->ci=L->base_ci; +L->base=L->ci->base; +luaF_close(L,L->base); +luaD_seterrorobj(L,status,L->base); +L->nCcalls=L->baseCcalls; +L->allowhook=1; +restore_stack_limit(L); +L->errfunc=0; +L->errorJmp=NULL; +} +static void luaD_throw(lua_State*L,int errcode){ +if(L->errorJmp){ +L->errorJmp->status=errcode; +LUAI_THROW(L,L->errorJmp); +} +else{ +L->status=cast_byte(errcode); +if(G(L)->panic){ +resetstack(L,errcode); +G(L)->panic(L); +} +exit(EXIT_FAILURE); +} +} +static int luaD_rawrunprotected(lua_State*L,Pfunc f,void*ud){ +struct lua_longjmp lj; +lj.status=0; +lj.previous=L->errorJmp; +L->errorJmp=&lj; +LUAI_TRY(L,&lj, +(*f)(L,ud); +); +L->errorJmp=lj.previous; +return lj.status; +} +static void correctstack(lua_State*L,TValue*oldstack){ +CallInfo*ci; +GCObject*up; +L->top=(L->top-oldstack)+L->stack; +for(up=L->openupval;up!=NULL;up=up->gch.next) +gco2uv(up)->v=(gco2uv(up)->v-oldstack)+L->stack; +for(ci=L->base_ci;ci<=L->ci;ci++){ +ci->top=(ci->top-oldstack)+L->stack; +ci->base=(ci->base-oldstack)+L->stack; +ci->func=(ci->func-oldstack)+L->stack; +} +L->base=(L->base-oldstack)+L->stack; +} +static void luaD_reallocstack(lua_State*L,int newsize){ +TValue*oldstack=L->stack; +int realsize=newsize+1+5; +luaM_reallocvector(L,L->stack,L->stacksize,realsize,TValue); +L->stacksize=realsize; +L->stack_last=L->stack+newsize; +correctstack(L,oldstack); +} +static void luaD_reallocCI(lua_State*L,int newsize){ +CallInfo*oldci=L->base_ci; +luaM_reallocvector(L,L->base_ci,L->size_ci,newsize,CallInfo); +L->size_ci=newsize; +L->ci=(L->ci-oldci)+L->base_ci; +L->end_ci=L->base_ci+L->size_ci-1; +} +static void luaD_growstack(lua_State*L,int n){ +if(n<=L->stacksize) +luaD_reallocstack(L,2*L->stacksize); +else +luaD_reallocstack(L,L->stacksize+n); +} +static CallInfo*growCI(lua_State*L){ +if(L->size_ci>20000) +luaD_throw(L,5); +else{ +luaD_reallocCI(L,2*L->size_ci); +if(L->size_ci>20000) +luaG_runerror(L,"stack overflow"); +} +return++L->ci; +} +static StkId adjust_varargs(lua_State*L,Proto*p,int actual){ +int i; +int nfixargs=p->numparams; +Table*htab=NULL; +StkId base,fixed; +for(;actualtop++); +fixed=L->top-actual; +base=L->top; +for(i=0;itop++,fixed+i); +setnilvalue(fixed+i); +} +if(htab){ +sethvalue(L,L->top++,htab); +} +return base; +} +static StkId tryfuncTM(lua_State*L,StkId func){ +const TValue*tm=luaT_gettmbyobj(L,func,TM_CALL); +StkId p; +ptrdiff_t funcr=savestack(L,func); +if(!ttisfunction(tm)) +luaG_typeerror(L,func,"call"); +for(p=L->top;p>func;p--)setobj(L,p,p-1); +incr_top(L); +func=restorestack(L,funcr); +setobj(L,func,tm); +return func; +} +#define inc_ci(L)((L->ci==L->end_ci)?growCI(L):(condhardstacktests(luaD_reallocCI(L,L->size_ci)),++L->ci)) +static int luaD_precall(lua_State*L,StkId func,int nresults){ +LClosure*cl; +ptrdiff_t funcr; +if(!ttisfunction(func)) +func=tryfuncTM(L,func); +funcr=savestack(L,func); +cl=&clvalue(func)->l; +L->ci->savedpc=L->savedpc; +if(!cl->isC){ +CallInfo*ci; +StkId st,base; +Proto*p=cl->p; +luaD_checkstack(L,p->maxstacksize); +func=restorestack(L,funcr); +if(!p->is_vararg){ +base=func+1; +if(L->top>base+p->numparams) +L->top=base+p->numparams; +} +else{ +int nargs=cast_int(L->top-func)-1; +base=adjust_varargs(L,p,nargs); +func=restorestack(L,funcr); +} +ci=inc_ci(L); +ci->func=func; +L->base=ci->base=base; +ci->top=L->base+p->maxstacksize; +L->savedpc=p->code; +ci->tailcalls=0; +ci->nresults=nresults; +for(st=L->top;sttop;st++) +setnilvalue(st); +L->top=ci->top; +return 0; +} +else{ +CallInfo*ci; +int n; +luaD_checkstack(L,20); +ci=inc_ci(L); +ci->func=restorestack(L,funcr); +L->base=ci->base=ci->func+1; +ci->top=L->top+20; +ci->nresults=nresults; +n=(*curr_func(L)->c.f)(L); +if(n<0) +return 2; +else{ +luaD_poscall(L,L->top-n); +return 1; +} +} +} +static int luaD_poscall(lua_State*L,StkId firstResult){ +StkId res; +int wanted,i; +CallInfo*ci; +ci=L->ci--; +res=ci->func; +wanted=ci->nresults; +L->base=(ci-1)->base; +L->savedpc=(ci-1)->savedpc; +for(i=wanted;i!=0&&firstResulttop;i--) +setobj(L,res++,firstResult++); +while(i-->0) +setnilvalue(res++); +L->top=res; +return(wanted-(-1)); +} +static void luaD_call(lua_State*L,StkId func,int nResults){ +if(++L->nCcalls>=200){ +if(L->nCcalls==200) +luaG_runerror(L,"C stack overflow"); +else if(L->nCcalls>=(200+(200>>3))) +luaD_throw(L,5); +} +if(luaD_precall(L,func,nResults)==0) +luaV_execute(L,1); +L->nCcalls--; +luaC_checkGC(L); +} +static int luaD_pcall(lua_State*L,Pfunc func,void*u, +ptrdiff_t old_top,ptrdiff_t ef){ +int status; +unsigned short oldnCcalls=L->nCcalls; +ptrdiff_t old_ci=saveci(L,L->ci); +lu_byte old_allowhooks=L->allowhook; +ptrdiff_t old_errfunc=L->errfunc; +L->errfunc=ef; +status=luaD_rawrunprotected(L,func,u); +if(status!=0){ +StkId oldtop=restorestack(L,old_top); +luaF_close(L,oldtop); +luaD_seterrorobj(L,status,oldtop); +L->nCcalls=oldnCcalls; +L->ci=restoreci(L,old_ci); +L->base=L->ci->base; +L->savedpc=L->ci->savedpc; +L->allowhook=old_allowhooks; +restore_stack_limit(L); +} +L->errfunc=old_errfunc; +return status; +} +struct SParser{ +ZIO*z; +Mbuffer buff; +const char*name; +}; +static void f_parser(lua_State*L,void*ud){ +int i; +Proto*tf; +Closure*cl; +struct SParser*p=cast(struct SParser*,ud); +luaC_checkGC(L); +tf=luaY_parser(L,p->z, +&p->buff,p->name); +cl=luaF_newLclosure(L,tf->nups,hvalue(gt(L))); +cl->l.p=tf; +for(i=0;inups;i++) +cl->l.upvals[i]=luaF_newupval(L); +setclvalue(L,L->top,cl); +incr_top(L); +} +static int luaD_protectedparser(lua_State*L,ZIO*z,const char*name){ +struct SParser p; +int status; +p.z=z;p.name=name; +luaZ_initbuffer(L,&p.buff); +status=luaD_pcall(L,f_parser,&p,savestack(L,L->top),L->errfunc); +luaZ_freebuffer(L,&p.buff); +return status; +} +static void luaS_resize(lua_State*L,int newsize){ +GCObject**newhash; +stringtable*tb; +int i; +if(G(L)->gcstate==2) +return; +newhash=luaM_newvector(L,newsize,GCObject*); +tb=&G(L)->strt; +for(i=0;isize;i++){ +GCObject*p=tb->hash[i]; +while(p){ +GCObject*next=p->gch.next; +unsigned int h=gco2ts(p)->hash; +int h1=lmod(h,newsize); +p->gch.next=newhash[h1]; +newhash[h1]=p; +p=next; +} +} +luaM_freearray(L,tb->hash,tb->size,TString*); +tb->size=newsize; +tb->hash=newhash; +} +static TString*newlstr(lua_State*L,const char*str,size_t l, +unsigned int h){ +TString*ts; +stringtable*tb; +if(l+1>(((size_t)(~(size_t)0)-2)-sizeof(TString))/sizeof(char)) +luaM_toobig(L); +ts=cast(TString*,luaM_malloc(L,(l+1)*sizeof(char)+sizeof(TString))); +ts->tsv.len=l; +ts->tsv.hash=h; +ts->tsv.marked=luaC_white(G(L)); +ts->tsv.tt=4; +ts->tsv.reserved=0; +memcpy(ts+1,str,l*sizeof(char)); +((char*)(ts+1))[l]='\0'; +tb=&G(L)->strt; +h=lmod(h,tb->size); +ts->tsv.next=tb->hash[h]; +tb->hash[h]=obj2gco(ts); +tb->nuse++; +if(tb->nuse>cast(lu_int32,tb->size)&&tb->size<=(INT_MAX-2)/2) +luaS_resize(L,tb->size*2); +return ts; +} +static TString*luaS_newlstr(lua_State*L,const char*str,size_t l){ +GCObject*o; +unsigned int h=cast(unsigned int,l); +size_t step=(l>>5)+1; +size_t l1; +for(l1=l;l1>=step;l1-=step) +h=h^((h<<5)+(h>>2)+cast(unsigned char,str[l1-1])); +for(o=G(L)->strt.hash[lmod(h,G(L)->strt.size)]; +o!=NULL; +o=o->gch.next){ +TString*ts=rawgco2ts(o); +if(ts->tsv.len==l&&(memcmp(str,getstr(ts),l)==0)){ +if(isdead(G(L),o))changewhite(o); +return ts; +} +} +return newlstr(L,str,l,h); +} +static Udata*luaS_newudata(lua_State*L,size_t s,Table*e){ +Udata*u; +if(s>((size_t)(~(size_t)0)-2)-sizeof(Udata)) +luaM_toobig(L); +u=cast(Udata*,luaM_malloc(L,s+sizeof(Udata))); +u->uv.marked=luaC_white(G(L)); +u->uv.tt=7; +u->uv.len=s; +u->uv.metatable=NULL; +u->uv.env=e; +u->uv.next=G(L)->mainthread->next; +G(L)->mainthread->next=obj2gco(u); +return u; +} +#define hashpow2(t,n)(gnode(t,lmod((n),sizenode(t)))) +#define hashstr(t,str)hashpow2(t,(str)->tsv.hash) +#define hashboolean(t,p)hashpow2(t,p) +#define hashmod(t,n)(gnode(t,((n)%((sizenode(t)-1)|1)))) +#define hashpointer(t,p)hashmod(t,IntPoint(p)) +static const Node dummynode_={ +{{NULL},0}, +{{{NULL},0,NULL}} +}; +static Node*hashnum(const Table*t,lua_Number n){ +unsigned int a[cast_int(sizeof(lua_Number)/sizeof(int))]; +int i; +if(luai_numeq(n,0)) +return gnode(t,0); +memcpy(a,&n,sizeof(a)); +for(i=1;isizearray) +return i-1; +else{ +Node*n=mainposition(t,key); +do{ +if(luaO_rawequalObj(key2tval(n),key)|| +(ttype(gkey(n))==(8+3)&&iscollectable(key)&& +gcvalue(gkey(n))==gcvalue(key))){ +i=cast_int(n-gnode(t,0)); +return i+t->sizearray; +} +else n=gnext(n); +}while(n); +luaG_runerror(L,"invalid key to "LUA_QL("next")); +return 0; +} +} +static int luaH_next(lua_State*L,Table*t,StkId key){ +int i=findindex(L,t,key); +for(i++;isizearray;i++){ +if(!ttisnil(&t->array[i])){ +setnvalue(key,cast_num(i+1)); +setobj(L,key+1,&t->array[i]); +return 1; +} +} +for(i-=t->sizearray;i<(int)sizenode(t);i++){ +if(!ttisnil(gval(gnode(t,i)))){ +setobj(L,key,key2tval(gnode(t,i))); +setobj(L,key+1,gval(gnode(t,i))); +return 1; +} +} +return 0; +} +static int computesizes(int nums[],int*narray){ +int i; +int twotoi; +int a=0; +int na=0; +int n=0; +for(i=0,twotoi=1;twotoi/2<*narray;i++,twotoi*=2){ +if(nums[i]>0){ +a+=nums[i]; +if(a>twotoi/2){ +n=twotoi; +na=a; +} +} +if(a==*narray)break; +} +*narray=n; +return na; +} +static int countint(const TValue*key,int*nums){ +int k=arrayindex(key); +if(0t->sizearray){ +lim=t->sizearray; +if(i>lim) +break; +} +for(;i<=lim;i++){ +if(!ttisnil(&t->array[i-1])) +lc++; +} +nums[lg]+=lc; +ause+=lc; +} +return ause; +} +static int numusehash(const Table*t,int*nums,int*pnasize){ +int totaluse=0; +int ause=0; +int i=sizenode(t); +while(i--){ +Node*n=&t->node[i]; +if(!ttisnil(gval(n))){ +ause+=countint(key2tval(n),nums); +totaluse++; +} +} +*pnasize+=ause; +return totaluse; +} +static void setarrayvector(lua_State*L,Table*t,int size){ +int i; +luaM_reallocvector(L,t->array,t->sizearray,size,TValue); +for(i=t->sizearray;iarray[i]); +t->sizearray=size; +} +static void setnodevector(lua_State*L,Table*t,int size){ +int lsize; +if(size==0){ +t->node=cast(Node*,(&dummynode_)); +lsize=0; +} +else{ +int i; +lsize=ceillog2(size); +if(lsize>(32-2)) +luaG_runerror(L,"table overflow"); +size=twoto(lsize); +t->node=luaM_newvector(L,size,Node); +for(i=0;ilsizenode=cast_byte(lsize); +t->lastfree=gnode(t,size); +} +static void resize(lua_State*L,Table*t,int nasize,int nhsize){ +int i; +int oldasize=t->sizearray; +int oldhsize=t->lsizenode; +Node*nold=t->node; +if(nasize>oldasize) +setarrayvector(L,t,nasize); +setnodevector(L,t,nhsize); +if(nasizesizearray=nasize; +for(i=nasize;iarray[i])) +setobj(L,luaH_setnum(L,t,i+1),&t->array[i]); +} +luaM_reallocvector(L,t->array,oldasize,nasize,TValue); +} +for(i=twoto(oldhsize)-1;i>=0;i--){ +Node*old=nold+i; +if(!ttisnil(gval(old))) +setobj(L,luaH_set(L,t,key2tval(old)),gval(old)); +} +if(nold!=(&dummynode_)) +luaM_freearray(L,nold,twoto(oldhsize),Node); +} +static void luaH_resizearray(lua_State*L,Table*t,int nasize){ +int nsize=(t->node==(&dummynode_))?0:sizenode(t); +resize(L,t,nasize,nsize); +} +static void rehash(lua_State*L,Table*t,const TValue*ek){ +int nasize,na; +int nums[(32-2)+1]; +int i; +int totaluse; +for(i=0;i<=(32-2);i++)nums[i]=0; +nasize=numusearray(t,nums); +totaluse=nasize; +totaluse+=numusehash(t,nums,&nasize); +nasize+=countint(ek,nums); +totaluse++; +na=computesizes(nums,&nasize); +resize(L,t,nasize,totaluse-na); +} +static Table*luaH_new(lua_State*L,int narray,int nhash){ +Table*t=luaM_new(L,Table); +luaC_link(L,obj2gco(t),5); +t->metatable=NULL; +t->flags=cast_byte(~0); +t->array=NULL; +t->sizearray=0; +t->lsizenode=0; +t->node=cast(Node*,(&dummynode_)); +setarrayvector(L,t,narray); +setnodevector(L,t,nhash); +return t; +} +static void luaH_free(lua_State*L,Table*t){ +if(t->node!=(&dummynode_)) +luaM_freearray(L,t->node,sizenode(t),Node); +luaM_freearray(L,t->array,t->sizearray,TValue); +luaM_free(L,t); +} +static Node*getfreepos(Table*t){ +while(t->lastfree-->t->node){ +if(ttisnil(gkey(t->lastfree))) +return t->lastfree; +} +return NULL; +} +static TValue*newkey(lua_State*L,Table*t,const TValue*key){ +Node*mp=mainposition(t,key); +if(!ttisnil(gval(mp))||mp==(&dummynode_)){ +Node*othern; +Node*n=getfreepos(t); +if(n==NULL){ +rehash(L,t,key); +return luaH_set(L,t,key); +} +othern=mainposition(t,key2tval(mp)); +if(othern!=mp){ +while(gnext(othern)!=mp)othern=gnext(othern); +gnext(othern)=n; +*n=*mp; +gnext(mp)=NULL; +setnilvalue(gval(mp)); +} +else{ +gnext(n)=gnext(mp); +gnext(mp)=n; +mp=n; +} +} +gkey(mp)->value=key->value;gkey(mp)->tt=key->tt; +luaC_barriert(L,t,key); +return gval(mp); +} +static const TValue*luaH_getnum(Table*t,int key){ +if(cast(unsigned int,key)-1sizearray)) +return&t->array[key-1]; +else{ +lua_Number nk=cast_num(key); +Node*n=hashnum(t,nk); +do{ +if(ttisnumber(gkey(n))&&luai_numeq(nvalue(gkey(n)),nk)) +return gval(n); +else n=gnext(n); +}while(n); +return(&luaO_nilobject_); +} +} +static const TValue*luaH_getstr(Table*t,TString*key){ +Node*n=hashstr(t,key); +do{ +if(ttisstring(gkey(n))&&rawtsvalue(gkey(n))==key) +return gval(n); +else n=gnext(n); +}while(n); +return(&luaO_nilobject_); +} +static const TValue*luaH_get(Table*t,const TValue*key){ +switch(ttype(key)){ +case 0:return(&luaO_nilobject_); +case 4:return luaH_getstr(t,rawtsvalue(key)); +case 3:{ +int k; +lua_Number n=nvalue(key); +lua_number2int(k,n); +if(luai_numeq(cast_num(k),nvalue(key))) +return luaH_getnum(t,k); +} +default:{ +Node*n=mainposition(t,key); +do{ +if(luaO_rawequalObj(key2tval(n),key)) +return gval(n); +else n=gnext(n); +}while(n); +return(&luaO_nilobject_); +} +} +} +static TValue*luaH_set(lua_State*L,Table*t,const TValue*key){ +const TValue*p=luaH_get(t,key); +t->flags=0; +if(p!=(&luaO_nilobject_)) +return cast(TValue*,p); +else{ +if(ttisnil(key))luaG_runerror(L,"table index is nil"); +else if(ttisnumber(key)&&luai_numisnan(nvalue(key))) +luaG_runerror(L,"table index is NaN"); +return newkey(L,t,key); +} +} +static TValue*luaH_setnum(lua_State*L,Table*t,int key){ +const TValue*p=luaH_getnum(t,key); +if(p!=(&luaO_nilobject_)) +return cast(TValue*,p); +else{ +TValue k; +setnvalue(&k,cast_num(key)); +return newkey(L,t,&k); +} +} +static TValue*luaH_setstr(lua_State*L,Table*t,TString*key){ +const TValue*p=luaH_getstr(t,key); +if(p!=(&luaO_nilobject_)) +return cast(TValue*,p); +else{ +TValue k; +setsvalue(L,&k,key); +return newkey(L,t,&k); +} +} +static int unbound_search(Table*t,unsigned int j){ +unsigned int i=j; +j++; +while(!ttisnil(luaH_getnum(t,j))){ +i=j; +j*=2; +if(j>cast(unsigned int,(INT_MAX-2))){ +i=1; +while(!ttisnil(luaH_getnum(t,i)))i++; +return i-1; +} +} +while(j-i>1){ +unsigned int m=(i+j)/2; +if(ttisnil(luaH_getnum(t,m)))j=m; +else i=m; +} +return i; +} +static int luaH_getn(Table*t){ +unsigned int j=t->sizearray; +if(j>0&&ttisnil(&t->array[j-1])){ +unsigned int i=0; +while(j-i>1){ +unsigned int m=(i+j)/2; +if(ttisnil(&t->array[m-1]))j=m; +else i=m; +} +return i; +} +else if(t->node==(&dummynode_)) +return j; +else return unbound_search(t,j); +} +#define makewhite(g,x)((x)->gch.marked=cast_byte(((x)->gch.marked&cast_byte(~(bitmask(2)|bit2mask(0,1))))|luaC_white(g))) +#define white2gray(x)reset2bits((x)->gch.marked,0,1) +#define black2gray(x)resetbit((x)->gch.marked,2) +#define stringmark(s)reset2bits((s)->tsv.marked,0,1) +#define isfinalized(u)testbit((u)->marked,3) +#define markfinalized(u)l_setbit((u)->marked,3) +#define markvalue(g,o){checkconsistency(o);if(iscollectable(o)&&iswhite(gcvalue(o)))reallymarkobject(g,gcvalue(o));} +#define markobject(g,t){if(iswhite(obj2gco(t)))reallymarkobject(g,obj2gco(t));} +#define setthreshold(g)(g->GCthreshold=(g->estimate/100)*g->gcpause) +static void removeentry(Node*n){ +if(iscollectable(gkey(n))) +setttype(gkey(n),(8+3)); +} +static void reallymarkobject(global_State*g,GCObject*o){ +white2gray(o); +switch(o->gch.tt){ +case 4:{ +return; +} +case 7:{ +Table*mt=gco2u(o)->metatable; +gray2black(o); +if(mt)markobject(g,mt); +markobject(g,gco2u(o)->env); +return; +} +case(8+2):{ +UpVal*uv=gco2uv(o); +markvalue(g,uv->v); +if(uv->v==&uv->u.value) +gray2black(o); +return; +} +case 6:{ +gco2cl(o)->c.gclist=g->gray; +g->gray=o; +break; +} +case 5:{ +gco2h(o)->gclist=g->gray; +g->gray=o; +break; +} +case 8:{ +gco2th(o)->gclist=g->gray; +g->gray=o; +break; +} +case(8+1):{ +gco2p(o)->gclist=g->gray; +g->gray=o; +break; +} +default:; +} +} +static void marktmu(global_State*g){ +GCObject*u=g->tmudata; +if(u){ +do{ +u=u->gch.next; +makewhite(g,u); +reallymarkobject(g,u); +}while(u!=g->tmudata); +} +} +static size_t luaC_separateudata(lua_State*L,int all){ +global_State*g=G(L); +size_t deadmem=0; +GCObject**p=&g->mainthread->next; +GCObject*curr; +while((curr=*p)!=NULL){ +if(!(iswhite(curr)||all)||isfinalized(gco2u(curr))) +p=&curr->gch.next; +else if(fasttm(L,gco2u(curr)->metatable,TM_GC)==NULL){ +markfinalized(gco2u(curr)); +p=&curr->gch.next; +} +else{ +deadmem+=sizeudata(gco2u(curr)); +markfinalized(gco2u(curr)); +*p=curr->gch.next; +if(g->tmudata==NULL) +g->tmudata=curr->gch.next=curr; +else{ +curr->gch.next=g->tmudata->gch.next; +g->tmudata->gch.next=curr; +g->tmudata=curr; +} +} +} +return deadmem; +} +static int traversetable(global_State*g,Table*h){ +int i; +int weakkey=0; +int weakvalue=0; +const TValue*mode; +if(h->metatable) +markobject(g,h->metatable); +mode=gfasttm(g,h->metatable,TM_MODE); +if(mode&&ttisstring(mode)){ +weakkey=(strchr(svalue(mode),'k')!=NULL); +weakvalue=(strchr(svalue(mode),'v')!=NULL); +if(weakkey||weakvalue){ +h->marked&=~(bitmask(3)|bitmask(4)); +h->marked|=cast_byte((weakkey<<3)| +(weakvalue<<4)); +h->gclist=g->weak; +g->weak=obj2gco(h); +} +} +if(weakkey&&weakvalue)return 1; +if(!weakvalue){ +i=h->sizearray; +while(i--) +markvalue(g,&h->array[i]); +} +i=sizenode(h); +while(i--){ +Node*n=gnode(h,i); +if(ttisnil(gval(n))) +removeentry(n); +else{ +if(!weakkey)markvalue(g,gkey(n)); +if(!weakvalue)markvalue(g,gval(n)); +} +} +return weakkey||weakvalue; +} +static void traverseproto(global_State*g,Proto*f){ +int i; +if(f->source)stringmark(f->source); +for(i=0;isizek;i++) +markvalue(g,&f->k[i]); +for(i=0;isizeupvalues;i++){ +if(f->upvalues[i]) +stringmark(f->upvalues[i]); +} +for(i=0;isizep;i++){ +if(f->p[i]) +markobject(g,f->p[i]); +} +for(i=0;isizelocvars;i++){ +if(f->locvars[i].varname) +stringmark(f->locvars[i].varname); +} +} +static void traverseclosure(global_State*g,Closure*cl){ +markobject(g,cl->c.env); +if(cl->c.isC){ +int i; +for(i=0;ic.nupvalues;i++) +markvalue(g,&cl->c.upvalue[i]); +} +else{ +int i; +markobject(g,cl->l.p); +for(i=0;il.nupvalues;i++) +markobject(g,cl->l.upvals[i]); +} +} +static void checkstacksizes(lua_State*L,StkId max){ +int ci_used=cast_int(L->ci-L->base_ci); +int s_used=cast_int(max-L->stack); +if(L->size_ci>20000) +return; +if(4*ci_usedsize_ci&&2*8size_ci) +luaD_reallocCI(L,L->size_ci/2); +condhardstacktests(luaD_reallocCI(L,ci_used+1)); +if(4*s_usedstacksize&& +2*((2*20)+5)stacksize) +luaD_reallocstack(L,L->stacksize/2); +condhardstacktests(luaD_reallocstack(L,s_used)); +} +static void traversestack(global_State*g,lua_State*l){ +StkId o,lim; +CallInfo*ci; +markvalue(g,gt(l)); +lim=l->top; +for(ci=l->base_ci;ci<=l->ci;ci++){ +if(limtop)lim=ci->top; +} +for(o=l->stack;otop;o++) +markvalue(g,o); +for(;o<=lim;o++) +setnilvalue(o); +checkstacksizes(l,lim); +} +static l_mem propagatemark(global_State*g){ +GCObject*o=g->gray; +gray2black(o); +switch(o->gch.tt){ +case 5:{ +Table*h=gco2h(o); +g->gray=h->gclist; +if(traversetable(g,h)) +black2gray(o); +return sizeof(Table)+sizeof(TValue)*h->sizearray+ +sizeof(Node)*sizenode(h); +} +case 6:{ +Closure*cl=gco2cl(o); +g->gray=cl->c.gclist; +traverseclosure(g,cl); +return(cl->c.isC)?sizeCclosure(cl->c.nupvalues): +sizeLclosure(cl->l.nupvalues); +} +case 8:{ +lua_State*th=gco2th(o); +g->gray=th->gclist; +th->gclist=g->grayagain; +g->grayagain=o; +black2gray(o); +traversestack(g,th); +return sizeof(lua_State)+sizeof(TValue)*th->stacksize+ +sizeof(CallInfo)*th->size_ci; +} +case(8+1):{ +Proto*p=gco2p(o); +g->gray=p->gclist; +traverseproto(g,p); +return sizeof(Proto)+sizeof(Instruction)*p->sizecode+ +sizeof(Proto*)*p->sizep+ +sizeof(TValue)*p->sizek+ +sizeof(int)*p->sizelineinfo+ +sizeof(LocVar)*p->sizelocvars+ +sizeof(TString*)*p->sizeupvalues; +} +default:return 0; +} +} +static size_t propagateall(global_State*g){ +size_t m=0; +while(g->gray)m+=propagatemark(g); +return m; +} +static int iscleared(const TValue*o,int iskey){ +if(!iscollectable(o))return 0; +if(ttisstring(o)){ +stringmark(rawtsvalue(o)); +return 0; +} +return iswhite(gcvalue(o))|| +(ttisuserdata(o)&&(!iskey&&isfinalized(uvalue(o)))); +} +static void cleartable(GCObject*l){ +while(l){ +Table*h=gco2h(l); +int i=h->sizearray; +if(testbit(h->marked,4)){ +while(i--){ +TValue*o=&h->array[i]; +if(iscleared(o,0)) +setnilvalue(o); +} +} +i=sizenode(h); +while(i--){ +Node*n=gnode(h,i); +if(!ttisnil(gval(n))&& +(iscleared(key2tval(n),1)||iscleared(gval(n),0))){ +setnilvalue(gval(n)); +removeentry(n); +} +} +l=h->gclist; +} +} +static void freeobj(lua_State*L,GCObject*o){ +switch(o->gch.tt){ +case(8+1):luaF_freeproto(L,gco2p(o));break; +case 6:luaF_freeclosure(L,gco2cl(o));break; +case(8+2):luaF_freeupval(L,gco2uv(o));break; +case 5:luaH_free(L,gco2h(o));break; +case 8:{ +luaE_freethread(L,gco2th(o)); +break; +} +case 4:{ +G(L)->strt.nuse--; +luaM_freemem(L,o,sizestring(gco2ts(o))); +break; +} +case 7:{ +luaM_freemem(L,o,sizeudata(gco2u(o))); +break; +} +default:; +} +} +#define sweepwholelist(L,p)sweeplist(L,p,((lu_mem)(~(lu_mem)0)-2)) +static GCObject**sweeplist(lua_State*L,GCObject**p,lu_mem count){ +GCObject*curr; +global_State*g=G(L); +int deadmask=otherwhite(g); +while((curr=*p)!=NULL&&count-->0){ +if(curr->gch.tt==8) +sweepwholelist(L,&gco2th(curr)->openupval); +if((curr->gch.marked^bit2mask(0,1))&deadmask){ +makewhite(g,curr); +p=&curr->gch.next; +} +else{ +*p=curr->gch.next; +if(curr==g->rootgc) +g->rootgc=curr->gch.next; +freeobj(L,curr); +} +} +return p; +} +static void checkSizes(lua_State*L){ +global_State*g=G(L); +if(g->strt.nusestrt.size/4)&& +g->strt.size>32*2) +luaS_resize(L,g->strt.size/2); +if(luaZ_sizebuffer(&g->buff)>32*2){ +size_t newsize=luaZ_sizebuffer(&g->buff)/2; +luaZ_resizebuffer(L,&g->buff,newsize); +} +} +static void GCTM(lua_State*L){ +global_State*g=G(L); +GCObject*o=g->tmudata->gch.next; +Udata*udata=rawgco2u(o); +const TValue*tm; +if(o==g->tmudata) +g->tmudata=NULL; +else +g->tmudata->gch.next=udata->uv.next; +udata->uv.next=g->mainthread->next; +g->mainthread->next=o; +makewhite(g,o); +tm=fasttm(L,udata->uv.metatable,TM_GC); +if(tm!=NULL){ +lu_byte oldah=L->allowhook; +lu_mem oldt=g->GCthreshold; +L->allowhook=0; +g->GCthreshold=2*g->totalbytes; +setobj(L,L->top,tm); +setuvalue(L,L->top+1,udata); +L->top+=2; +luaD_call(L,L->top-2,0); +L->allowhook=oldah; +g->GCthreshold=oldt; +} +} +static void luaC_callGCTM(lua_State*L){ +while(G(L)->tmudata) +GCTM(L); +} +static void luaC_freeall(lua_State*L){ +global_State*g=G(L); +int i; +g->currentwhite=bit2mask(0,1)|bitmask(6); +sweepwholelist(L,&g->rootgc); +for(i=0;istrt.size;i++) +sweepwholelist(L,&g->strt.hash[i]); +} +static void markmt(global_State*g){ +int i; +for(i=0;i<(8+1);i++) +if(g->mt[i])markobject(g,g->mt[i]); +} +static void markroot(lua_State*L){ +global_State*g=G(L); +g->gray=NULL; +g->grayagain=NULL; +g->weak=NULL; +markobject(g,g->mainthread); +markvalue(g,gt(g->mainthread)); +markvalue(g,registry(L)); +markmt(g); +g->gcstate=1; +} +static void remarkupvals(global_State*g){ +UpVal*uv; +for(uv=g->uvhead.u.l.next;uv!=&g->uvhead;uv=uv->u.l.next){ +if(isgray(obj2gco(uv))) +markvalue(g,uv->v); +} +} +static void atomic(lua_State*L){ +global_State*g=G(L); +size_t udsize; +remarkupvals(g); +propagateall(g); +g->gray=g->weak; +g->weak=NULL; +markobject(g,L); +markmt(g); +propagateall(g); +g->gray=g->grayagain; +g->grayagain=NULL; +propagateall(g); +udsize=luaC_separateudata(L,0); +marktmu(g); +udsize+=propagateall(g); +cleartable(g->weak); +g->currentwhite=cast_byte(otherwhite(g)); +g->sweepstrgc=0; +g->sweepgc=&g->rootgc; +g->gcstate=2; +g->estimate=g->totalbytes-udsize; +} +static l_mem singlestep(lua_State*L){ +global_State*g=G(L); +switch(g->gcstate){ +case 0:{ +markroot(L); +return 0; +} +case 1:{ +if(g->gray) +return propagatemark(g); +else{ +atomic(L); +return 0; +} +} +case 2:{ +lu_mem old=g->totalbytes; +sweepwholelist(L,&g->strt.hash[g->sweepstrgc++]); +if(g->sweepstrgc>=g->strt.size) +g->gcstate=3; +g->estimate-=old-g->totalbytes; +return 10; +} +case 3:{ +lu_mem old=g->totalbytes; +g->sweepgc=sweeplist(L,g->sweepgc,40); +if(*g->sweepgc==NULL){ +checkSizes(L); +g->gcstate=4; +} +g->estimate-=old-g->totalbytes; +return 40*10; +} +case 4:{ +if(g->tmudata){ +GCTM(L); +if(g->estimate>100) +g->estimate-=100; +return 100; +} +else{ +g->gcstate=0; +g->gcdept=0; +return 0; +} +} +default:return 0; +} +} +static void luaC_step(lua_State*L){ +global_State*g=G(L); +l_mem lim=(1024u/100)*g->gcstepmul; +if(lim==0) +lim=(((lu_mem)(~(lu_mem)0)-2)-1)/2; +g->gcdept+=g->totalbytes-g->GCthreshold; +do{ +lim-=singlestep(L); +if(g->gcstate==0) +break; +}while(lim>0); +if(g->gcstate!=0){ +if(g->gcdept<1024u) +g->GCthreshold=g->totalbytes+1024u; +else{ +g->gcdept-=1024u; +g->GCthreshold=g->totalbytes; +} +} +else{ +setthreshold(g); +} +} +static void luaC_barrierf(lua_State*L,GCObject*o,GCObject*v){ +global_State*g=G(L); +if(g->gcstate==1) +reallymarkobject(g,v); +else +makewhite(g,o); +} +static void luaC_barrierback(lua_State*L,Table*t){ +global_State*g=G(L); +GCObject*o=obj2gco(t); +black2gray(o); +t->gclist=g->grayagain; +g->grayagain=o; +} +static void luaC_link(lua_State*L,GCObject*o,lu_byte tt){ +global_State*g=G(L); +o->gch.next=g->rootgc; +g->rootgc=o; +o->gch.marked=luaC_white(g); +o->gch.tt=tt; +} +static void luaC_linkupval(lua_State*L,UpVal*uv){ +global_State*g=G(L); +GCObject*o=obj2gco(uv); +o->gch.next=g->rootgc; +g->rootgc=o; +if(isgray(o)){ +if(g->gcstate==1){ +gray2black(o); +luaC_barrier(L,uv,uv->v); +} +else{ +makewhite(g,o); +} +} +} +typedef union{ +lua_Number r; +TString*ts; +}SemInfo; +typedef struct Token{ +int token; +SemInfo seminfo; +}Token; +typedef struct LexState{ +int current; +int linenumber; +int lastline; +Token t; +Token lookahead; +struct FuncState*fs; +struct lua_State*L; +ZIO*z; +Mbuffer*buff; +TString*source; +char decpoint; +}LexState; +static void luaX_init(lua_State*L); +static void luaX_lexerror(LexState*ls,const char*msg,int token); +#define state_size(x)(sizeof(x)+0) +#define fromstate(l)(cast(lu_byte*,(l))-0) +#define tostate(l)(cast(lua_State*,cast(lu_byte*,l)+0)) +typedef struct LG{ +lua_State l; +global_State g; +}LG; +static void stack_init(lua_State*L1,lua_State*L){ +L1->base_ci=luaM_newvector(L,8,CallInfo); +L1->ci=L1->base_ci; +L1->size_ci=8; +L1->end_ci=L1->base_ci+L1->size_ci-1; +L1->stack=luaM_newvector(L,(2*20)+5,TValue); +L1->stacksize=(2*20)+5; +L1->top=L1->stack; +L1->stack_last=L1->stack+(L1->stacksize-5)-1; +L1->ci->func=L1->top; +setnilvalue(L1->top++); +L1->base=L1->ci->base=L1->top; +L1->ci->top=L1->top+20; +} +static void freestack(lua_State*L,lua_State*L1){ +luaM_freearray(L,L1->base_ci,L1->size_ci,CallInfo); +luaM_freearray(L,L1->stack,L1->stacksize,TValue); +} +static void f_luaopen(lua_State*L,void*ud){ +global_State*g=G(L); +UNUSED(ud); +stack_init(L,L); +sethvalue(L,gt(L),luaH_new(L,0,2)); +sethvalue(L,registry(L),luaH_new(L,0,2)); +luaS_resize(L,32); +luaT_init(L); +luaX_init(L); +luaS_fix(luaS_newliteral(L,"not enough memory")); +g->GCthreshold=4*g->totalbytes; +} +static void preinit_state(lua_State*L,global_State*g){ +G(L)=g; +L->stack=NULL; +L->stacksize=0; +L->errorJmp=NULL; +L->hook=NULL; +L->hookmask=0; +L->basehookcount=0; +L->allowhook=1; +resethookcount(L); +L->openupval=NULL; +L->size_ci=0; +L->nCcalls=L->baseCcalls=0; +L->status=0; +L->base_ci=L->ci=NULL; +L->savedpc=NULL; +L->errfunc=0; +setnilvalue(gt(L)); +} +static void close_state(lua_State*L){ +global_State*g=G(L); +luaF_close(L,L->stack); +luaC_freeall(L); +luaM_freearray(L,G(L)->strt.hash,G(L)->strt.size,TString*); +luaZ_freebuffer(L,&g->buff); +freestack(L,L); +(*g->frealloc)(g->ud,fromstate(L),state_size(LG),0); +} +static void luaE_freethread(lua_State*L,lua_State*L1){ +luaF_close(L1,L1->stack); +freestack(L,L1); +luaM_freemem(L,fromstate(L1),state_size(lua_State)); +} +static lua_State*lua_newstate(lua_Alloc f,void*ud){ +int i; +lua_State*L; +global_State*g; +void*l=(*f)(ud,NULL,0,state_size(LG)); +if(l==NULL)return NULL; +L=tostate(l); +g=&((LG*)L)->g; +L->next=NULL; +L->tt=8; +g->currentwhite=bit2mask(0,5); +L->marked=luaC_white(g); +set2bits(L->marked,5,6); +preinit_state(L,g); +g->frealloc=f; +g->ud=ud; +g->mainthread=L; +g->uvhead.u.l.prev=&g->uvhead; +g->uvhead.u.l.next=&g->uvhead; +g->GCthreshold=0; +g->strt.size=0; +g->strt.nuse=0; +g->strt.hash=NULL; +setnilvalue(registry(L)); +luaZ_initbuffer(L,&g->buff); +g->panic=NULL; +g->gcstate=0; +g->rootgc=obj2gco(L); +g->sweepstrgc=0; +g->sweepgc=&g->rootgc; +g->gray=NULL; +g->grayagain=NULL; +g->weak=NULL; +g->tmudata=NULL; +g->totalbytes=sizeof(LG); +g->gcpause=200; +g->gcstepmul=200; +g->gcdept=0; +for(i=0;i<(8+1);i++)g->mt[i]=NULL; +if(luaD_rawrunprotected(L,f_luaopen,NULL)!=0){ +close_state(L); +L=NULL; +} +else +{} +return L; +} +static void callallgcTM(lua_State*L,void*ud){ +UNUSED(ud); +luaC_callGCTM(L); +} +static void lua_close(lua_State*L){ +L=G(L)->mainthread; +luaF_close(L,L->stack); +luaC_separateudata(L,1); +L->errfunc=0; +do{ +L->ci=L->base_ci; +L->base=L->top=L->ci->base; +L->nCcalls=L->baseCcalls=0; +}while(luaD_rawrunprotected(L,callallgcTM,NULL)!=0); +close_state(L); +} +#define getcode(fs,e)((fs)->f->code[(e)->u.s.info]) +#define luaK_codeAsBx(fs,o,A,sBx)luaK_codeABx(fs,o,A,(sBx)+(((1<<(9+9))-1)>>1)) +#define luaK_setmultret(fs,e)luaK_setreturns(fs,e,(-1)) +static int luaK_codeABx(FuncState*fs,OpCode o,int A,unsigned int Bx); +static int luaK_codeABC(FuncState*fs,OpCode o,int A,int B,int C); +static void luaK_setreturns(FuncState*fs,expdesc*e,int nresults); +static void luaK_patchtohere(FuncState*fs,int list); +static void luaK_concat(FuncState*fs,int*l1,int l2); +static int currentpc(lua_State*L,CallInfo*ci){ +if(!isLua(ci))return-1; +if(ci==L->ci) +ci->savedpc=L->savedpc; +return pcRel(ci->savedpc,ci_func(ci)->l.p); +} +static int currentline(lua_State*L,CallInfo*ci){ +int pc=currentpc(L,ci); +if(pc<0) +return-1; +else +return getline_(ci_func(ci)->l.p,pc); +} +static int lua_getstack(lua_State*L,int level,lua_Debug*ar){ +int status; +CallInfo*ci; +for(ci=L->ci;level>0&&ci>L->base_ci;ci--){ +level--; +if(f_isLua(ci)) +level-=ci->tailcalls; +} +if(level==0&&ci>L->base_ci){ +status=1; +ar->i_ci=cast_int(ci-L->base_ci); +} +else if(level<0){ +status=1; +ar->i_ci=0; +} +else status=0; +return status; +} +static Proto*getluaproto(CallInfo*ci){ +return(isLua(ci)?ci_func(ci)->l.p:NULL); +} +static void funcinfo(lua_Debug*ar,Closure*cl){ +if(cl->c.isC){ +ar->source="=[C]"; +ar->linedefined=-1; +ar->lastlinedefined=-1; +ar->what="C"; +} +else{ +ar->source=getstr(cl->l.p->source); +ar->linedefined=cl->l.p->linedefined; +ar->lastlinedefined=cl->l.p->lastlinedefined; +ar->what=(ar->linedefined==0)?"main":"Lua"; +} +luaO_chunkid(ar->short_src,ar->source,60); +} +static void info_tailcall(lua_Debug*ar){ +ar->name=ar->namewhat=""; +ar->what="tail"; +ar->lastlinedefined=ar->linedefined=ar->currentline=-1; +ar->source="=(tail call)"; +luaO_chunkid(ar->short_src,ar->source,60); +ar->nups=0; +} +static void collectvalidlines(lua_State*L,Closure*f){ +if(f==NULL||f->c.isC){ +setnilvalue(L->top); +} +else{ +Table*t=luaH_new(L,0,0); +int*lineinfo=f->l.p->lineinfo; +int i; +for(i=0;il.p->sizelineinfo;i++) +setbvalue(luaH_setnum(L,t,lineinfo[i]),1); +sethvalue(L,L->top,t); +} +incr_top(L); +} +static int auxgetinfo(lua_State*L,const char*what,lua_Debug*ar, +Closure*f,CallInfo*ci){ +int status=1; +if(f==NULL){ +info_tailcall(ar); +return status; +} +for(;*what;what++){ +switch(*what){ +case'S':{ +funcinfo(ar,f); +break; +} +case'l':{ +ar->currentline=(ci)?currentline(L,ci):-1; +break; +} +case'u':{ +ar->nups=f->c.nupvalues; +break; +} +case'n':{ +ar->namewhat=(ci)?NULL:NULL; +if(ar->namewhat==NULL){ +ar->namewhat=""; +ar->name=NULL; +} +break; +} +case'L': +case'f': +break; +default:status=0; +} +} +return status; +} +static int lua_getinfo(lua_State*L,const char*what,lua_Debug*ar){ +int status; +Closure*f=NULL; +CallInfo*ci=NULL; +if(*what=='>'){ +StkId func=L->top-1; +luai_apicheck(L,ttisfunction(func)); +what++; +f=clvalue(func); +L->top--; +} +else if(ar->i_ci!=0){ +ci=L->base_ci+ar->i_ci; +f=clvalue(ci->func); +} +status=auxgetinfo(L,what,ar,f,ci); +if(strchr(what,'f')){ +if(f==NULL)setnilvalue(L->top); +else setclvalue(L,L->top,f); +incr_top(L); +} +if(strchr(what,'L')) +collectvalidlines(L,f); +return status; +} +static int isinstack(CallInfo*ci,const TValue*o){ +StkId p; +for(p=ci->base;ptop;p++) +if(o==p)return 1; +return 0; +} +static void luaG_typeerror(lua_State*L,const TValue*o,const char*op){ +const char*name=NULL; +const char*t=luaT_typenames[ttype(o)]; +const char*kind=(isinstack(L->ci,o))? +NULL: +NULL; +if(kind) +luaG_runerror(L,"attempt to %s %s "LUA_QL("%s")" (a %s value)", +op,kind,name,t); +else +luaG_runerror(L,"attempt to %s a %s value",op,t); +} +static void luaG_concaterror(lua_State*L,StkId p1,StkId p2){ +if(ttisstring(p1)||ttisnumber(p1))p1=p2; +luaG_typeerror(L,p1,"concatenate"); +} +static void luaG_aritherror(lua_State*L,const TValue*p1,const TValue*p2){ +TValue temp; +if(luaV_tonumber(p1,&temp)==NULL) +p2=p1; +luaG_typeerror(L,p2,"perform arithmetic on"); +} +static int luaG_ordererror(lua_State*L,const TValue*p1,const TValue*p2){ +const char*t1=luaT_typenames[ttype(p1)]; +const char*t2=luaT_typenames[ttype(p2)]; +if(t1[2]==t2[2]) +luaG_runerror(L,"attempt to compare two %s values",t1); +else +luaG_runerror(L,"attempt to compare %s with %s",t1,t2); +return 0; +} +static void addinfo(lua_State*L,const char*msg){ +CallInfo*ci=L->ci; +if(isLua(ci)){ +char buff[60]; +int line=currentline(L,ci); +luaO_chunkid(buff,getstr(getluaproto(ci)->source),60); +luaO_pushfstring(L,"%s:%d: %s",buff,line,msg); +} +} +static void luaG_errormsg(lua_State*L){ +if(L->errfunc!=0){ +StkId errfunc=restorestack(L,L->errfunc); +if(!ttisfunction(errfunc))luaD_throw(L,5); +setobj(L,L->top,L->top-1); +setobj(L,L->top-1,errfunc); +incr_top(L); +luaD_call(L,L->top-2,1); +} +luaD_throw(L,2); +} +static void luaG_runerror(lua_State*L,const char*fmt,...){ +va_list argp; +va_start(argp,fmt); +addinfo(L,luaO_pushvfstring(L,fmt,argp)); +va_end(argp); +luaG_errormsg(L); +} +static int luaZ_fill(ZIO*z){ +size_t size; +lua_State*L=z->L; +const char*buff; +buff=z->reader(L,z->data,&size); +if(buff==NULL||size==0)return(-1); +z->n=size-1; +z->p=buff; +return char2int(*(z->p++)); +} +static void luaZ_init(lua_State*L,ZIO*z,lua_Reader reader,void*data){ +z->L=L; +z->reader=reader; +z->data=data; +z->n=0; +z->p=NULL; +} +static char*luaZ_openspace(lua_State*L,Mbuffer*buff,size_t n){ +if(n>buff->buffsize){ +if(n<32)n=32; +luaZ_resizebuffer(L,buff,n); +} +return buff->buffer; +} +#define opmode(t,a,b,c,m)(((t)<<7)|((a)<<6)|((b)<<4)|((c)<<2)|(m)) +static const lu_byte luaP_opmodes[(cast(int,OP_VARARG)+1)]={ +opmode(0,1,OpArgR,OpArgN,iABC) +,opmode(0,1,OpArgK,OpArgN,iABx) +,opmode(0,1,OpArgU,OpArgU,iABC) +,opmode(0,1,OpArgR,OpArgN,iABC) +,opmode(0,1,OpArgU,OpArgN,iABC) +,opmode(0,1,OpArgK,OpArgN,iABx) +,opmode(0,1,OpArgR,OpArgK,iABC) +,opmode(0,0,OpArgK,OpArgN,iABx) +,opmode(0,0,OpArgU,OpArgN,iABC) +,opmode(0,0,OpArgK,OpArgK,iABC) +,opmode(0,1,OpArgU,OpArgU,iABC) +,opmode(0,1,OpArgR,OpArgK,iABC) +,opmode(0,1,OpArgK,OpArgK,iABC) +,opmode(0,1,OpArgK,OpArgK,iABC) +,opmode(0,1,OpArgK,OpArgK,iABC) +,opmode(0,1,OpArgK,OpArgK,iABC) +,opmode(0,1,OpArgK,OpArgK,iABC) +,opmode(0,1,OpArgK,OpArgK,iABC) +,opmode(0,1,OpArgR,OpArgN,iABC) +,opmode(0,1,OpArgR,OpArgN,iABC) +,opmode(0,1,OpArgR,OpArgN,iABC) +,opmode(0,1,OpArgR,OpArgR,iABC) +,opmode(0,0,OpArgR,OpArgN,iAsBx) +,opmode(1,0,OpArgK,OpArgK,iABC) +,opmode(1,0,OpArgK,OpArgK,iABC) +,opmode(1,0,OpArgK,OpArgK,iABC) +,opmode(1,1,OpArgR,OpArgU,iABC) +,opmode(1,1,OpArgR,OpArgU,iABC) +,opmode(0,1,OpArgU,OpArgU,iABC) +,opmode(0,1,OpArgU,OpArgU,iABC) +,opmode(0,0,OpArgU,OpArgN,iABC) +,opmode(0,1,OpArgR,OpArgN,iAsBx) +,opmode(0,1,OpArgR,OpArgN,iAsBx) +,opmode(1,0,OpArgN,OpArgU,iABC) +,opmode(0,0,OpArgU,OpArgU,iABC) +,opmode(0,0,OpArgN,OpArgN,iABC) +,opmode(0,1,OpArgU,OpArgN,iABx) +,opmode(0,1,OpArgU,OpArgN,iABC) +}; +#define next(ls)(ls->current=zgetc(ls->z)) +#define currIsNewline(ls)(ls->current=='\n'||ls->current=='\r') +static const char*const luaX_tokens[]={ +"and","break","do","else","elseif", +"end","false","for","function","if", +"in","local","nil","not","or","repeat", +"return","then","true","until","while", +"..","...","==",">=","<=","~=", +"","","","", +NULL +}; +#define save_and_next(ls)(save(ls,ls->current),next(ls)) +static void save(LexState*ls,int c){ +Mbuffer*b=ls->buff; +if(b->n+1>b->buffsize){ +size_t newsize; +if(b->buffsize>=((size_t)(~(size_t)0)-2)/2) +luaX_lexerror(ls,"lexical element too long",0); +newsize=b->buffsize*2; +luaZ_resizebuffer(ls->L,b,newsize); +} +b->buffer[b->n++]=cast(char,c); +} +static void luaX_init(lua_State*L){ +int i; +for(i=0;i<(cast(int,TK_WHILE-257+1));i++){ +TString*ts=luaS_new(L,luaX_tokens[i]); +luaS_fix(ts); +ts->tsv.reserved=cast_byte(i+1); +} +} +static const char*luaX_token2str(LexState*ls,int token){ +if(token<257){ +return(iscntrl(token))?luaO_pushfstring(ls->L,"char(%d)",token): +luaO_pushfstring(ls->L,"%c",token); +} +else +return luaX_tokens[token-257]; +} +static const char*txtToken(LexState*ls,int token){ +switch(token){ +case TK_NAME: +case TK_STRING: +case TK_NUMBER: +save(ls,'\0'); +return luaZ_buffer(ls->buff); +default: +return luaX_token2str(ls,token); +} +} +static void luaX_lexerror(LexState*ls,const char*msg,int token){ +char buff[80]; +luaO_chunkid(buff,getstr(ls->source),80); +msg=luaO_pushfstring(ls->L,"%s:%d: %s",buff,ls->linenumber,msg); +if(token) +luaO_pushfstring(ls->L,"%s near "LUA_QL("%s"),msg,txtToken(ls,token)); +luaD_throw(ls->L,3); +} +static void luaX_syntaxerror(LexState*ls,const char*msg){ +luaX_lexerror(ls,msg,ls->t.token); +} +static TString*luaX_newstring(LexState*ls,const char*str,size_t l){ +lua_State*L=ls->L; +TString*ts=luaS_newlstr(L,str,l); +TValue*o=luaH_setstr(L,ls->fs->h,ts); +if(ttisnil(o)){ +setbvalue(o,1); +luaC_checkGC(L); +} +return ts; +} +static void inclinenumber(LexState*ls){ +int old=ls->current; +next(ls); +if(currIsNewline(ls)&&ls->current!=old) +next(ls); +if(++ls->linenumber>=(INT_MAX-2)) +luaX_syntaxerror(ls,"chunk has too many lines"); +} +static void luaX_setinput(lua_State*L,LexState*ls,ZIO*z,TString*source){ +ls->decpoint='.'; +ls->L=L; +ls->lookahead.token=TK_EOS; +ls->z=z; +ls->fs=NULL; +ls->linenumber=1; +ls->lastline=1; +ls->source=source; +luaZ_resizebuffer(ls->L,ls->buff,32); +next(ls); +} +static int check_next(LexState*ls,const char*set){ +if(!strchr(set,ls->current)) +return 0; +save_and_next(ls); +return 1; +} +static void buffreplace(LexState*ls,char from,char to){ +size_t n=luaZ_bufflen(ls->buff); +char*p=luaZ_buffer(ls->buff); +while(n--) +if(p[n]==from)p[n]=to; +} +static void read_numeral(LexState*ls,SemInfo*seminfo){ +do{ +save_and_next(ls); +}while(isdigit(ls->current)||ls->current=='.'); +if(check_next(ls,"Ee")) +check_next(ls,"+-"); +while(isalnum(ls->current)||ls->current=='_') +save_and_next(ls); +save(ls,'\0'); +buffreplace(ls,'.',ls->decpoint); +if(!luaO_str2d(luaZ_buffer(ls->buff),&seminfo->r)) +luaX_lexerror(ls,"malformed number",TK_NUMBER); +} +static int skip_sep(LexState*ls){ +int count=0; +int s=ls->current; +save_and_next(ls); +while(ls->current=='='){ +save_and_next(ls); +count++; +} +return(ls->current==s)?count:(-count)-1; +} +static void read_long_string(LexState*ls,SemInfo*seminfo,int sep){ +int cont=0; +(void)(cont); +save_and_next(ls); +if(currIsNewline(ls)) +inclinenumber(ls); +for(;;){ +switch(ls->current){ +case(-1): +luaX_lexerror(ls,(seminfo)?"unfinished long string": +"unfinished long comment",TK_EOS); +break; +case']':{ +if(skip_sep(ls)==sep){ +save_and_next(ls); +goto endloop; +} +break; +} +case'\n': +case'\r':{ +save(ls,'\n'); +inclinenumber(ls); +if(!seminfo)luaZ_resetbuffer(ls->buff); +break; +} +default:{ +if(seminfo)save_and_next(ls); +else next(ls); +} +} +}endloop: +if(seminfo) +seminfo->ts=luaX_newstring(ls,luaZ_buffer(ls->buff)+(2+sep), +luaZ_bufflen(ls->buff)-2*(2+sep)); +} +static void read_string(LexState*ls,int del,SemInfo*seminfo){ +save_and_next(ls); +while(ls->current!=del){ +switch(ls->current){ +case(-1): +luaX_lexerror(ls,"unfinished string",TK_EOS); +continue; +case'\n': +case'\r': +luaX_lexerror(ls,"unfinished string",TK_STRING); +continue; +case'\\':{ +int c; +next(ls); +switch(ls->current){ +case'a':c='\a';break; +case'b':c='\b';break; +case'f':c='\f';break; +case'n':c='\n';break; +case'r':c='\r';break; +case't':c='\t';break; +case'v':c='\v';break; +case'\n': +case'\r':save(ls,'\n');inclinenumber(ls);continue; +case(-1):continue; +default:{ +if(!isdigit(ls->current)) +save_and_next(ls); +else{ +int i=0; +c=0; +do{ +c=10*c+(ls->current-'0'); +next(ls); +}while(++i<3&&isdigit(ls->current)); +if(c>UCHAR_MAX) +luaX_lexerror(ls,"escape sequence too large",TK_STRING); +save(ls,c); +} +continue; +} +} +save(ls,c); +next(ls); +continue; +} +default: +save_and_next(ls); +} +} +save_and_next(ls); +seminfo->ts=luaX_newstring(ls,luaZ_buffer(ls->buff)+1, +luaZ_bufflen(ls->buff)-2); +} +static int llex(LexState*ls,SemInfo*seminfo){ +luaZ_resetbuffer(ls->buff); +for(;;){ +switch(ls->current){ +case'\n': +case'\r':{ +inclinenumber(ls); +continue; +} +case'-':{ +next(ls); +if(ls->current!='-')return'-'; +next(ls); +if(ls->current=='['){ +int sep=skip_sep(ls); +luaZ_resetbuffer(ls->buff); +if(sep>=0){ +read_long_string(ls,NULL,sep); +luaZ_resetbuffer(ls->buff); +continue; +} +} +while(!currIsNewline(ls)&&ls->current!=(-1)) +next(ls); +continue; +} +case'[':{ +int sep=skip_sep(ls); +if(sep>=0){ +read_long_string(ls,seminfo,sep); +return TK_STRING; +} +else if(sep==-1)return'['; +else luaX_lexerror(ls,"invalid long string delimiter",TK_STRING); +} +case'=':{ +next(ls); +if(ls->current!='=')return'='; +else{next(ls);return TK_EQ;} +} +case'<':{ +next(ls); +if(ls->current!='=')return'<'; +else{next(ls);return TK_LE;} +} +case'>':{ +next(ls); +if(ls->current!='=')return'>'; +else{next(ls);return TK_GE;} +} +case'~':{ +next(ls); +if(ls->current!='=')return'~'; +else{next(ls);return TK_NE;} +} +case'"': +case'\'':{ +read_string(ls,ls->current,seminfo); +return TK_STRING; +} +case'.':{ +save_and_next(ls); +if(check_next(ls,".")){ +if(check_next(ls,".")) +return TK_DOTS; +else return TK_CONCAT; +} +else if(!isdigit(ls->current))return'.'; +else{ +read_numeral(ls,seminfo); +return TK_NUMBER; +} +} +case(-1):{ +return TK_EOS; +} +default:{ +if(isspace(ls->current)){ +next(ls); +continue; +} +else if(isdigit(ls->current)){ +read_numeral(ls,seminfo); +return TK_NUMBER; +} +else if(isalpha(ls->current)||ls->current=='_'){ +TString*ts; +do{ +save_and_next(ls); +}while(isalnum(ls->current)||ls->current=='_'); +ts=luaX_newstring(ls,luaZ_buffer(ls->buff), +luaZ_bufflen(ls->buff)); +if(ts->tsv.reserved>0) +return ts->tsv.reserved-1+257; +else{ +seminfo->ts=ts; +return TK_NAME; +} +} +else{ +int c=ls->current; +next(ls); +return c; +} +} +} +} +} +static void luaX_next(LexState*ls){ +ls->lastline=ls->linenumber; +if(ls->lookahead.token!=TK_EOS){ +ls->t=ls->lookahead; +ls->lookahead.token=TK_EOS; +} +else +ls->t.token=llex(ls,&ls->t.seminfo); +} +static void luaX_lookahead(LexState*ls){ +ls->lookahead.token=llex(ls,&ls->lookahead.seminfo); +} +#define hasjumps(e)((e)->t!=(e)->f) +static int isnumeral(expdesc*e){ +return(e->k==VKNUM&&e->t==(-1)&&e->f==(-1)); +} +static void luaK_nil(FuncState*fs,int from,int n){ +Instruction*previous; +if(fs->pc>fs->lasttarget){ +if(fs->pc==0){ +if(from>=fs->nactvar) +return; +} +else{ +previous=&fs->f->code[fs->pc-1]; +if(GET_OPCODE(*previous)==OP_LOADNIL){ +int pfrom=GETARG_A(*previous); +int pto=GETARG_B(*previous); +if(pfrom<=from&&from<=pto+1){ +if(from+n-1>pto) +SETARG_B(*previous,from+n-1); +return; +} +} +} +} +luaK_codeABC(fs,OP_LOADNIL,from,from+n-1,0); +} +static int luaK_jump(FuncState*fs){ +int jpc=fs->jpc; +int j; +fs->jpc=(-1); +j=luaK_codeAsBx(fs,OP_JMP,0,(-1)); +luaK_concat(fs,&j,jpc); +return j; +} +static void luaK_ret(FuncState*fs,int first,int nret){ +luaK_codeABC(fs,OP_RETURN,first,nret+1,0); +} +static int condjump(FuncState*fs,OpCode op,int A,int B,int C){ +luaK_codeABC(fs,op,A,B,C); +return luaK_jump(fs); +} +static void fixjump(FuncState*fs,int pc,int dest){ +Instruction*jmp=&fs->f->code[pc]; +int offset=dest-(pc+1); +if(abs(offset)>(((1<<(9+9))-1)>>1)) +luaX_syntaxerror(fs->ls,"control structure too long"); +SETARG_sBx(*jmp,offset); +} +static int luaK_getlabel(FuncState*fs){ +fs->lasttarget=fs->pc; +return fs->pc; +} +static int getjump(FuncState*fs,int pc){ +int offset=GETARG_sBx(fs->f->code[pc]); +if(offset==(-1)) +return(-1); +else +return(pc+1)+offset; +} +static Instruction*getjumpcontrol(FuncState*fs,int pc){ +Instruction*pi=&fs->f->code[pc]; +if(pc>=1&&testTMode(GET_OPCODE(*(pi-1)))) +return pi-1; +else +return pi; +} +static int need_value(FuncState*fs,int list){ +for(;list!=(-1);list=getjump(fs,list)){ +Instruction i=*getjumpcontrol(fs,list); +if(GET_OPCODE(i)!=OP_TESTSET)return 1; +} +return 0; +} +static int patchtestreg(FuncState*fs,int node,int reg){ +Instruction*i=getjumpcontrol(fs,node); +if(GET_OPCODE(*i)!=OP_TESTSET) +return 0; +if(reg!=((1<<8)-1)&®!=GETARG_B(*i)) +SETARG_A(*i,reg); +else +*i=CREATE_ABC(OP_TEST,GETARG_B(*i),0,GETARG_C(*i)); +return 1; +} +static void removevalues(FuncState*fs,int list){ +for(;list!=(-1);list=getjump(fs,list)) +patchtestreg(fs,list,((1<<8)-1)); +} +static void patchlistaux(FuncState*fs,int list,int vtarget,int reg, +int dtarget){ +while(list!=(-1)){ +int next=getjump(fs,list); +if(patchtestreg(fs,list,reg)) +fixjump(fs,list,vtarget); +else +fixjump(fs,list,dtarget); +list=next; +} +} +static void dischargejpc(FuncState*fs){ +patchlistaux(fs,fs->jpc,fs->pc,((1<<8)-1),fs->pc); +fs->jpc=(-1); +} +static void luaK_patchlist(FuncState*fs,int list,int target){ +if(target==fs->pc) +luaK_patchtohere(fs,list); +else{ +patchlistaux(fs,list,target,((1<<8)-1),target); +} +} +static void luaK_patchtohere(FuncState*fs,int list){ +luaK_getlabel(fs); +luaK_concat(fs,&fs->jpc,list); +} +static void luaK_concat(FuncState*fs,int*l1,int l2){ +if(l2==(-1))return; +else if(*l1==(-1)) +*l1=l2; +else{ +int list=*l1; +int next; +while((next=getjump(fs,list))!=(-1)) +list=next; +fixjump(fs,list,l2); +} +} +static void luaK_checkstack(FuncState*fs,int n){ +int newstack=fs->freereg+n; +if(newstack>fs->f->maxstacksize){ +if(newstack>=250) +luaX_syntaxerror(fs->ls,"function or expression too complex"); +fs->f->maxstacksize=cast_byte(newstack); +} +} +static void luaK_reserveregs(FuncState*fs,int n){ +luaK_checkstack(fs,n); +fs->freereg+=n; +} +static void freereg(FuncState*fs,int reg){ +if(!ISK(reg)&®>=fs->nactvar){ +fs->freereg--; +} +} +static void freeexp(FuncState*fs,expdesc*e){ +if(e->k==VNONRELOC) +freereg(fs,e->u.s.info); +} +static int addk(FuncState*fs,TValue*k,TValue*v){ +lua_State*L=fs->L; +TValue*idx=luaH_set(L,fs->h,k); +Proto*f=fs->f; +int oldsize=f->sizek; +if(ttisnumber(idx)){ +return cast_int(nvalue(idx)); +} +else{ +setnvalue(idx,cast_num(fs->nk)); +luaM_growvector(L,f->k,fs->nk,f->sizek,TValue, +((1<<(9+9))-1),"constant table overflow"); +while(oldsizesizek)setnilvalue(&f->k[oldsize++]); +setobj(L,&f->k[fs->nk],v); +luaC_barrier(L,f,v); +return fs->nk++; +} +} +static int luaK_stringK(FuncState*fs,TString*s){ +TValue o; +setsvalue(fs->L,&o,s); +return addk(fs,&o,&o); +} +static int luaK_numberK(FuncState*fs,lua_Number r){ +TValue o; +setnvalue(&o,r); +return addk(fs,&o,&o); +} +static int boolK(FuncState*fs,int b){ +TValue o; +setbvalue(&o,b); +return addk(fs,&o,&o); +} +static int nilK(FuncState*fs){ +TValue k,v; +setnilvalue(&v); +sethvalue(fs->L,&k,fs->h); +return addk(fs,&k,&v); +} +static void luaK_setreturns(FuncState*fs,expdesc*e,int nresults){ +if(e->k==VCALL){ +SETARG_C(getcode(fs,e),nresults+1); +} +else if(e->k==VVARARG){ +SETARG_B(getcode(fs,e),nresults+1); +SETARG_A(getcode(fs,e),fs->freereg); +luaK_reserveregs(fs,1); +} +} +static void luaK_setoneret(FuncState*fs,expdesc*e){ +if(e->k==VCALL){ +e->k=VNONRELOC; +e->u.s.info=GETARG_A(getcode(fs,e)); +} +else if(e->k==VVARARG){ +SETARG_B(getcode(fs,e),2); +e->k=VRELOCABLE; +} +} +static void luaK_dischargevars(FuncState*fs,expdesc*e){ +switch(e->k){ +case VLOCAL:{ +e->k=VNONRELOC; +break; +} +case VUPVAL:{ +e->u.s.info=luaK_codeABC(fs,OP_GETUPVAL,0,e->u.s.info,0); +e->k=VRELOCABLE; +break; +} +case VGLOBAL:{ +e->u.s.info=luaK_codeABx(fs,OP_GETGLOBAL,0,e->u.s.info); +e->k=VRELOCABLE; +break; +} +case VINDEXED:{ +freereg(fs,e->u.s.aux); +freereg(fs,e->u.s.info); +e->u.s.info=luaK_codeABC(fs,OP_GETTABLE,0,e->u.s.info,e->u.s.aux); +e->k=VRELOCABLE; +break; +} +case VVARARG: +case VCALL:{ +luaK_setoneret(fs,e); +break; +} +default:break; +} +} +static int code_label(FuncState*fs,int A,int b,int jump){ +luaK_getlabel(fs); +return luaK_codeABC(fs,OP_LOADBOOL,A,b,jump); +} +static void discharge2reg(FuncState*fs,expdesc*e,int reg){ +luaK_dischargevars(fs,e); +switch(e->k){ +case VNIL:{ +luaK_nil(fs,reg,1); +break; +} +case VFALSE:case VTRUE:{ +luaK_codeABC(fs,OP_LOADBOOL,reg,e->k==VTRUE,0); +break; +} +case VK:{ +luaK_codeABx(fs,OP_LOADK,reg,e->u.s.info); +break; +} +case VKNUM:{ +luaK_codeABx(fs,OP_LOADK,reg,luaK_numberK(fs,e->u.nval)); +break; +} +case VRELOCABLE:{ +Instruction*pc=&getcode(fs,e); +SETARG_A(*pc,reg); +break; +} +case VNONRELOC:{ +if(reg!=e->u.s.info) +luaK_codeABC(fs,OP_MOVE,reg,e->u.s.info,0); +break; +} +default:{ +return; +} +} +e->u.s.info=reg; +e->k=VNONRELOC; +} +static void discharge2anyreg(FuncState*fs,expdesc*e){ +if(e->k!=VNONRELOC){ +luaK_reserveregs(fs,1); +discharge2reg(fs,e,fs->freereg-1); +} +} +static void exp2reg(FuncState*fs,expdesc*e,int reg){ +discharge2reg(fs,e,reg); +if(e->k==VJMP) +luaK_concat(fs,&e->t,e->u.s.info); +if(hasjumps(e)){ +int final; +int p_f=(-1); +int p_t=(-1); +if(need_value(fs,e->t)||need_value(fs,e->f)){ +int fj=(e->k==VJMP)?(-1):luaK_jump(fs); +p_f=code_label(fs,reg,0,1); +p_t=code_label(fs,reg,1,0); +luaK_patchtohere(fs,fj); +} +final=luaK_getlabel(fs); +patchlistaux(fs,e->f,final,reg,p_f); +patchlistaux(fs,e->t,final,reg,p_t); +} +e->f=e->t=(-1); +e->u.s.info=reg; +e->k=VNONRELOC; +} +static void luaK_exp2nextreg(FuncState*fs,expdesc*e){ +luaK_dischargevars(fs,e); +freeexp(fs,e); +luaK_reserveregs(fs,1); +exp2reg(fs,e,fs->freereg-1); +} +static int luaK_exp2anyreg(FuncState*fs,expdesc*e){ +luaK_dischargevars(fs,e); +if(e->k==VNONRELOC){ +if(!hasjumps(e))return e->u.s.info; +if(e->u.s.info>=fs->nactvar){ +exp2reg(fs,e,e->u.s.info); +return e->u.s.info; +} +} +luaK_exp2nextreg(fs,e); +return e->u.s.info; +} +static void luaK_exp2val(FuncState*fs,expdesc*e){ +if(hasjumps(e)) +luaK_exp2anyreg(fs,e); +else +luaK_dischargevars(fs,e); +} +static int luaK_exp2RK(FuncState*fs,expdesc*e){ +luaK_exp2val(fs,e); +switch(e->k){ +case VKNUM: +case VTRUE: +case VFALSE: +case VNIL:{ +if(fs->nk<=((1<<(9-1))-1)){ +e->u.s.info=(e->k==VNIL)?nilK(fs): +(e->k==VKNUM)?luaK_numberK(fs,e->u.nval): +boolK(fs,(e->k==VTRUE)); +e->k=VK; +return RKASK(e->u.s.info); +} +else break; +} +case VK:{ +if(e->u.s.info<=((1<<(9-1))-1)) +return RKASK(e->u.s.info); +else break; +} +default:break; +} +return luaK_exp2anyreg(fs,e); +} +static void luaK_storevar(FuncState*fs,expdesc*var,expdesc*ex){ +switch(var->k){ +case VLOCAL:{ +freeexp(fs,ex); +exp2reg(fs,ex,var->u.s.info); +return; +} +case VUPVAL:{ +int e=luaK_exp2anyreg(fs,ex); +luaK_codeABC(fs,OP_SETUPVAL,e,var->u.s.info,0); +break; +} +case VGLOBAL:{ +int e=luaK_exp2anyreg(fs,ex); +luaK_codeABx(fs,OP_SETGLOBAL,e,var->u.s.info); +break; +} +case VINDEXED:{ +int e=luaK_exp2RK(fs,ex); +luaK_codeABC(fs,OP_SETTABLE,var->u.s.info,var->u.s.aux,e); +break; +} +default:{ +break; +} +} +freeexp(fs,ex); +} +static void luaK_self(FuncState*fs,expdesc*e,expdesc*key){ +int func; +luaK_exp2anyreg(fs,e); +freeexp(fs,e); +func=fs->freereg; +luaK_reserveregs(fs,2); +luaK_codeABC(fs,OP_SELF,func,e->u.s.info,luaK_exp2RK(fs,key)); +freeexp(fs,key); +e->u.s.info=func; +e->k=VNONRELOC; +} +static void invertjump(FuncState*fs,expdesc*e){ +Instruction*pc=getjumpcontrol(fs,e->u.s.info); +SETARG_A(*pc,!(GETARG_A(*pc))); +} +static int jumponcond(FuncState*fs,expdesc*e,int cond){ +if(e->k==VRELOCABLE){ +Instruction ie=getcode(fs,e); +if(GET_OPCODE(ie)==OP_NOT){ +fs->pc--; +return condjump(fs,OP_TEST,GETARG_B(ie),0,!cond); +} +} +discharge2anyreg(fs,e); +freeexp(fs,e); +return condjump(fs,OP_TESTSET,((1<<8)-1),e->u.s.info,cond); +} +static void luaK_goiftrue(FuncState*fs,expdesc*e){ +int pc; +luaK_dischargevars(fs,e); +switch(e->k){ +case VK:case VKNUM:case VTRUE:{ +pc=(-1); +break; +} +case VJMP:{ +invertjump(fs,e); +pc=e->u.s.info; +break; +} +default:{ +pc=jumponcond(fs,e,0); +break; +} +} +luaK_concat(fs,&e->f,pc); +luaK_patchtohere(fs,e->t); +e->t=(-1); +} +static void luaK_goiffalse(FuncState*fs,expdesc*e){ +int pc; +luaK_dischargevars(fs,e); +switch(e->k){ +case VNIL:case VFALSE:{ +pc=(-1); +break; +} +case VJMP:{ +pc=e->u.s.info; +break; +} +default:{ +pc=jumponcond(fs,e,1); +break; +} +} +luaK_concat(fs,&e->t,pc); +luaK_patchtohere(fs,e->f); +e->f=(-1); +} +static void codenot(FuncState*fs,expdesc*e){ +luaK_dischargevars(fs,e); +switch(e->k){ +case VNIL:case VFALSE:{ +e->k=VTRUE; +break; +} +case VK:case VKNUM:case VTRUE:{ +e->k=VFALSE; +break; +} +case VJMP:{ +invertjump(fs,e); +break; +} +case VRELOCABLE: +case VNONRELOC:{ +discharge2anyreg(fs,e); +freeexp(fs,e); +e->u.s.info=luaK_codeABC(fs,OP_NOT,0,e->u.s.info,0); +e->k=VRELOCABLE; +break; +} +default:{ +break; +} +} +{int temp=e->f;e->f=e->t;e->t=temp;} +removevalues(fs,e->f); +removevalues(fs,e->t); +} +static void luaK_indexed(FuncState*fs,expdesc*t,expdesc*k){ +t->u.s.aux=luaK_exp2RK(fs,k); +t->k=VINDEXED; +} +static int constfolding(OpCode op,expdesc*e1,expdesc*e2){ +lua_Number v1,v2,r; +if(!isnumeral(e1)||!isnumeral(e2))return 0; +v1=e1->u.nval; +v2=e2->u.nval; +switch(op){ +case OP_ADD:r=luai_numadd(v1,v2);break; +case OP_SUB:r=luai_numsub(v1,v2);break; +case OP_MUL:r=luai_nummul(v1,v2);break; +case OP_DIV: +if(v2==0)return 0; +r=luai_numdiv(v1,v2);break; +case OP_MOD: +if(v2==0)return 0; +r=luai_nummod(v1,v2);break; +case OP_POW:r=luai_numpow(v1,v2);break; +case OP_UNM:r=luai_numunm(v1);break; +case OP_LEN:return 0; +default:r=0;break; +} +if(luai_numisnan(r))return 0; +e1->u.nval=r; +return 1; +} +static void codearith(FuncState*fs,OpCode op,expdesc*e1,expdesc*e2){ +if(constfolding(op,e1,e2)) +return; +else{ +int o2=(op!=OP_UNM&&op!=OP_LEN)?luaK_exp2RK(fs,e2):0; +int o1=luaK_exp2RK(fs,e1); +if(o1>o2){ +freeexp(fs,e1); +freeexp(fs,e2); +} +else{ +freeexp(fs,e2); +freeexp(fs,e1); +} +e1->u.s.info=luaK_codeABC(fs,op,0,o1,o2); +e1->k=VRELOCABLE; +} +} +static void codecomp(FuncState*fs,OpCode op,int cond,expdesc*e1, +expdesc*e2){ +int o1=luaK_exp2RK(fs,e1); +int o2=luaK_exp2RK(fs,e2); +freeexp(fs,e2); +freeexp(fs,e1); +if(cond==0&&op!=OP_EQ){ +int temp; +temp=o1;o1=o2;o2=temp; +cond=1; +} +e1->u.s.info=condjump(fs,op,cond,o1,o2); +e1->k=VJMP; +} +static void luaK_prefix(FuncState*fs,UnOpr op,expdesc*e){ +expdesc e2; +e2.t=e2.f=(-1);e2.k=VKNUM;e2.u.nval=0; +switch(op){ +case OPR_MINUS:{ +if(!isnumeral(e)) +luaK_exp2anyreg(fs,e); +codearith(fs,OP_UNM,e,&e2); +break; +} +case OPR_NOT:codenot(fs,e);break; +case OPR_LEN:{ +luaK_exp2anyreg(fs,e); +codearith(fs,OP_LEN,e,&e2); +break; +} +default:; +} +} +static void luaK_infix(FuncState*fs,BinOpr op,expdesc*v){ +switch(op){ +case OPR_AND:{ +luaK_goiftrue(fs,v); +break; +} +case OPR_OR:{ +luaK_goiffalse(fs,v); +break; +} +case OPR_CONCAT:{ +luaK_exp2nextreg(fs,v); +break; +} +case OPR_ADD:case OPR_SUB:case OPR_MUL:case OPR_DIV: +case OPR_MOD:case OPR_POW:{ +if(!isnumeral(v))luaK_exp2RK(fs,v); +break; +} +default:{ +luaK_exp2RK(fs,v); +break; +} +} +} +static void luaK_posfix(FuncState*fs,BinOpr op,expdesc*e1,expdesc*e2){ +switch(op){ +case OPR_AND:{ +luaK_dischargevars(fs,e2); +luaK_concat(fs,&e2->f,e1->f); +*e1=*e2; +break; +} +case OPR_OR:{ +luaK_dischargevars(fs,e2); +luaK_concat(fs,&e2->t,e1->t); +*e1=*e2; +break; +} +case OPR_CONCAT:{ +luaK_exp2val(fs,e2); +if(e2->k==VRELOCABLE&&GET_OPCODE(getcode(fs,e2))==OP_CONCAT){ +freeexp(fs,e1); +SETARG_B(getcode(fs,e2),e1->u.s.info); +e1->k=VRELOCABLE;e1->u.s.info=e2->u.s.info; +} +else{ +luaK_exp2nextreg(fs,e2); +codearith(fs,OP_CONCAT,e1,e2); +} +break; +} +case OPR_ADD:codearith(fs,OP_ADD,e1,e2);break; +case OPR_SUB:codearith(fs,OP_SUB,e1,e2);break; +case OPR_MUL:codearith(fs,OP_MUL,e1,e2);break; +case OPR_DIV:codearith(fs,OP_DIV,e1,e2);break; +case OPR_MOD:codearith(fs,OP_MOD,e1,e2);break; +case OPR_POW:codearith(fs,OP_POW,e1,e2);break; +case OPR_EQ:codecomp(fs,OP_EQ,1,e1,e2);break; +case OPR_NE:codecomp(fs,OP_EQ,0,e1,e2);break; +case OPR_LT:codecomp(fs,OP_LT,1,e1,e2);break; +case OPR_LE:codecomp(fs,OP_LE,1,e1,e2);break; +case OPR_GT:codecomp(fs,OP_LT,0,e1,e2);break; +case OPR_GE:codecomp(fs,OP_LE,0,e1,e2);break; +default:; +} +} +static void luaK_fixline(FuncState*fs,int line){ +fs->f->lineinfo[fs->pc-1]=line; +} +static int luaK_code(FuncState*fs,Instruction i,int line){ +Proto*f=fs->f; +dischargejpc(fs); +luaM_growvector(fs->L,f->code,fs->pc,f->sizecode,Instruction, +(INT_MAX-2),"code size overflow"); +f->code[fs->pc]=i; +luaM_growvector(fs->L,f->lineinfo,fs->pc,f->sizelineinfo,int, +(INT_MAX-2),"code size overflow"); +f->lineinfo[fs->pc]=line; +return fs->pc++; +} +static int luaK_codeABC(FuncState*fs,OpCode o,int a,int b,int c){ +return luaK_code(fs,CREATE_ABC(o,a,b,c),fs->ls->lastline); +} +static int luaK_codeABx(FuncState*fs,OpCode o,int a,unsigned int bc){ +return luaK_code(fs,CREATE_ABx(o,a,bc),fs->ls->lastline); +} +static void luaK_setlist(FuncState*fs,int base,int nelems,int tostore){ +int c=(nelems-1)/50+1; +int b=(tostore==(-1))?0:tostore; +if(c<=((1<<9)-1)) +luaK_codeABC(fs,OP_SETLIST,base,b,c); +else{ +luaK_codeABC(fs,OP_SETLIST,base,b,0); +luaK_code(fs,cast(Instruction,c),fs->ls->lastline); +} +fs->freereg=base+1; +} +#define hasmultret(k)((k)==VCALL||(k)==VVARARG) +#define getlocvar(fs,i)((fs)->f->locvars[(fs)->actvar[i]]) +#define luaY_checklimit(fs,v,l,m)if((v)>(l))errorlimit(fs,l,m) +typedef struct BlockCnt{ +struct BlockCnt*previous; +int breaklist; +lu_byte nactvar; +lu_byte upval; +lu_byte isbreakable; +}BlockCnt; +static void chunk(LexState*ls); +static void expr(LexState*ls,expdesc*v); +static void anchor_token(LexState*ls){ +if(ls->t.token==TK_NAME||ls->t.token==TK_STRING){ +TString*ts=ls->t.seminfo.ts; +luaX_newstring(ls,getstr(ts),ts->tsv.len); +} +} +static void error_expected(LexState*ls,int token){ +luaX_syntaxerror(ls, +luaO_pushfstring(ls->L,LUA_QL("%s")" expected",luaX_token2str(ls,token))); +} +static void errorlimit(FuncState*fs,int limit,const char*what){ +const char*msg=(fs->f->linedefined==0)? +luaO_pushfstring(fs->L,"main function has more than %d %s",limit,what): +luaO_pushfstring(fs->L,"function at line %d has more than %d %s", +fs->f->linedefined,limit,what); +luaX_lexerror(fs->ls,msg,0); +} +static int testnext(LexState*ls,int c){ +if(ls->t.token==c){ +luaX_next(ls); +return 1; +} +else return 0; +} +static void check(LexState*ls,int c){ +if(ls->t.token!=c) +error_expected(ls,c); +} +static void checknext(LexState*ls,int c){ +check(ls,c); +luaX_next(ls); +} +#define check_condition(ls,c,msg){if(!(c))luaX_syntaxerror(ls,msg);} +static void check_match(LexState*ls,int what,int who,int where){ +if(!testnext(ls,what)){ +if(where==ls->linenumber) +error_expected(ls,what); +else{ +luaX_syntaxerror(ls,luaO_pushfstring(ls->L, +LUA_QL("%s")" expected (to close "LUA_QL("%s")" at line %d)", +luaX_token2str(ls,what),luaX_token2str(ls,who),where)); +} +} +} +static TString*str_checkname(LexState*ls){ +TString*ts; +check(ls,TK_NAME); +ts=ls->t.seminfo.ts; +luaX_next(ls); +return ts; +} +static void init_exp(expdesc*e,expkind k,int i){ +e->f=e->t=(-1); +e->k=k; +e->u.s.info=i; +} +static void codestring(LexState*ls,expdesc*e,TString*s){ +init_exp(e,VK,luaK_stringK(ls->fs,s)); +} +static void checkname(LexState*ls,expdesc*e){ +codestring(ls,e,str_checkname(ls)); +} +static int registerlocalvar(LexState*ls,TString*varname){ +FuncState*fs=ls->fs; +Proto*f=fs->f; +int oldsize=f->sizelocvars; +luaM_growvector(ls->L,f->locvars,fs->nlocvars,f->sizelocvars, +LocVar,SHRT_MAX,"too many local variables"); +while(oldsizesizelocvars)f->locvars[oldsize++].varname=NULL; +f->locvars[fs->nlocvars].varname=varname; +luaC_objbarrier(ls->L,f,varname); +return fs->nlocvars++; +} +#define new_localvarliteral(ls,v,n)new_localvar(ls,luaX_newstring(ls,""v,(sizeof(v)/sizeof(char))-1),n) +static void new_localvar(LexState*ls,TString*name,int n){ +FuncState*fs=ls->fs; +luaY_checklimit(fs,fs->nactvar+n+1,200,"local variables"); +fs->actvar[fs->nactvar+n]=cast(unsigned short,registerlocalvar(ls,name)); +} +static void adjustlocalvars(LexState*ls,int nvars){ +FuncState*fs=ls->fs; +fs->nactvar=cast_byte(fs->nactvar+nvars); +for(;nvars;nvars--){ +getlocvar(fs,fs->nactvar-nvars).startpc=fs->pc; +} +} +static void removevars(LexState*ls,int tolevel){ +FuncState*fs=ls->fs; +while(fs->nactvar>tolevel) +getlocvar(fs,--fs->nactvar).endpc=fs->pc; +} +static int indexupvalue(FuncState*fs,TString*name,expdesc*v){ +int i; +Proto*f=fs->f; +int oldsize=f->sizeupvalues; +for(i=0;inups;i++){ +if(fs->upvalues[i].k==v->k&&fs->upvalues[i].info==v->u.s.info){ +return i; +} +} +luaY_checklimit(fs,f->nups+1,60,"upvalues"); +luaM_growvector(fs->L,f->upvalues,f->nups,f->sizeupvalues, +TString*,(INT_MAX-2),""); +while(oldsizesizeupvalues)f->upvalues[oldsize++]=NULL; +f->upvalues[f->nups]=name; +luaC_objbarrier(fs->L,f,name); +fs->upvalues[f->nups].k=cast_byte(v->k); +fs->upvalues[f->nups].info=cast_byte(v->u.s.info); +return f->nups++; +} +static int searchvar(FuncState*fs,TString*n){ +int i; +for(i=fs->nactvar-1;i>=0;i--){ +if(n==getlocvar(fs,i).varname) +return i; +} +return-1; +} +static void markupval(FuncState*fs,int level){ +BlockCnt*bl=fs->bl; +while(bl&&bl->nactvar>level)bl=bl->previous; +if(bl)bl->upval=1; +} +static int singlevaraux(FuncState*fs,TString*n,expdesc*var,int base){ +if(fs==NULL){ +init_exp(var,VGLOBAL,((1<<8)-1)); +return VGLOBAL; +} +else{ +int v=searchvar(fs,n); +if(v>=0){ +init_exp(var,VLOCAL,v); +if(!base) +markupval(fs,v); +return VLOCAL; +} +else{ +if(singlevaraux(fs->prev,n,var,0)==VGLOBAL) +return VGLOBAL; +var->u.s.info=indexupvalue(fs,n,var); +var->k=VUPVAL; +return VUPVAL; +} +} +} +static void singlevar(LexState*ls,expdesc*var){ +TString*varname=str_checkname(ls); +FuncState*fs=ls->fs; +if(singlevaraux(fs,varname,var,1)==VGLOBAL) +var->u.s.info=luaK_stringK(fs,varname); +} +static void adjust_assign(LexState*ls,int nvars,int nexps,expdesc*e){ +FuncState*fs=ls->fs; +int extra=nvars-nexps; +if(hasmultret(e->k)){ +extra++; +if(extra<0)extra=0; +luaK_setreturns(fs,e,extra); +if(extra>1)luaK_reserveregs(fs,extra-1); +} +else{ +if(e->k!=VVOID)luaK_exp2nextreg(fs,e); +if(extra>0){ +int reg=fs->freereg; +luaK_reserveregs(fs,extra); +luaK_nil(fs,reg,extra); +} +} +} +static void enterlevel(LexState*ls){ +if(++ls->L->nCcalls>200) +luaX_lexerror(ls,"chunk has too many syntax levels",0); +} +#define leavelevel(ls)((ls)->L->nCcalls--) +static void enterblock(FuncState*fs,BlockCnt*bl,lu_byte isbreakable){ +bl->breaklist=(-1); +bl->isbreakable=isbreakable; +bl->nactvar=fs->nactvar; +bl->upval=0; +bl->previous=fs->bl; +fs->bl=bl; +} +static void leaveblock(FuncState*fs){ +BlockCnt*bl=fs->bl; +fs->bl=bl->previous; +removevars(fs->ls,bl->nactvar); +if(bl->upval) +luaK_codeABC(fs,OP_CLOSE,bl->nactvar,0,0); +fs->freereg=fs->nactvar; +luaK_patchtohere(fs,bl->breaklist); +} +static void pushclosure(LexState*ls,FuncState*func,expdesc*v){ +FuncState*fs=ls->fs; +Proto*f=fs->f; +int oldsize=f->sizep; +int i; +luaM_growvector(ls->L,f->p,fs->np,f->sizep,Proto*, +((1<<(9+9))-1),"constant table overflow"); +while(oldsizesizep)f->p[oldsize++]=NULL; +f->p[fs->np++]=func->f; +luaC_objbarrier(ls->L,f,func->f); +init_exp(v,VRELOCABLE,luaK_codeABx(fs,OP_CLOSURE,0,fs->np-1)); +for(i=0;if->nups;i++){ +OpCode o=(func->upvalues[i].k==VLOCAL)?OP_MOVE:OP_GETUPVAL; +luaK_codeABC(fs,o,0,func->upvalues[i].info,0); +} +} +static void open_func(LexState*ls,FuncState*fs){ +lua_State*L=ls->L; +Proto*f=luaF_newproto(L); +fs->f=f; +fs->prev=ls->fs; +fs->ls=ls; +fs->L=L; +ls->fs=fs; +fs->pc=0; +fs->lasttarget=-1; +fs->jpc=(-1); +fs->freereg=0; +fs->nk=0; +fs->np=0; +fs->nlocvars=0; +fs->nactvar=0; +fs->bl=NULL; +f->source=ls->source; +f->maxstacksize=2; +fs->h=luaH_new(L,0,0); +sethvalue(L,L->top,fs->h); +incr_top(L); +setptvalue(L,L->top,f); +incr_top(L); +} +static void close_func(LexState*ls){ +lua_State*L=ls->L; +FuncState*fs=ls->fs; +Proto*f=fs->f; +removevars(ls,0); +luaK_ret(fs,0,0); +luaM_reallocvector(L,f->code,f->sizecode,fs->pc,Instruction); +f->sizecode=fs->pc; +luaM_reallocvector(L,f->lineinfo,f->sizelineinfo,fs->pc,int); +f->sizelineinfo=fs->pc; +luaM_reallocvector(L,f->k,f->sizek,fs->nk,TValue); +f->sizek=fs->nk; +luaM_reallocvector(L,f->p,f->sizep,fs->np,Proto*); +f->sizep=fs->np; +luaM_reallocvector(L,f->locvars,f->sizelocvars,fs->nlocvars,LocVar); +f->sizelocvars=fs->nlocvars; +luaM_reallocvector(L,f->upvalues,f->sizeupvalues,f->nups,TString*); +f->sizeupvalues=f->nups; +ls->fs=fs->prev; +if(fs)anchor_token(ls); +L->top-=2; +} +static Proto*luaY_parser(lua_State*L,ZIO*z,Mbuffer*buff,const char*name){ +struct LexState lexstate; +struct FuncState funcstate; +lexstate.buff=buff; +luaX_setinput(L,&lexstate,z,luaS_new(L,name)); +open_func(&lexstate,&funcstate); +funcstate.f->is_vararg=2; +luaX_next(&lexstate); +chunk(&lexstate); +check(&lexstate,TK_EOS); +close_func(&lexstate); +return funcstate.f; +} +static void field(LexState*ls,expdesc*v){ +FuncState*fs=ls->fs; +expdesc key; +luaK_exp2anyreg(fs,v); +luaX_next(ls); +checkname(ls,&key); +luaK_indexed(fs,v,&key); +} +static void yindex(LexState*ls,expdesc*v){ +luaX_next(ls); +expr(ls,v); +luaK_exp2val(ls->fs,v); +checknext(ls,']'); +} +struct ConsControl{ +expdesc v; +expdesc*t; +int nh; +int na; +int tostore; +}; +static void recfield(LexState*ls,struct ConsControl*cc){ +FuncState*fs=ls->fs; +int reg=ls->fs->freereg; +expdesc key,val; +int rkkey; +if(ls->t.token==TK_NAME){ +luaY_checklimit(fs,cc->nh,(INT_MAX-2),"items in a constructor"); +checkname(ls,&key); +} +else +yindex(ls,&key); +cc->nh++; +checknext(ls,'='); +rkkey=luaK_exp2RK(fs,&key); +expr(ls,&val); +luaK_codeABC(fs,OP_SETTABLE,cc->t->u.s.info,rkkey,luaK_exp2RK(fs,&val)); +fs->freereg=reg; +} +static void closelistfield(FuncState*fs,struct ConsControl*cc){ +if(cc->v.k==VVOID)return; +luaK_exp2nextreg(fs,&cc->v); +cc->v.k=VVOID; +if(cc->tostore==50){ +luaK_setlist(fs,cc->t->u.s.info,cc->na,cc->tostore); +cc->tostore=0; +} +} +static void lastlistfield(FuncState*fs,struct ConsControl*cc){ +if(cc->tostore==0)return; +if(hasmultret(cc->v.k)){ +luaK_setmultret(fs,&cc->v); +luaK_setlist(fs,cc->t->u.s.info,cc->na,(-1)); +cc->na--; +} +else{ +if(cc->v.k!=VVOID) +luaK_exp2nextreg(fs,&cc->v); +luaK_setlist(fs,cc->t->u.s.info,cc->na,cc->tostore); +} +} +static void listfield(LexState*ls,struct ConsControl*cc){ +expr(ls,&cc->v); +luaY_checklimit(ls->fs,cc->na,(INT_MAX-2),"items in a constructor"); +cc->na++; +cc->tostore++; +} +static void constructor(LexState*ls,expdesc*t){ +FuncState*fs=ls->fs; +int line=ls->linenumber; +int pc=luaK_codeABC(fs,OP_NEWTABLE,0,0,0); +struct ConsControl cc; +cc.na=cc.nh=cc.tostore=0; +cc.t=t; +init_exp(t,VRELOCABLE,pc); +init_exp(&cc.v,VVOID,0); +luaK_exp2nextreg(ls->fs,t); +checknext(ls,'{'); +do{ +if(ls->t.token=='}')break; +closelistfield(fs,&cc); +switch(ls->t.token){ +case TK_NAME:{ +luaX_lookahead(ls); +if(ls->lookahead.token!='=') +listfield(ls,&cc); +else +recfield(ls,&cc); +break; +} +case'[':{ +recfield(ls,&cc); +break; +} +default:{ +listfield(ls,&cc); +break; +} +} +}while(testnext(ls,',')||testnext(ls,';')); +check_match(ls,'}','{',line); +lastlistfield(fs,&cc); +SETARG_B(fs->f->code[pc],luaO_int2fb(cc.na)); +SETARG_C(fs->f->code[pc],luaO_int2fb(cc.nh)); +} +static void parlist(LexState*ls){ +FuncState*fs=ls->fs; +Proto*f=fs->f; +int nparams=0; +f->is_vararg=0; +if(ls->t.token!=')'){ +do{ +switch(ls->t.token){ +case TK_NAME:{ +new_localvar(ls,str_checkname(ls),nparams++); +break; +} +case TK_DOTS:{ +luaX_next(ls); +f->is_vararg|=2; +break; +} +default:luaX_syntaxerror(ls," or "LUA_QL("...")" expected"); +} +}while(!f->is_vararg&&testnext(ls,',')); +} +adjustlocalvars(ls,nparams); +f->numparams=cast_byte(fs->nactvar-(f->is_vararg&1)); +luaK_reserveregs(fs,fs->nactvar); +} +static void body(LexState*ls,expdesc*e,int needself,int line){ +FuncState new_fs; +open_func(ls,&new_fs); +new_fs.f->linedefined=line; +checknext(ls,'('); +if(needself){ +new_localvarliteral(ls,"self",0); +adjustlocalvars(ls,1); +} +parlist(ls); +checknext(ls,')'); +chunk(ls); +new_fs.f->lastlinedefined=ls->linenumber; +check_match(ls,TK_END,TK_FUNCTION,line); +close_func(ls); +pushclosure(ls,&new_fs,e); +} +static int explist1(LexState*ls,expdesc*v){ +int n=1; +expr(ls,v); +while(testnext(ls,',')){ +luaK_exp2nextreg(ls->fs,v); +expr(ls,v); +n++; +} +return n; +} +static void funcargs(LexState*ls,expdesc*f){ +FuncState*fs=ls->fs; +expdesc args; +int base,nparams; +int line=ls->linenumber; +switch(ls->t.token){ +case'(':{ +if(line!=ls->lastline) +luaX_syntaxerror(ls,"ambiguous syntax (function call x new statement)"); +luaX_next(ls); +if(ls->t.token==')') +args.k=VVOID; +else{ +explist1(ls,&args); +luaK_setmultret(fs,&args); +} +check_match(ls,')','(',line); +break; +} +case'{':{ +constructor(ls,&args); +break; +} +case TK_STRING:{ +codestring(ls,&args,ls->t.seminfo.ts); +luaX_next(ls); +break; +} +default:{ +luaX_syntaxerror(ls,"function arguments expected"); +return; +} +} +base=f->u.s.info; +if(hasmultret(args.k)) +nparams=(-1); +else{ +if(args.k!=VVOID) +luaK_exp2nextreg(fs,&args); +nparams=fs->freereg-(base+1); +} +init_exp(f,VCALL,luaK_codeABC(fs,OP_CALL,base,nparams+1,2)); +luaK_fixline(fs,line); +fs->freereg=base+1; +} +static void prefixexp(LexState*ls,expdesc*v){ +switch(ls->t.token){ +case'(':{ +int line=ls->linenumber; +luaX_next(ls); +expr(ls,v); +check_match(ls,')','(',line); +luaK_dischargevars(ls->fs,v); +return; +} +case TK_NAME:{ +singlevar(ls,v); +return; +} +default:{ +luaX_syntaxerror(ls,"unexpected symbol"); +return; +} +} +} +static void primaryexp(LexState*ls,expdesc*v){ +FuncState*fs=ls->fs; +prefixexp(ls,v); +for(;;){ +switch(ls->t.token){ +case'.':{ +field(ls,v); +break; +} +case'[':{ +expdesc key; +luaK_exp2anyreg(fs,v); +yindex(ls,&key); +luaK_indexed(fs,v,&key); +break; +} +case':':{ +expdesc key; +luaX_next(ls); +checkname(ls,&key); +luaK_self(fs,v,&key); +funcargs(ls,v); +break; +} +case'(':case TK_STRING:case'{':{ +luaK_exp2nextreg(fs,v); +funcargs(ls,v); +break; +} +default:return; +} +} +} +static void simpleexp(LexState*ls,expdesc*v){ +switch(ls->t.token){ +case TK_NUMBER:{ +init_exp(v,VKNUM,0); +v->u.nval=ls->t.seminfo.r; +break; +} +case TK_STRING:{ +codestring(ls,v,ls->t.seminfo.ts); +break; +} +case TK_NIL:{ +init_exp(v,VNIL,0); +break; +} +case TK_TRUE:{ +init_exp(v,VTRUE,0); +break; +} +case TK_FALSE:{ +init_exp(v,VFALSE,0); +break; +} +case TK_DOTS:{ +FuncState*fs=ls->fs; +check_condition(ls,fs->f->is_vararg, +"cannot use "LUA_QL("...")" outside a vararg function"); +fs->f->is_vararg&=~4; +init_exp(v,VVARARG,luaK_codeABC(fs,OP_VARARG,0,1,0)); +break; +} +case'{':{ +constructor(ls,v); +return; +} +case TK_FUNCTION:{ +luaX_next(ls); +body(ls,v,0,ls->linenumber); +return; +} +default:{ +primaryexp(ls,v); +return; +} +} +luaX_next(ls); +} +static UnOpr getunopr(int op){ +switch(op){ +case TK_NOT:return OPR_NOT; +case'-':return OPR_MINUS; +case'#':return OPR_LEN; +default:return OPR_NOUNOPR; +} +} +static BinOpr getbinopr(int op){ +switch(op){ +case'+':return OPR_ADD; +case'-':return OPR_SUB; +case'*':return OPR_MUL; +case'/':return OPR_DIV; +case'%':return OPR_MOD; +case'^':return OPR_POW; +case TK_CONCAT:return OPR_CONCAT; +case TK_NE:return OPR_NE; +case TK_EQ:return OPR_EQ; +case'<':return OPR_LT; +case TK_LE:return OPR_LE; +case'>':return OPR_GT; +case TK_GE:return OPR_GE; +case TK_AND:return OPR_AND; +case TK_OR:return OPR_OR; +default:return OPR_NOBINOPR; +} +} +static const struct{ +lu_byte left; +lu_byte right; +}priority[]={ +{6,6},{6,6},{7,7},{7,7},{7,7}, +{10,9},{5,4}, +{3,3},{3,3}, +{3,3},{3,3},{3,3},{3,3}, +{2,2},{1,1} +}; +static BinOpr subexpr(LexState*ls,expdesc*v,unsigned int limit){ +BinOpr op; +UnOpr uop; +enterlevel(ls); +uop=getunopr(ls->t.token); +if(uop!=OPR_NOUNOPR){ +luaX_next(ls); +subexpr(ls,v,8); +luaK_prefix(ls->fs,uop,v); +} +else simpleexp(ls,v); +op=getbinopr(ls->t.token); +while(op!=OPR_NOBINOPR&&priority[op].left>limit){ +expdesc v2; +BinOpr nextop; +luaX_next(ls); +luaK_infix(ls->fs,op,v); +nextop=subexpr(ls,&v2,priority[op].right); +luaK_posfix(ls->fs,op,v,&v2); +op=nextop; +} +leavelevel(ls); +return op; +} +static void expr(LexState*ls,expdesc*v){ +subexpr(ls,v,0); +} +static int block_follow(int token){ +switch(token){ +case TK_ELSE:case TK_ELSEIF:case TK_END: +case TK_UNTIL:case TK_EOS: +return 1; +default:return 0; +} +} +static void block(LexState*ls){ +FuncState*fs=ls->fs; +BlockCnt bl; +enterblock(fs,&bl,0); +chunk(ls); +leaveblock(fs); +} +struct LHS_assign{ +struct LHS_assign*prev; +expdesc v; +}; +static void check_conflict(LexState*ls,struct LHS_assign*lh,expdesc*v){ +FuncState*fs=ls->fs; +int extra=fs->freereg; +int conflict=0; +for(;lh;lh=lh->prev){ +if(lh->v.k==VINDEXED){ +if(lh->v.u.s.info==v->u.s.info){ +conflict=1; +lh->v.u.s.info=extra; +} +if(lh->v.u.s.aux==v->u.s.info){ +conflict=1; +lh->v.u.s.aux=extra; +} +} +} +if(conflict){ +luaK_codeABC(fs,OP_MOVE,fs->freereg,v->u.s.info,0); +luaK_reserveregs(fs,1); +} +} +static void assignment(LexState*ls,struct LHS_assign*lh,int nvars){ +expdesc e; +check_condition(ls,VLOCAL<=lh->v.k&&lh->v.k<=VINDEXED, +"syntax error"); +if(testnext(ls,',')){ +struct LHS_assign nv; +nv.prev=lh; +primaryexp(ls,&nv.v); +if(nv.v.k==VLOCAL) +check_conflict(ls,lh,&nv.v); +luaY_checklimit(ls->fs,nvars,200-ls->L->nCcalls, +"variables in assignment"); +assignment(ls,&nv,nvars+1); +} +else{ +int nexps; +checknext(ls,'='); +nexps=explist1(ls,&e); +if(nexps!=nvars){ +adjust_assign(ls,nvars,nexps,&e); +if(nexps>nvars) +ls->fs->freereg-=nexps-nvars; +} +else{ +luaK_setoneret(ls->fs,&e); +luaK_storevar(ls->fs,&lh->v,&e); +return; +} +} +init_exp(&e,VNONRELOC,ls->fs->freereg-1); +luaK_storevar(ls->fs,&lh->v,&e); +} +static int cond(LexState*ls){ +expdesc v; +expr(ls,&v); +if(v.k==VNIL)v.k=VFALSE; +luaK_goiftrue(ls->fs,&v); +return v.f; +} +static void breakstat(LexState*ls){ +FuncState*fs=ls->fs; +BlockCnt*bl=fs->bl; +int upval=0; +while(bl&&!bl->isbreakable){ +upval|=bl->upval; +bl=bl->previous; +} +if(!bl) +luaX_syntaxerror(ls,"no loop to break"); +if(upval) +luaK_codeABC(fs,OP_CLOSE,bl->nactvar,0,0); +luaK_concat(fs,&bl->breaklist,luaK_jump(fs)); +} +static void whilestat(LexState*ls,int line){ +FuncState*fs=ls->fs; +int whileinit; +int condexit; +BlockCnt bl; +luaX_next(ls); +whileinit=luaK_getlabel(fs); +condexit=cond(ls); +enterblock(fs,&bl,1); +checknext(ls,TK_DO); +block(ls); +luaK_patchlist(fs,luaK_jump(fs),whileinit); +check_match(ls,TK_END,TK_WHILE,line); +leaveblock(fs); +luaK_patchtohere(fs,condexit); +} +static void repeatstat(LexState*ls,int line){ +int condexit; +FuncState*fs=ls->fs; +int repeat_init=luaK_getlabel(fs); +BlockCnt bl1,bl2; +enterblock(fs,&bl1,1); +enterblock(fs,&bl2,0); +luaX_next(ls); +chunk(ls); +check_match(ls,TK_UNTIL,TK_REPEAT,line); +condexit=cond(ls); +if(!bl2.upval){ +leaveblock(fs); +luaK_patchlist(ls->fs,condexit,repeat_init); +} +else{ +breakstat(ls); +luaK_patchtohere(ls->fs,condexit); +leaveblock(fs); +luaK_patchlist(ls->fs,luaK_jump(fs),repeat_init); +} +leaveblock(fs); +} +static int exp1(LexState*ls){ +expdesc e; +int k; +expr(ls,&e); +k=e.k; +luaK_exp2nextreg(ls->fs,&e); +return k; +} +static void forbody(LexState*ls,int base,int line,int nvars,int isnum){ +BlockCnt bl; +FuncState*fs=ls->fs; +int prep,endfor; +adjustlocalvars(ls,3); +checknext(ls,TK_DO); +prep=isnum?luaK_codeAsBx(fs,OP_FORPREP,base,(-1)):luaK_jump(fs); +enterblock(fs,&bl,0); +adjustlocalvars(ls,nvars); +luaK_reserveregs(fs,nvars); +block(ls); +leaveblock(fs); +luaK_patchtohere(fs,prep); +endfor=(isnum)?luaK_codeAsBx(fs,OP_FORLOOP,base,(-1)): +luaK_codeABC(fs,OP_TFORLOOP,base,0,nvars); +luaK_fixline(fs,line); +luaK_patchlist(fs,(isnum?endfor:luaK_jump(fs)),prep+1); +} +static void fornum(LexState*ls,TString*varname,int line){ +FuncState*fs=ls->fs; +int base=fs->freereg; +new_localvarliteral(ls,"(for index)",0); +new_localvarliteral(ls,"(for limit)",1); +new_localvarliteral(ls,"(for step)",2); +new_localvar(ls,varname,3); +checknext(ls,'='); +exp1(ls); +checknext(ls,','); +exp1(ls); +if(testnext(ls,',')) +exp1(ls); +else{ +luaK_codeABx(fs,OP_LOADK,fs->freereg,luaK_numberK(fs,1)); +luaK_reserveregs(fs,1); +} +forbody(ls,base,line,1,1); +} +static void forlist(LexState*ls,TString*indexname){ +FuncState*fs=ls->fs; +expdesc e; +int nvars=0; +int line; +int base=fs->freereg; +new_localvarliteral(ls,"(for generator)",nvars++); +new_localvarliteral(ls,"(for state)",nvars++); +new_localvarliteral(ls,"(for control)",nvars++); +new_localvar(ls,indexname,nvars++); +while(testnext(ls,',')) +new_localvar(ls,str_checkname(ls),nvars++); +checknext(ls,TK_IN); +line=ls->linenumber; +adjust_assign(ls,3,explist1(ls,&e),&e); +luaK_checkstack(fs,3); +forbody(ls,base,line,nvars-3,0); +} +static void forstat(LexState*ls,int line){ +FuncState*fs=ls->fs; +TString*varname; +BlockCnt bl; +enterblock(fs,&bl,1); +luaX_next(ls); +varname=str_checkname(ls); +switch(ls->t.token){ +case'=':fornum(ls,varname,line);break; +case',':case TK_IN:forlist(ls,varname);break; +default:luaX_syntaxerror(ls,LUA_QL("=")" or "LUA_QL("in")" expected"); +} +check_match(ls,TK_END,TK_FOR,line); +leaveblock(fs); +} +static int test_then_block(LexState*ls){ +int condexit; +luaX_next(ls); +condexit=cond(ls); +checknext(ls,TK_THEN); +block(ls); +return condexit; +} +static void ifstat(LexState*ls,int line){ +FuncState*fs=ls->fs; +int flist; +int escapelist=(-1); +flist=test_then_block(ls); +while(ls->t.token==TK_ELSEIF){ +luaK_concat(fs,&escapelist,luaK_jump(fs)); +luaK_patchtohere(fs,flist); +flist=test_then_block(ls); +} +if(ls->t.token==TK_ELSE){ +luaK_concat(fs,&escapelist,luaK_jump(fs)); +luaK_patchtohere(fs,flist); +luaX_next(ls); +block(ls); +} +else +luaK_concat(fs,&escapelist,flist); +luaK_patchtohere(fs,escapelist); +check_match(ls,TK_END,TK_IF,line); +} +static void localfunc(LexState*ls){ +expdesc v,b; +FuncState*fs=ls->fs; +new_localvar(ls,str_checkname(ls),0); +init_exp(&v,VLOCAL,fs->freereg); +luaK_reserveregs(fs,1); +adjustlocalvars(ls,1); +body(ls,&b,0,ls->linenumber); +luaK_storevar(fs,&v,&b); +getlocvar(fs,fs->nactvar-1).startpc=fs->pc; +} +static void localstat(LexState*ls){ +int nvars=0; +int nexps; +expdesc e; +do{ +new_localvar(ls,str_checkname(ls),nvars++); +}while(testnext(ls,',')); +if(testnext(ls,'=')) +nexps=explist1(ls,&e); +else{ +e.k=VVOID; +nexps=0; +} +adjust_assign(ls,nvars,nexps,&e); +adjustlocalvars(ls,nvars); +} +static int funcname(LexState*ls,expdesc*v){ +int needself=0; +singlevar(ls,v); +while(ls->t.token=='.') +field(ls,v); +if(ls->t.token==':'){ +needself=1; +field(ls,v); +} +return needself; +} +static void funcstat(LexState*ls,int line){ +int needself; +expdesc v,b; +luaX_next(ls); +needself=funcname(ls,&v); +body(ls,&b,needself,line); +luaK_storevar(ls->fs,&v,&b); +luaK_fixline(ls->fs,line); +} +static void exprstat(LexState*ls){ +FuncState*fs=ls->fs; +struct LHS_assign v; +primaryexp(ls,&v.v); +if(v.v.k==VCALL) +SETARG_C(getcode(fs,&v.v),1); +else{ +v.prev=NULL; +assignment(ls,&v,1); +} +} +static void retstat(LexState*ls){ +FuncState*fs=ls->fs; +expdesc e; +int first,nret; +luaX_next(ls); +if(block_follow(ls->t.token)||ls->t.token==';') +first=nret=0; +else{ +nret=explist1(ls,&e); +if(hasmultret(e.k)){ +luaK_setmultret(fs,&e); +if(e.k==VCALL&&nret==1){ +SET_OPCODE(getcode(fs,&e),OP_TAILCALL); +} +first=fs->nactvar; +nret=(-1); +} +else{ +if(nret==1) +first=luaK_exp2anyreg(fs,&e); +else{ +luaK_exp2nextreg(fs,&e); +first=fs->nactvar; +} +} +} +luaK_ret(fs,first,nret); +} +static int statement(LexState*ls){ +int line=ls->linenumber; +switch(ls->t.token){ +case TK_IF:{ +ifstat(ls,line); +return 0; +} +case TK_WHILE:{ +whilestat(ls,line); +return 0; +} +case TK_DO:{ +luaX_next(ls); +block(ls); +check_match(ls,TK_END,TK_DO,line); +return 0; +} +case TK_FOR:{ +forstat(ls,line); +return 0; +} +case TK_REPEAT:{ +repeatstat(ls,line); +return 0; +} +case TK_FUNCTION:{ +funcstat(ls,line); +return 0; +} +case TK_LOCAL:{ +luaX_next(ls); +if(testnext(ls,TK_FUNCTION)) +localfunc(ls); +else +localstat(ls); +return 0; +} +case TK_RETURN:{ +retstat(ls); +return 1; +} +case TK_BREAK:{ +luaX_next(ls); +breakstat(ls); +return 1; +} +default:{ +exprstat(ls); +return 0; +} +} +} +static void chunk(LexState*ls){ +int islast=0; +enterlevel(ls); +while(!islast&&!block_follow(ls->t.token)){ +islast=statement(ls); +testnext(ls,';'); +ls->fs->freereg=ls->fs->nactvar; +} +leavelevel(ls); +} +static const TValue*luaV_tonumber(const TValue*obj,TValue*n){ +lua_Number num; +if(ttisnumber(obj))return obj; +if(ttisstring(obj)&&luaO_str2d(svalue(obj),&num)){ +setnvalue(n,num); +return n; +} +else +return NULL; +} +static int luaV_tostring(lua_State*L,StkId obj){ +if(!ttisnumber(obj)) +return 0; +else{ +char s[32]; +lua_Number n=nvalue(obj); +lua_number2str(s,n); +setsvalue(L,obj,luaS_new(L,s)); +return 1; +} +} +static void callTMres(lua_State*L,StkId res,const TValue*f, +const TValue*p1,const TValue*p2){ +ptrdiff_t result=savestack(L,res); +setobj(L,L->top,f); +setobj(L,L->top+1,p1); +setobj(L,L->top+2,p2); +luaD_checkstack(L,3); +L->top+=3; +luaD_call(L,L->top-3,1); +res=restorestack(L,result); +L->top--; +setobj(L,res,L->top); +} +static void callTM(lua_State*L,const TValue*f,const TValue*p1, +const TValue*p2,const TValue*p3){ +setobj(L,L->top,f); +setobj(L,L->top+1,p1); +setobj(L,L->top+2,p2); +setobj(L,L->top+3,p3); +luaD_checkstack(L,4); +L->top+=4; +luaD_call(L,L->top-4,0); +} +static void luaV_gettable(lua_State*L,const TValue*t,TValue*key,StkId val){ +int loop; +for(loop=0;loop<100;loop++){ +const TValue*tm; +if(ttistable(t)){ +Table*h=hvalue(t); +const TValue*res=luaH_get(h,key); +if(!ttisnil(res)|| +(tm=fasttm(L,h->metatable,TM_INDEX))==NULL){ +setobj(L,val,res); +return; +} +} +else if(ttisnil(tm=luaT_gettmbyobj(L,t,TM_INDEX))) +luaG_typeerror(L,t,"index"); +if(ttisfunction(tm)){ +callTMres(L,val,tm,t,key); +return; +} +t=tm; +} +luaG_runerror(L,"loop in gettable"); +} +static void luaV_settable(lua_State*L,const TValue*t,TValue*key,StkId val){ +int loop; +TValue temp; +for(loop=0;loop<100;loop++){ +const TValue*tm; +if(ttistable(t)){ +Table*h=hvalue(t); +TValue*oldval=luaH_set(L,h,key); +if(!ttisnil(oldval)|| +(tm=fasttm(L,h->metatable,TM_NEWINDEX))==NULL){ +setobj(L,oldval,val); +h->flags=0; +luaC_barriert(L,h,val); +return; +} +} +else if(ttisnil(tm=luaT_gettmbyobj(L,t,TM_NEWINDEX))) +luaG_typeerror(L,t,"index"); +if(ttisfunction(tm)){ +callTM(L,tm,t,key,val); +return; +} +setobj(L,&temp,tm); +t=&temp; +} +luaG_runerror(L,"loop in settable"); +} +static int call_binTM(lua_State*L,const TValue*p1,const TValue*p2, +StkId res,TMS event){ +const TValue*tm=luaT_gettmbyobj(L,p1,event); +if(ttisnil(tm)) +tm=luaT_gettmbyobj(L,p2,event); +if(ttisnil(tm))return 0; +callTMres(L,res,tm,p1,p2); +return 1; +} +static const TValue*get_compTM(lua_State*L,Table*mt1,Table*mt2, +TMS event){ +const TValue*tm1=fasttm(L,mt1,event); +const TValue*tm2; +if(tm1==NULL)return NULL; +if(mt1==mt2)return tm1; +tm2=fasttm(L,mt2,event); +if(tm2==NULL)return NULL; +if(luaO_rawequalObj(tm1,tm2)) +return tm1; +return NULL; +} +static int call_orderTM(lua_State*L,const TValue*p1,const TValue*p2, +TMS event){ +const TValue*tm1=luaT_gettmbyobj(L,p1,event); +const TValue*tm2; +if(ttisnil(tm1))return-1; +tm2=luaT_gettmbyobj(L,p2,event); +if(!luaO_rawequalObj(tm1,tm2)) +return-1; +callTMres(L,L->top,tm1,p1,p2); +return!l_isfalse(L->top); +} +static int l_strcmp(const TString*ls,const TString*rs){ +const char*l=getstr(ls); +size_t ll=ls->tsv.len; +const char*r=getstr(rs); +size_t lr=rs->tsv.len; +for(;;){ +int temp=strcoll(l,r); +if(temp!=0)return temp; +else{ +size_t len=strlen(l); +if(len==lr) +return(len==ll)?0:1; +else if(len==ll) +return-1; +len++; +l+=len;ll-=len;r+=len;lr-=len; +} +} +} +static int luaV_lessthan(lua_State*L,const TValue*l,const TValue*r){ +int res; +if(ttype(l)!=ttype(r)) +return luaG_ordererror(L,l,r); +else if(ttisnumber(l)) +return luai_numlt(nvalue(l),nvalue(r)); +else if(ttisstring(l)) +return l_strcmp(rawtsvalue(l),rawtsvalue(r))<0; +else if((res=call_orderTM(L,l,r,TM_LT))!=-1) +return res; +return luaG_ordererror(L,l,r); +} +static int lessequal(lua_State*L,const TValue*l,const TValue*r){ +int res; +if(ttype(l)!=ttype(r)) +return luaG_ordererror(L,l,r); +else if(ttisnumber(l)) +return luai_numle(nvalue(l),nvalue(r)); +else if(ttisstring(l)) +return l_strcmp(rawtsvalue(l),rawtsvalue(r))<=0; +else if((res=call_orderTM(L,l,r,TM_LE))!=-1) +return res; +else if((res=call_orderTM(L,r,l,TM_LT))!=-1) +return!res; +return luaG_ordererror(L,l,r); +} +static int luaV_equalval(lua_State*L,const TValue*t1,const TValue*t2){ +const TValue*tm; +switch(ttype(t1)){ +case 0:return 1; +case 3:return luai_numeq(nvalue(t1),nvalue(t2)); +case 1:return bvalue(t1)==bvalue(t2); +case 2:return pvalue(t1)==pvalue(t2); +case 7:{ +if(uvalue(t1)==uvalue(t2))return 1; +tm=get_compTM(L,uvalue(t1)->metatable,uvalue(t2)->metatable, +TM_EQ); +break; +} +case 5:{ +if(hvalue(t1)==hvalue(t2))return 1; +tm=get_compTM(L,hvalue(t1)->metatable,hvalue(t2)->metatable,TM_EQ); +break; +} +default:return gcvalue(t1)==gcvalue(t2); +} +if(tm==NULL)return 0; +callTMres(L,L->top,tm,t1,t2); +return!l_isfalse(L->top); +} +static void luaV_concat(lua_State*L,int total,int last){ +do{ +StkId top=L->base+last+1; +int n=2; +if(!(ttisstring(top-2)||ttisnumber(top-2))||!tostring(L,top-1)){ +if(!call_binTM(L,top-2,top-1,top-2,TM_CONCAT)) +luaG_concaterror(L,top-2,top-1); +}else if(tsvalue(top-1)->len==0) +(void)tostring(L,top-2); +else{ +size_t tl=tsvalue(top-1)->len; +char*buffer; +int i; +for(n=1;nlen; +if(l>=((size_t)(~(size_t)0)-2)-tl)luaG_runerror(L,"string length overflow"); +tl+=l; +} +buffer=luaZ_openspace(L,&G(L)->buff,tl); +tl=0; +for(i=n;i>0;i--){ +size_t l=tsvalue(top-i)->len; +memcpy(buffer+tl,svalue(top-i),l); +tl+=l; +} +setsvalue(L,top-n,luaS_newlstr(L,buffer,tl)); +} +total-=n-1; +last-=n-1; +}while(total>1); +} +static void Arith(lua_State*L,StkId ra,const TValue*rb, +const TValue*rc,TMS op){ +TValue tempb,tempc; +const TValue*b,*c; +if((b=luaV_tonumber(rb,&tempb))!=NULL&& +(c=luaV_tonumber(rc,&tempc))!=NULL){ +lua_Number nb=nvalue(b),nc=nvalue(c); +switch(op){ +case TM_ADD:setnvalue(ra,luai_numadd(nb,nc));break; +case TM_SUB:setnvalue(ra,luai_numsub(nb,nc));break; +case TM_MUL:setnvalue(ra,luai_nummul(nb,nc));break; +case TM_DIV:setnvalue(ra,luai_numdiv(nb,nc));break; +case TM_MOD:setnvalue(ra,luai_nummod(nb,nc));break; +case TM_POW:setnvalue(ra,luai_numpow(nb,nc));break; +case TM_UNM:setnvalue(ra,luai_numunm(nb));break; +default:break; +} +} +else if(!call_binTM(L,rb,rc,ra,op)) +luaG_aritherror(L,rb,rc); +} +#define runtime_check(L,c){if(!(c))break;} +#define RA(i)(base+GETARG_A(i)) +#define RB(i)check_exp(getBMode(GET_OPCODE(i))==OpArgR,base+GETARG_B(i)) +#define RKB(i)check_exp(getBMode(GET_OPCODE(i))==OpArgK,ISK(GETARG_B(i))?k+INDEXK(GETARG_B(i)):base+GETARG_B(i)) +#define RKC(i)check_exp(getCMode(GET_OPCODE(i))==OpArgK,ISK(GETARG_C(i))?k+INDEXK(GETARG_C(i)):base+GETARG_C(i)) +#define KBx(i)check_exp(getBMode(GET_OPCODE(i))==OpArgK,k+GETARG_Bx(i)) +#define dojump(L,pc,i){(pc)+=(i);} +#define Protect(x){L->savedpc=pc;{x;};base=L->base;} +#define arith_op(op,tm){TValue*rb=RKB(i);TValue*rc=RKC(i);if(ttisnumber(rb)&&ttisnumber(rc)){lua_Number nb=nvalue(rb),nc=nvalue(rc);setnvalue(ra,op(nb,nc));}else Protect(Arith(L,ra,rb,rc,tm));} +static void luaV_execute(lua_State*L,int nexeccalls){ +LClosure*cl; +StkId base; +TValue*k; +const Instruction*pc; +reentry: +pc=L->savedpc; +cl=&clvalue(L->ci->func)->l; +base=L->base; +k=cl->p->k; +for(;;){ +const Instruction i=*pc++; +StkId ra; +ra=RA(i); +switch(GET_OPCODE(i)){ +case OP_MOVE:{ +setobj(L,ra,RB(i)); +continue; +} +case OP_LOADK:{ +setobj(L,ra,KBx(i)); +continue; +} +case OP_LOADBOOL:{ +setbvalue(ra,GETARG_B(i)); +if(GETARG_C(i))pc++; +continue; +} +case OP_LOADNIL:{ +TValue*rb=RB(i); +do{ +setnilvalue(rb--); +}while(rb>=ra); +continue; +} +case OP_GETUPVAL:{ +int b=GETARG_B(i); +setobj(L,ra,cl->upvals[b]->v); +continue; +} +case OP_GETGLOBAL:{ +TValue g; +TValue*rb=KBx(i); +sethvalue(L,&g,cl->env); +Protect(luaV_gettable(L,&g,rb,ra)); +continue; +} +case OP_GETTABLE:{ +Protect(luaV_gettable(L,RB(i),RKC(i),ra)); +continue; +} +case OP_SETGLOBAL:{ +TValue g; +sethvalue(L,&g,cl->env); +Protect(luaV_settable(L,&g,KBx(i),ra)); +continue; +} +case OP_SETUPVAL:{ +UpVal*uv=cl->upvals[GETARG_B(i)]; +setobj(L,uv->v,ra); +luaC_barrier(L,uv,ra); +continue; +} +case OP_SETTABLE:{ +Protect(luaV_settable(L,ra,RKB(i),RKC(i))); +continue; +} +case OP_NEWTABLE:{ +int b=GETARG_B(i); +int c=GETARG_C(i); +sethvalue(L,ra,luaH_new(L,luaO_fb2int(b),luaO_fb2int(c))); +Protect(luaC_checkGC(L)); +continue; +} +case OP_SELF:{ +StkId rb=RB(i); +setobj(L,ra+1,rb); +Protect(luaV_gettable(L,rb,RKC(i),ra)); +continue; +} +case OP_ADD:{ +arith_op(luai_numadd,TM_ADD); +continue; +} +case OP_SUB:{ +arith_op(luai_numsub,TM_SUB); +continue; +} +case OP_MUL:{ +arith_op(luai_nummul,TM_MUL); +continue; +} +case OP_DIV:{ +arith_op(luai_numdiv,TM_DIV); +continue; +} +case OP_MOD:{ +arith_op(luai_nummod,TM_MOD); +continue; +} +case OP_POW:{ +arith_op(luai_numpow,TM_POW); +continue; +} +case OP_UNM:{ +TValue*rb=RB(i); +if(ttisnumber(rb)){ +lua_Number nb=nvalue(rb); +setnvalue(ra,luai_numunm(nb)); +} +else{ +Protect(Arith(L,ra,rb,rb,TM_UNM)); +} +continue; +} +case OP_NOT:{ +int res=l_isfalse(RB(i)); +setbvalue(ra,res); +continue; +} +case OP_LEN:{ +const TValue*rb=RB(i); +switch(ttype(rb)){ +case 5:{ +setnvalue(ra,cast_num(luaH_getn(hvalue(rb)))); +break; +} +case 4:{ +setnvalue(ra,cast_num(tsvalue(rb)->len)); +break; +} +default:{ +Protect( +if(!call_binTM(L,rb,(&luaO_nilobject_),ra,TM_LEN)) +luaG_typeerror(L,rb,"get length of"); +) +} +} +continue; +} +case OP_CONCAT:{ +int b=GETARG_B(i); +int c=GETARG_C(i); +Protect(luaV_concat(L,c-b+1,c);luaC_checkGC(L)); +setobj(L,RA(i),base+b); +continue; +} +case OP_JMP:{ +dojump(L,pc,GETARG_sBx(i)); +continue; +} +case OP_EQ:{ +TValue*rb=RKB(i); +TValue*rc=RKC(i); +Protect( +if(equalobj(L,rb,rc)==GETARG_A(i)) +dojump(L,pc,GETARG_sBx(*pc)); +) +pc++; +continue; +} +case OP_LT:{ +Protect( +if(luaV_lessthan(L,RKB(i),RKC(i))==GETARG_A(i)) +dojump(L,pc,GETARG_sBx(*pc)); +) +pc++; +continue; +} +case OP_LE:{ +Protect( +if(lessequal(L,RKB(i),RKC(i))==GETARG_A(i)) +dojump(L,pc,GETARG_sBx(*pc)); +) +pc++; +continue; +} +case OP_TEST:{ +if(l_isfalse(ra)!=GETARG_C(i)) +dojump(L,pc,GETARG_sBx(*pc)); +pc++; +continue; +} +case OP_TESTSET:{ +TValue*rb=RB(i); +if(l_isfalse(rb)!=GETARG_C(i)){ +setobj(L,ra,rb); +dojump(L,pc,GETARG_sBx(*pc)); +} +pc++; +continue; +} +case OP_CALL:{ +int b=GETARG_B(i); +int nresults=GETARG_C(i)-1; +if(b!=0)L->top=ra+b; +L->savedpc=pc; +switch(luaD_precall(L,ra,nresults)){ +case 0:{ +nexeccalls++; +goto reentry; +} +case 1:{ +if(nresults>=0)L->top=L->ci->top; +base=L->base; +continue; +} +default:{ +return; +} +} +} +case OP_TAILCALL:{ +int b=GETARG_B(i); +if(b!=0)L->top=ra+b; +L->savedpc=pc; +switch(luaD_precall(L,ra,(-1))){ +case 0:{ +CallInfo*ci=L->ci-1; +int aux; +StkId func=ci->func; +StkId pfunc=(ci+1)->func; +if(L->openupval)luaF_close(L,ci->base); +L->base=ci->base=ci->func+((ci+1)->base-pfunc); +for(aux=0;pfunc+auxtop;aux++) +setobj(L,func+aux,pfunc+aux); +ci->top=L->top=func+aux; +ci->savedpc=L->savedpc; +ci->tailcalls++; +L->ci--; +goto reentry; +} +case 1:{ +base=L->base; +continue; +} +default:{ +return; +} +} +} +case OP_RETURN:{ +int b=GETARG_B(i); +if(b!=0)L->top=ra+b-1; +if(L->openupval)luaF_close(L,base); +L->savedpc=pc; +b=luaD_poscall(L,ra); +if(--nexeccalls==0) +return; +else{ +if(b)L->top=L->ci->top; +goto reentry; +} +} +case OP_FORLOOP:{ +lua_Number step=nvalue(ra+2); +lua_Number idx=luai_numadd(nvalue(ra),step); +lua_Number limit=nvalue(ra+1); +if(luai_numlt(0,step)?luai_numle(idx,limit) +:luai_numle(limit,idx)){ +dojump(L,pc,GETARG_sBx(i)); +setnvalue(ra,idx); +setnvalue(ra+3,idx); +} +continue; +} +case OP_FORPREP:{ +const TValue*init=ra; +const TValue*plimit=ra+1; +const TValue*pstep=ra+2; +L->savedpc=pc; +if(!tonumber(init,ra)) +luaG_runerror(L,LUA_QL("for")" initial value must be a number"); +else if(!tonumber(plimit,ra+1)) +luaG_runerror(L,LUA_QL("for")" limit must be a number"); +else if(!tonumber(pstep,ra+2)) +luaG_runerror(L,LUA_QL("for")" step must be a number"); +setnvalue(ra,luai_numsub(nvalue(ra),nvalue(pstep))); +dojump(L,pc,GETARG_sBx(i)); +continue; +} +case OP_TFORLOOP:{ +StkId cb=ra+3; +setobj(L,cb+2,ra+2); +setobj(L,cb+1,ra+1); +setobj(L,cb,ra); +L->top=cb+3; +Protect(luaD_call(L,cb,GETARG_C(i))); +L->top=L->ci->top; +cb=RA(i)+3; +if(!ttisnil(cb)){ +setobj(L,cb-1,cb); +dojump(L,pc,GETARG_sBx(*pc)); +} +pc++; +continue; +} +case OP_SETLIST:{ +int n=GETARG_B(i); +int c=GETARG_C(i); +int last; +Table*h; +if(n==0){ +n=cast_int(L->top-ra)-1; +L->top=L->ci->top; +} +if(c==0)c=cast_int(*pc++); +runtime_check(L,ttistable(ra)); +h=hvalue(ra); +last=((c-1)*50)+n; +if(last>h->sizearray) +luaH_resizearray(L,h,last); +for(;n>0;n--){ +TValue*val=ra+n; +setobj(L,luaH_setnum(L,h,last--),val); +luaC_barriert(L,h,val); +} +continue; +} +case OP_CLOSE:{ +luaF_close(L,ra); +continue; +} +case OP_CLOSURE:{ +Proto*p; +Closure*ncl; +int nup,j; +p=cl->p->p[GETARG_Bx(i)]; +nup=p->nups; +ncl=luaF_newLclosure(L,nup,cl->env); +ncl->l.p=p; +for(j=0;jl.upvals[j]=cl->upvals[GETARG_B(*pc)]; +else{ +ncl->l.upvals[j]=luaF_findupval(L,base+GETARG_B(*pc)); +} +} +setclvalue(L,ra,ncl); +Protect(luaC_checkGC(L)); +continue; +} +case OP_VARARG:{ +int b=GETARG_B(i)-1; +int j; +CallInfo*ci=L->ci; +int n=cast_int(ci->base-ci->func)-cl->p->numparams-1; +if(b==(-1)){ +Protect(luaD_checkstack(L,n)); +ra=RA(i); +b=n; +L->top=ra+n; +} +for(j=0;jbase-n+j); +} +else{ +setnilvalue(ra+j); +} +} +continue; +} +} +} +} +#define api_checknelems(L,n)luai_apicheck(L,(n)<=(L->top-L->base)) +#define api_checkvalidindex(L,i)luai_apicheck(L,(i)!=(&luaO_nilobject_)) +#define api_incr_top(L){luai_apicheck(L,L->topci->top);L->top++;} +static TValue*index2adr(lua_State*L,int idx){ +if(idx>0){ +TValue*o=L->base+(idx-1); +luai_apicheck(L,idx<=L->ci->top-L->base); +if(o>=L->top)return cast(TValue*,(&luaO_nilobject_)); +else return o; +} +else if(idx>(-10000)){ +luai_apicheck(L,idx!=0&&-idx<=L->top-L->base); +return L->top+idx; +} +else switch(idx){ +case(-10000):return registry(L); +case(-10001):{ +Closure*func=curr_func(L); +sethvalue(L,&L->env,func->c.env); +return&L->env; +} +case(-10002):return gt(L); +default:{ +Closure*func=curr_func(L); +idx=(-10002)-idx; +return(idx<=func->c.nupvalues) +?&func->c.upvalue[idx-1] +:cast(TValue*,(&luaO_nilobject_)); +} +} +} +static Table*getcurrenv(lua_State*L){ +if(L->ci==L->base_ci) +return hvalue(gt(L)); +else{ +Closure*func=curr_func(L); +return func->c.env; +} +} +static int lua_checkstack(lua_State*L,int size){ +int res=1; +if(size>8000||(L->top-L->base+size)>8000) +res=0; +else if(size>0){ +luaD_checkstack(L,size); +if(L->ci->toptop+size) +L->ci->top=L->top+size; +} +return res; +} +static lua_CFunction lua_atpanic(lua_State*L,lua_CFunction panicf){ +lua_CFunction old; +old=G(L)->panic; +G(L)->panic=panicf; +return old; +} +static int lua_gettop(lua_State*L){ +return cast_int(L->top-L->base); +} +static void lua_settop(lua_State*L,int idx){ +if(idx>=0){ +luai_apicheck(L,idx<=L->stack_last-L->base); +while(L->topbase+idx) +setnilvalue(L->top++); +L->top=L->base+idx; +} +else{ +luai_apicheck(L,-(idx+1)<=(L->top-L->base)); +L->top+=idx+1; +} +} +static void lua_remove(lua_State*L,int idx){ +StkId p; +p=index2adr(L,idx); +api_checkvalidindex(L,p); +while(++ptop)setobj(L,p-1,p); +L->top--; +} +static void lua_insert(lua_State*L,int idx){ +StkId p; +StkId q; +p=index2adr(L,idx); +api_checkvalidindex(L,p); +for(q=L->top;q>p;q--)setobj(L,q,q-1); +setobj(L,p,L->top); +} +static void lua_replace(lua_State*L,int idx){ +StkId o; +if(idx==(-10001)&&L->ci==L->base_ci) +luaG_runerror(L,"no calling environment"); +api_checknelems(L,1); +o=index2adr(L,idx); +api_checkvalidindex(L,o); +if(idx==(-10001)){ +Closure*func=curr_func(L); +luai_apicheck(L,ttistable(L->top-1)); +func->c.env=hvalue(L->top-1); +luaC_barrier(L,func,L->top-1); +} +else{ +setobj(L,o,L->top-1); +if(idx<(-10002)) +luaC_barrier(L,curr_func(L),L->top-1); +} +L->top--; +} +static void lua_pushvalue(lua_State*L,int idx){ +setobj(L,L->top,index2adr(L,idx)); +api_incr_top(L); +} +static int lua_type(lua_State*L,int idx){ +StkId o=index2adr(L,idx); +return(o==(&luaO_nilobject_))?(-1):ttype(o); +} +static const char*lua_typename(lua_State*L,int t){ +UNUSED(L); +return(t==(-1))?"no value":luaT_typenames[t]; +} +static int lua_iscfunction(lua_State*L,int idx){ +StkId o=index2adr(L,idx); +return iscfunction(o); +} +static int lua_isnumber(lua_State*L,int idx){ +TValue n; +const TValue*o=index2adr(L,idx); +return tonumber(o,&n); +} +static int lua_isstring(lua_State*L,int idx){ +int t=lua_type(L,idx); +return(t==4||t==3); +} +static int lua_rawequal(lua_State*L,int index1,int index2){ +StkId o1=index2adr(L,index1); +StkId o2=index2adr(L,index2); +return(o1==(&luaO_nilobject_)||o2==(&luaO_nilobject_))?0 +:luaO_rawequalObj(o1,o2); +} +static int lua_lessthan(lua_State*L,int index1,int index2){ +StkId o1,o2; +int i; +o1=index2adr(L,index1); +o2=index2adr(L,index2); +i=(o1==(&luaO_nilobject_)||o2==(&luaO_nilobject_))?0 +:luaV_lessthan(L,o1,o2); +return i; +} +static lua_Number lua_tonumber(lua_State*L,int idx){ +TValue n; +const TValue*o=index2adr(L,idx); +if(tonumber(o,&n)) +return nvalue(o); +else +return 0; +} +static lua_Integer lua_tointeger(lua_State*L,int idx){ +TValue n; +const TValue*o=index2adr(L,idx); +if(tonumber(o,&n)){ +lua_Integer res; +lua_Number num=nvalue(o); +lua_number2integer(res,num); +return res; +} +else +return 0; +} +static int lua_toboolean(lua_State*L,int idx){ +const TValue*o=index2adr(L,idx); +return!l_isfalse(o); +} +static const char*lua_tolstring(lua_State*L,int idx,size_t*len){ +StkId o=index2adr(L,idx); +if(!ttisstring(o)){ +if(!luaV_tostring(L,o)){ +if(len!=NULL)*len=0; +return NULL; +} +luaC_checkGC(L); +o=index2adr(L,idx); +} +if(len!=NULL)*len=tsvalue(o)->len; +return svalue(o); +} +static size_t lua_objlen(lua_State*L,int idx){ +StkId o=index2adr(L,idx); +switch(ttype(o)){ +case 4:return tsvalue(o)->len; +case 7:return uvalue(o)->len; +case 5:return luaH_getn(hvalue(o)); +case 3:{ +size_t l; +l=(luaV_tostring(L,o)?tsvalue(o)->len:0); +return l; +} +default:return 0; +} +} +static lua_CFunction lua_tocfunction(lua_State*L,int idx){ +StkId o=index2adr(L,idx); +return(!iscfunction(o))?NULL:clvalue(o)->c.f; +} +static void*lua_touserdata(lua_State*L,int idx){ +StkId o=index2adr(L,idx); +switch(ttype(o)){ +case 7:return(rawuvalue(o)+1); +case 2:return pvalue(o); +default:return NULL; +} +} +static void lua_pushnil(lua_State*L){ +setnilvalue(L->top); +api_incr_top(L); +} +static void lua_pushnumber(lua_State*L,lua_Number n){ +setnvalue(L->top,n); +api_incr_top(L); +} +static void lua_pushinteger(lua_State*L,lua_Integer n){ +setnvalue(L->top,cast_num(n)); +api_incr_top(L); +} +static void lua_pushlstring(lua_State*L,const char*s,size_t len){ +luaC_checkGC(L); +setsvalue(L,L->top,luaS_newlstr(L,s,len)); +api_incr_top(L); +} +static void lua_pushstring(lua_State*L,const char*s){ +if(s==NULL) +lua_pushnil(L); +else +lua_pushlstring(L,s,strlen(s)); +} +static const char*lua_pushvfstring(lua_State*L,const char*fmt, +va_list argp){ +const char*ret; +luaC_checkGC(L); +ret=luaO_pushvfstring(L,fmt,argp); +return ret; +} +static const char*lua_pushfstring(lua_State*L,const char*fmt,...){ +const char*ret; +va_list argp; +luaC_checkGC(L); +va_start(argp,fmt); +ret=luaO_pushvfstring(L,fmt,argp); +va_end(argp); +return ret; +} +static void lua_pushcclosure(lua_State*L,lua_CFunction fn,int n){ +Closure*cl; +luaC_checkGC(L); +api_checknelems(L,n); +cl=luaF_newCclosure(L,n,getcurrenv(L)); +cl->c.f=fn; +L->top-=n; +while(n--) +setobj(L,&cl->c.upvalue[n],L->top+n); +setclvalue(L,L->top,cl); +api_incr_top(L); +} +static void lua_pushboolean(lua_State*L,int b){ +setbvalue(L->top,(b!=0)); +api_incr_top(L); +} +static int lua_pushthread(lua_State*L){ +setthvalue(L,L->top,L); +api_incr_top(L); +return(G(L)->mainthread==L); +} +static void lua_gettable(lua_State*L,int idx){ +StkId t; +t=index2adr(L,idx); +api_checkvalidindex(L,t); +luaV_gettable(L,t,L->top-1,L->top-1); +} +static void lua_getfield(lua_State*L,int idx,const char*k){ +StkId t; +TValue key; +t=index2adr(L,idx); +api_checkvalidindex(L,t); +setsvalue(L,&key,luaS_new(L,k)); +luaV_gettable(L,t,&key,L->top); +api_incr_top(L); +} +static void lua_rawget(lua_State*L,int idx){ +StkId t; +t=index2adr(L,idx); +luai_apicheck(L,ttistable(t)); +setobj(L,L->top-1,luaH_get(hvalue(t),L->top-1)); +} +static void lua_rawgeti(lua_State*L,int idx,int n){ +StkId o; +o=index2adr(L,idx); +luai_apicheck(L,ttistable(o)); +setobj(L,L->top,luaH_getnum(hvalue(o),n)); +api_incr_top(L); +} +static void lua_createtable(lua_State*L,int narray,int nrec){ +luaC_checkGC(L); +sethvalue(L,L->top,luaH_new(L,narray,nrec)); +api_incr_top(L); +} +static int lua_getmetatable(lua_State*L,int objindex){ +const TValue*obj; +Table*mt=NULL; +int res; +obj=index2adr(L,objindex); +switch(ttype(obj)){ +case 5: +mt=hvalue(obj)->metatable; +break; +case 7: +mt=uvalue(obj)->metatable; +break; +default: +mt=G(L)->mt[ttype(obj)]; +break; +} +if(mt==NULL) +res=0; +else{ +sethvalue(L,L->top,mt); +api_incr_top(L); +res=1; +} +return res; +} +static void lua_getfenv(lua_State*L,int idx){ +StkId o; +o=index2adr(L,idx); +api_checkvalidindex(L,o); +switch(ttype(o)){ +case 6: +sethvalue(L,L->top,clvalue(o)->c.env); +break; +case 7: +sethvalue(L,L->top,uvalue(o)->env); +break; +case 8: +setobj(L,L->top,gt(thvalue(o))); +break; +default: +setnilvalue(L->top); +break; +} +api_incr_top(L); +} +static void lua_settable(lua_State*L,int idx){ +StkId t; +api_checknelems(L,2); +t=index2adr(L,idx); +api_checkvalidindex(L,t); +luaV_settable(L,t,L->top-2,L->top-1); +L->top-=2; +} +static void lua_setfield(lua_State*L,int idx,const char*k){ +StkId t; +TValue key; +api_checknelems(L,1); +t=index2adr(L,idx); +api_checkvalidindex(L,t); +setsvalue(L,&key,luaS_new(L,k)); +luaV_settable(L,t,&key,L->top-1); +L->top--; +} +static void lua_rawset(lua_State*L,int idx){ +StkId t; +api_checknelems(L,2); +t=index2adr(L,idx); +luai_apicheck(L,ttistable(t)); +setobj(L,luaH_set(L,hvalue(t),L->top-2),L->top-1); +luaC_barriert(L,hvalue(t),L->top-1); +L->top-=2; +} +static void lua_rawseti(lua_State*L,int idx,int n){ +StkId o; +api_checknelems(L,1); +o=index2adr(L,idx); +luai_apicheck(L,ttistable(o)); +setobj(L,luaH_setnum(L,hvalue(o),n),L->top-1); +luaC_barriert(L,hvalue(o),L->top-1); +L->top--; +} +static int lua_setmetatable(lua_State*L,int objindex){ +TValue*obj; +Table*mt; +api_checknelems(L,1); +obj=index2adr(L,objindex); +api_checkvalidindex(L,obj); +if(ttisnil(L->top-1)) +mt=NULL; +else{ +luai_apicheck(L,ttistable(L->top-1)); +mt=hvalue(L->top-1); +} +switch(ttype(obj)){ +case 5:{ +hvalue(obj)->metatable=mt; +if(mt) +luaC_objbarriert(L,hvalue(obj),mt); +break; +} +case 7:{ +uvalue(obj)->metatable=mt; +if(mt) +luaC_objbarrier(L,rawuvalue(obj),mt); +break; +} +default:{ +G(L)->mt[ttype(obj)]=mt; +break; +} +} +L->top--; +return 1; +} +static int lua_setfenv(lua_State*L,int idx){ +StkId o; +int res=1; +api_checknelems(L,1); +o=index2adr(L,idx); +api_checkvalidindex(L,o); +luai_apicheck(L,ttistable(L->top-1)); +switch(ttype(o)){ +case 6: +clvalue(o)->c.env=hvalue(L->top-1); +break; +case 7: +uvalue(o)->env=hvalue(L->top-1); +break; +case 8: +sethvalue(L,gt(thvalue(o)),hvalue(L->top-1)); +break; +default: +res=0; +break; +} +if(res)luaC_objbarrier(L,gcvalue(o),hvalue(L->top-1)); +L->top--; +return res; +} +#define adjustresults(L,nres){if(nres==(-1)&&L->top>=L->ci->top)L->ci->top=L->top;} +#define checkresults(L,na,nr)luai_apicheck(L,(nr)==(-1)||(L->ci->top-L->top>=(nr)-(na))) +static void lua_call(lua_State*L,int nargs,int nresults){ +StkId func; +api_checknelems(L,nargs+1); +checkresults(L,nargs,nresults); +func=L->top-(nargs+1); +luaD_call(L,func,nresults); +adjustresults(L,nresults); +} +struct CallS{ +StkId func; +int nresults; +}; +static void f_call(lua_State*L,void*ud){ +struct CallS*c=cast(struct CallS*,ud); +luaD_call(L,c->func,c->nresults); +} +static int lua_pcall(lua_State*L,int nargs,int nresults,int errfunc){ +struct CallS c; +int status; +ptrdiff_t func; +api_checknelems(L,nargs+1); +checkresults(L,nargs,nresults); +if(errfunc==0) +func=0; +else{ +StkId o=index2adr(L,errfunc); +api_checkvalidindex(L,o); +func=savestack(L,o); +} +c.func=L->top-(nargs+1); +c.nresults=nresults; +status=luaD_pcall(L,f_call,&c,savestack(L,c.func),func); +adjustresults(L,nresults); +return status; +} +static int lua_load(lua_State*L,lua_Reader reader,void*data, +const char*chunkname){ +ZIO z; +int status; +if(!chunkname)chunkname="?"; +luaZ_init(L,&z,reader,data); +status=luaD_protectedparser(L,&z,chunkname); +return status; +} +static int lua_error(lua_State*L){ +api_checknelems(L,1); +luaG_errormsg(L); +return 0; +} +static int lua_next(lua_State*L,int idx){ +StkId t; +int more; +t=index2adr(L,idx); +luai_apicheck(L,ttistable(t)); +more=luaH_next(L,hvalue(t),L->top-1); +if(more){ +api_incr_top(L); +} +else +L->top-=1; +return more; +} +static void lua_concat(lua_State*L,int n){ +api_checknelems(L,n); +if(n>=2){ +luaC_checkGC(L); +luaV_concat(L,n,cast_int(L->top-L->base)-1); +L->top-=(n-1); +} +else if(n==0){ +setsvalue(L,L->top,luaS_newlstr(L,"",0)); +api_incr_top(L); +} +} +static void*lua_newuserdata(lua_State*L,size_t size){ +Udata*u; +luaC_checkGC(L); +u=luaS_newudata(L,size,getcurrenv(L)); +setuvalue(L,L->top,u); +api_incr_top(L); +return u+1; +} +#define luaL_getn(L,i)((int)lua_objlen(L,i)) +#define luaL_setn(L,i,j)((void)0) +typedef struct luaL_Reg{ +const char*name; +lua_CFunction func; +}luaL_Reg; +static void luaI_openlib(lua_State*L,const char*libname, +const luaL_Reg*l,int nup); +static int luaL_argerror(lua_State*L,int numarg,const char*extramsg); +static const char* luaL_checklstring(lua_State*L,int numArg, +size_t*l); +static const char* luaL_optlstring(lua_State*L,int numArg, +const char*def,size_t*l); +static lua_Integer luaL_checkinteger(lua_State*L,int numArg); +static lua_Integer luaL_optinteger(lua_State*L,int nArg, +lua_Integer def); +static int luaL_error(lua_State*L,const char*fmt,...); +static const char* luaL_findtable(lua_State*L,int idx, +const char*fname,int szhint); +#define luaL_argcheck(L,cond,numarg,extramsg)((void)((cond)||luaL_argerror(L,(numarg),(extramsg)))) +#define luaL_checkstring(L,n)(luaL_checklstring(L,(n),NULL)) +#define luaL_optstring(L,n,d)(luaL_optlstring(L,(n),(d),NULL)) +#define luaL_checkint(L,n)((int)luaL_checkinteger(L,(n))) +#define luaL_optint(L,n,d)((int)luaL_optinteger(L,(n),(d))) +#define luaL_typename(L,i)lua_typename(L,lua_type(L,(i))) +#define luaL_getmetatable(L,n)(lua_getfield(L,(-10000),(n))) +#define luaL_opt(L,f,n,d)(lua_isnoneornil(L,(n))?(d):f(L,(n))) +typedef struct luaL_Buffer{ +char*p; +int lvl; +lua_State*L; +char buffer[BUFSIZ]; +}luaL_Buffer; +#define luaL_addchar(B,c)((void)((B)->p<((B)->buffer+BUFSIZ)||luaL_prepbuffer(B)),(*(B)->p++=(char)(c))) +#define luaL_addsize(B,n)((B)->p+=(n)) +static char* luaL_prepbuffer(luaL_Buffer*B); +static int luaL_argerror(lua_State*L,int narg,const char*extramsg){ +lua_Debug ar; +if(!lua_getstack(L,0,&ar)) +return luaL_error(L,"bad argument #%d (%s)",narg,extramsg); +lua_getinfo(L,"n",&ar); +if(strcmp(ar.namewhat,"method")==0){ +narg--; +if(narg==0) +return luaL_error(L,"calling "LUA_QL("%s")" on bad self (%s)", +ar.name,extramsg); +} +if(ar.name==NULL) +ar.name="?"; +return luaL_error(L,"bad argument #%d to "LUA_QL("%s")" (%s)", +narg,ar.name,extramsg); +} +static int luaL_typerror(lua_State*L,int narg,const char*tname){ +const char*msg=lua_pushfstring(L,"%s expected, got %s", +tname,luaL_typename(L,narg)); +return luaL_argerror(L,narg,msg); +} +static void tag_error(lua_State*L,int narg,int tag){ +luaL_typerror(L,narg,lua_typename(L,tag)); +} +static void luaL_where(lua_State*L,int level){ +lua_Debug ar; +if(lua_getstack(L,level,&ar)){ +lua_getinfo(L,"Sl",&ar); +if(ar.currentline>0){ +lua_pushfstring(L,"%s:%d: ",ar.short_src,ar.currentline); +return; +} +} +lua_pushliteral(L,""); +} +static int luaL_error(lua_State*L,const char*fmt,...){ +va_list argp; +va_start(argp,fmt); +luaL_where(L,1); +lua_pushvfstring(L,fmt,argp); +va_end(argp); +lua_concat(L,2); +return lua_error(L); +} +static int luaL_newmetatable(lua_State*L,const char*tname){ +lua_getfield(L,(-10000),tname); +if(!lua_isnil(L,-1)) +return 0; +lua_pop(L,1); +lua_newtable(L); +lua_pushvalue(L,-1); +lua_setfield(L,(-10000),tname); +return 1; +} +static void*luaL_checkudata(lua_State*L,int ud,const char*tname){ +void*p=lua_touserdata(L,ud); +if(p!=NULL){ +if(lua_getmetatable(L,ud)){ +lua_getfield(L,(-10000),tname); +if(lua_rawequal(L,-1,-2)){ +lua_pop(L,2); +return p; +} +} +} +luaL_typerror(L,ud,tname); +return NULL; +} +static void luaL_checkstack(lua_State*L,int space,const char*mes){ +if(!lua_checkstack(L,space)) +luaL_error(L,"stack overflow (%s)",mes); +} +static void luaL_checktype(lua_State*L,int narg,int t){ +if(lua_type(L,narg)!=t) +tag_error(L,narg,t); +} +static void luaL_checkany(lua_State*L,int narg){ +if(lua_type(L,narg)==(-1)) +luaL_argerror(L,narg,"value expected"); +} +static const char*luaL_checklstring(lua_State*L,int narg,size_t*len){ +const char*s=lua_tolstring(L,narg,len); +if(!s)tag_error(L,narg,4); +return s; +} +static const char*luaL_optlstring(lua_State*L,int narg, +const char*def,size_t*len){ +if(lua_isnoneornil(L,narg)){ +if(len) +*len=(def?strlen(def):0); +return def; +} +else return luaL_checklstring(L,narg,len); +} +static lua_Number luaL_checknumber(lua_State*L,int narg){ +lua_Number d=lua_tonumber(L,narg); +if(d==0&&!lua_isnumber(L,narg)) +tag_error(L,narg,3); +return d; +} +static lua_Integer luaL_checkinteger(lua_State*L,int narg){ +lua_Integer d=lua_tointeger(L,narg); +if(d==0&&!lua_isnumber(L,narg)) +tag_error(L,narg,3); +return d; +} +static lua_Integer luaL_optinteger(lua_State*L,int narg, +lua_Integer def){ +return luaL_opt(L,luaL_checkinteger,narg,def); +} +static int luaL_getmetafield(lua_State*L,int obj,const char*event){ +if(!lua_getmetatable(L,obj)) +return 0; +lua_pushstring(L,event); +lua_rawget(L,-2); +if(lua_isnil(L,-1)){ +lua_pop(L,2); +return 0; +} +else{ +lua_remove(L,-2); +return 1; +} +} +static void luaL_register(lua_State*L,const char*libname, +const luaL_Reg*l){ +luaI_openlib(L,libname,l,0); +} +static int libsize(const luaL_Reg*l){ +int size=0; +for(;l->name;l++)size++; +return size; +} +static void luaI_openlib(lua_State*L,const char*libname, +const luaL_Reg*l,int nup){ +if(libname){ +int size=libsize(l); +luaL_findtable(L,(-10000),"_LOADED",1); +lua_getfield(L,-1,libname); +if(!lua_istable(L,-1)){ +lua_pop(L,1); +if(luaL_findtable(L,(-10002),libname,size)!=NULL) +luaL_error(L,"name conflict for module "LUA_QL("%s"),libname); +lua_pushvalue(L,-1); +lua_setfield(L,-3,libname); +} +lua_remove(L,-2); +lua_insert(L,-(nup+1)); +} +for(;l->name;l++){ +int i; +for(i=0;ifunc,nup); +lua_setfield(L,-(nup+2),l->name); +} +lua_pop(L,nup); +} +static const char*luaL_findtable(lua_State*L,int idx, +const char*fname,int szhint){ +const char*e; +lua_pushvalue(L,idx); +do{ +e=strchr(fname,'.'); +if(e==NULL)e=fname+strlen(fname); +lua_pushlstring(L,fname,e-fname); +lua_rawget(L,-2); +if(lua_isnil(L,-1)){ +lua_pop(L,1); +lua_createtable(L,0,(*e=='.'?1:szhint)); +lua_pushlstring(L,fname,e-fname); +lua_pushvalue(L,-2); +lua_settable(L,-4); +} +else if(!lua_istable(L,-1)){ +lua_pop(L,2); +return fname; +} +lua_remove(L,-2); +fname=e+1; +}while(*e=='.'); +return NULL; +} +#define bufflen(B)((B)->p-(B)->buffer) +#define bufffree(B)((size_t)(BUFSIZ-bufflen(B))) +static int emptybuffer(luaL_Buffer*B){ +size_t l=bufflen(B); +if(l==0)return 0; +else{ +lua_pushlstring(B->L,B->buffer,l); +B->p=B->buffer; +B->lvl++; +return 1; +} +} +static void adjuststack(luaL_Buffer*B){ +if(B->lvl>1){ +lua_State*L=B->L; +int toget=1; +size_t toplen=lua_strlen(L,-1); +do{ +size_t l=lua_strlen(L,-(toget+1)); +if(B->lvl-toget+1>=(20/2)||toplen>l){ +toplen+=l; +toget++; +} +else break; +}while(togetlvl); +lua_concat(L,toget); +B->lvl=B->lvl-toget+1; +} +} +static char*luaL_prepbuffer(luaL_Buffer*B){ +if(emptybuffer(B)) +adjuststack(B); +return B->buffer; +} +static void luaL_addlstring(luaL_Buffer*B,const char*s,size_t l){ +while(l--) +luaL_addchar(B,*s++); +} +static void luaL_pushresult(luaL_Buffer*B){ +emptybuffer(B); +lua_concat(B->L,B->lvl); +B->lvl=1; +} +static void luaL_addvalue(luaL_Buffer*B){ +lua_State*L=B->L; +size_t vl; +const char*s=lua_tolstring(L,-1,&vl); +if(vl<=bufffree(B)){ +memcpy(B->p,s,vl); +B->p+=vl; +lua_pop(L,1); +} +else{ +if(emptybuffer(B)) +lua_insert(L,-2); +B->lvl++; +adjuststack(B); +} +} +static void luaL_buffinit(lua_State*L,luaL_Buffer*B){ +B->L=L; +B->p=B->buffer; +B->lvl=0; +} +typedef struct LoadF{ +int extraline; +FILE*f; +char buff[BUFSIZ]; +}LoadF; +static const char*getF(lua_State*L,void*ud,size_t*size){ +LoadF*lf=(LoadF*)ud; +(void)L; +if(lf->extraline){ +lf->extraline=0; +*size=1; +return"\n"; +} +if(feof(lf->f))return NULL; +*size=fread(lf->buff,1,sizeof(lf->buff),lf->f); +return(*size>0)?lf->buff:NULL; +} +static int errfile(lua_State*L,const char*what,int fnameindex){ +const char*serr=strerror(errno); +const char*filename=lua_tostring(L,fnameindex)+1; +lua_pushfstring(L,"cannot %s %s: %s",what,filename,serr); +lua_remove(L,fnameindex); +return(5+1); +} +static int luaL_loadfile(lua_State*L,const char*filename){ +LoadF lf; +int status,readstatus; +int c; +int fnameindex=lua_gettop(L)+1; +lf.extraline=0; +if(filename==NULL){ +lua_pushliteral(L,"=stdin"); +lf.f=stdin; +} +else{ +lua_pushfstring(L,"@%s",filename); +lf.f=fopen(filename,"r"); +if(lf.f==NULL)return errfile(L,"open",fnameindex); +} +c=getc(lf.f); +if(c=='#'){ +lf.extraline=1; +while((c=getc(lf.f))!=EOF&&c!='\n'); +if(c=='\n')c=getc(lf.f); +} +if(c=="\033Lua"[0]&&filename){ +lf.f=freopen(filename,"rb",lf.f); +if(lf.f==NULL)return errfile(L,"reopen",fnameindex); +while((c=getc(lf.f))!=EOF&&c!="\033Lua"[0]); +lf.extraline=0; +} +ungetc(c,lf.f); +status=lua_load(L,getF,&lf,lua_tostring(L,-1)); +readstatus=ferror(lf.f); +if(filename)fclose(lf.f); +if(readstatus){ +lua_settop(L,fnameindex); +return errfile(L,"read",fnameindex); +} +lua_remove(L,fnameindex); +return status; +} +typedef struct LoadS{ +const char*s; +size_t size; +}LoadS; +static const char*getS(lua_State*L,void*ud,size_t*size){ +LoadS*ls=(LoadS*)ud; +(void)L; +if(ls->size==0)return NULL; +*size=ls->size; +ls->size=0; +return ls->s; +} +static int luaL_loadbuffer(lua_State*L,const char*buff,size_t size, +const char*name){ +LoadS ls; +ls.s=buff; +ls.size=size; +return lua_load(L,getS,&ls,name); +} +static void*l_alloc(void*ud,void*ptr,size_t osize,size_t nsize){ +(void)ud; +(void)osize; +if(nsize==0){ +free(ptr); +return NULL; +} +else +return realloc(ptr,nsize); +} +static int panic(lua_State*L){ +(void)L; +fprintf(stderr,"PANIC: unprotected error in call to Lua API (%s)\n", +lua_tostring(L,-1)); +return 0; +} +static lua_State*luaL_newstate(void){ +lua_State*L=lua_newstate(l_alloc,NULL); +if(L)lua_atpanic(L,&panic); +return L; +} +static int luaB_tonumber(lua_State*L){ +int base=luaL_optint(L,2,10); +if(base==10){ +luaL_checkany(L,1); +if(lua_isnumber(L,1)){ +lua_pushnumber(L,lua_tonumber(L,1)); +return 1; +} +} +else{ +const char*s1=luaL_checkstring(L,1); +char*s2; +unsigned long n; +luaL_argcheck(L,2<=base&&base<=36,2,"base out of range"); +n=strtoul(s1,&s2,base); +if(s1!=s2){ +while(isspace((unsigned char)(*s2)))s2++; +if(*s2=='\0'){ +lua_pushnumber(L,(lua_Number)n); +return 1; +} +} +} +lua_pushnil(L); +return 1; +} +static int luaB_error(lua_State*L){ +int level=luaL_optint(L,2,1); +lua_settop(L,1); +if(lua_isstring(L,1)&&level>0){ +luaL_where(L,level); +lua_pushvalue(L,1); +lua_concat(L,2); +} +return lua_error(L); +} +static int luaB_setmetatable(lua_State*L){ +int t=lua_type(L,2); +luaL_checktype(L,1,5); +luaL_argcheck(L,t==0||t==5,2, +"nil or table expected"); +if(luaL_getmetafield(L,1,"__metatable")) +luaL_error(L,"cannot change a protected metatable"); +lua_settop(L,2); +lua_setmetatable(L,1); +return 1; +} +static void getfunc(lua_State*L,int opt){ +if(lua_isfunction(L,1))lua_pushvalue(L,1); +else{ +lua_Debug ar; +int level=opt?luaL_optint(L,1,1):luaL_checkint(L,1); +luaL_argcheck(L,level>=0,1,"level must be non-negative"); +if(lua_getstack(L,level,&ar)==0) +luaL_argerror(L,1,"invalid level"); +lua_getinfo(L,"f",&ar); +if(lua_isnil(L,-1)) +luaL_error(L,"no function environment for tail call at level %d", +level); +} +} +static int luaB_setfenv(lua_State*L){ +luaL_checktype(L,2,5); +getfunc(L,0); +lua_pushvalue(L,2); +if(lua_isnumber(L,1)&&lua_tonumber(L,1)==0){ +lua_pushthread(L); +lua_insert(L,-2); +lua_setfenv(L,-2); +return 0; +} +else if(lua_iscfunction(L,-2)||lua_setfenv(L,-2)==0) +luaL_error(L, +LUA_QL("setfenv")" cannot change environment of given object"); +return 1; +} +static int luaB_rawget(lua_State*L){ +luaL_checktype(L,1,5); +luaL_checkany(L,2); +lua_settop(L,2); +lua_rawget(L,1); +return 1; +} +static int luaB_type(lua_State*L){ +luaL_checkany(L,1); +lua_pushstring(L,luaL_typename(L,1)); +return 1; +} +static int luaB_next(lua_State*L){ +luaL_checktype(L,1,5); +lua_settop(L,2); +if(lua_next(L,1)) +return 2; +else{ +lua_pushnil(L); +return 1; +} +} +static int luaB_pairs(lua_State*L){ +luaL_checktype(L,1,5); +lua_pushvalue(L,lua_upvalueindex(1)); +lua_pushvalue(L,1); +lua_pushnil(L); +return 3; +} +static int ipairsaux(lua_State*L){ +int i=luaL_checkint(L,2); +luaL_checktype(L,1,5); +i++; +lua_pushinteger(L,i); +lua_rawgeti(L,1,i); +return(lua_isnil(L,-1))?0:2; +} +static int luaB_ipairs(lua_State*L){ +luaL_checktype(L,1,5); +lua_pushvalue(L,lua_upvalueindex(1)); +lua_pushvalue(L,1); +lua_pushinteger(L,0); +return 3; +} +static int load_aux(lua_State*L,int status){ +if(status==0) +return 1; +else{ +lua_pushnil(L); +lua_insert(L,-2); +return 2; +} +} +static int luaB_loadstring(lua_State*L){ +size_t l; +const char*s=luaL_checklstring(L,1,&l); +const char*chunkname=luaL_optstring(L,2,s); +return load_aux(L,luaL_loadbuffer(L,s,l,chunkname)); +} +static int luaB_loadfile(lua_State*L){ +const char*fname=luaL_optstring(L,1,NULL); +return load_aux(L,luaL_loadfile(L,fname)); +} +static int luaB_assert(lua_State*L){ +luaL_checkany(L,1); +if(!lua_toboolean(L,1)) +return luaL_error(L,"%s",luaL_optstring(L,2,"assertion failed!")); +return lua_gettop(L); +} +static int luaB_unpack(lua_State*L){ +int i,e,n; +luaL_checktype(L,1,5); +i=luaL_optint(L,2,1); +e=luaL_opt(L,luaL_checkint,3,luaL_getn(L,1)); +if(i>e)return 0; +n=e-i+1; +if(n<=0||!lua_checkstack(L,n)) +return luaL_error(L,"too many results to unpack"); +lua_rawgeti(L,1,i); +while(i++e)e=pos; +for(i=e;i>pos;i--){ +lua_rawgeti(L,1,i-1); +lua_rawseti(L,1,i); +} +break; +} +default:{ +return luaL_error(L,"wrong number of arguments to "LUA_QL("insert")); +} +} +luaL_setn(L,1,e); +lua_rawseti(L,1,pos); +return 0; +} +static int tremove(lua_State*L){ +int e=aux_getn(L,1); +int pos=luaL_optint(L,2,e); +if(!(1<=pos&&pos<=e)) +return 0; +luaL_setn(L,1,e-1); +lua_rawgeti(L,1,pos); +for(;posu)luaL_error(L,"invalid order function for sorting"); +lua_pop(L,1); +} +while(lua_rawgeti(L,1,--j),sort_comp(L,-3,-1)){ +if(j0); +} +l=strlen(p); +if(l==0||p[l-1]!='\n') +luaL_addsize(&b,l); +else{ +luaL_addsize(&b,l-1); +luaL_pushresult(&b); +return 1; +} +} +} +static int read_chars(lua_State*L,FILE*f,size_t n){ +size_t rlen; +size_t nr; +luaL_Buffer b; +luaL_buffinit(L,&b); +rlen=BUFSIZ; +do{ +char*p=luaL_prepbuffer(&b); +if(rlen>n)rlen=n; +nr=fread(p,sizeof(char),rlen,f); +luaL_addsize(&b,nr); +n-=nr; +}while(n>0&&nr==rlen); +luaL_pushresult(&b); +return(n==0||lua_objlen(L,-1)>0); +} +static int g_read(lua_State*L,FILE*f,int first){ +int nargs=lua_gettop(L)-1; +int success; +int n; +clearerr(f); +if(nargs==0){ +success=read_line(L,f); +n=first+1; +} +else{ +luaL_checkstack(L,nargs+20,"too many arguments"); +success=1; +for(n=first;nargs--&&success;n++){ +if(lua_type(L,n)==3){ +size_t l=(size_t)lua_tointeger(L,n); +success=(l==0)?test_eof(L,f):read_chars(L,f,l); +} +else{ +const char*p=lua_tostring(L,n); +luaL_argcheck(L,p&&p[0]=='*',n,"invalid option"); +switch(p[1]){ +case'n': +success=read_number(L,f); +break; +case'l': +success=read_line(L,f); +break; +case'a': +read_chars(L,f,~((size_t)0)); +success=1; +break; +default: +return luaL_argerror(L,n,"invalid format"); +} +} +} +} +if(ferror(f)) +return pushresult(L,0,NULL); +if(!success){ +lua_pop(L,1); +lua_pushnil(L); +} +return n-first; +} +static int io_read(lua_State*L){ +return g_read(L,getiofile(L,1),1); +} +static int f_read(lua_State*L){ +return g_read(L,tofile(L),2); +} +static int io_readline(lua_State*L){ +FILE*f=*(FILE**)lua_touserdata(L,lua_upvalueindex(1)); +int sucess; +if(f==NULL) +luaL_error(L,"file is already closed"); +sucess=read_line(L,f); +if(ferror(f)) +return luaL_error(L,"%s",strerror(errno)); +if(sucess)return 1; +else{ +if(lua_toboolean(L,lua_upvalueindex(2))){ +lua_settop(L,0); +lua_pushvalue(L,lua_upvalueindex(1)); +aux_close(L); +} +return 0; +} +} +static int g_write(lua_State*L,FILE*f,int arg){ +int nargs=lua_gettop(L)-1; +int status=1; +for(;nargs--;arg++){ +if(lua_type(L,arg)==3){ +status=status&& +fprintf(f,"%.14g",lua_tonumber(L,arg))>0; +} +else{ +size_t l; +const char*s=luaL_checklstring(L,arg,&l); +status=status&&(fwrite(s,sizeof(char),l,f)==l); +} +} +return pushresult(L,status,NULL); +} +static int io_write(lua_State*L){ +return g_write(L,getiofile(L,2),1); +} +static int f_write(lua_State*L){ +return g_write(L,tofile(L),2); +} +static int io_flush(lua_State*L){ +return pushresult(L,fflush(getiofile(L,2))==0,NULL); +} +static int f_flush(lua_State*L){ +return pushresult(L,fflush(tofile(L))==0,NULL); +} +static const luaL_Reg iolib[]={ +{"close",io_close}, +{"flush",io_flush}, +{"input",io_input}, +{"lines",io_lines}, +{"open",io_open}, +{"output",io_output}, +{"read",io_read}, +{"type",io_type}, +{"write",io_write}, +{NULL,NULL} +}; +static const luaL_Reg flib[]={ +{"close",io_close}, +{"flush",f_flush}, +{"lines",f_lines}, +{"read",f_read}, +{"write",f_write}, +{"__gc",io_gc}, +{NULL,NULL} +}; +static void createmeta(lua_State*L){ +luaL_newmetatable(L,"FILE*"); +lua_pushvalue(L,-1); +lua_setfield(L,-2,"__index"); +luaL_register(L,NULL,flib); +} +static void createstdfile(lua_State*L,FILE*f,int k,const char*fname){ +*newfile(L)=f; +if(k>0){ +lua_pushvalue(L,-1); +lua_rawseti(L,(-10001),k); +} +lua_pushvalue(L,-2); +lua_setfenv(L,-2); +lua_setfield(L,-3,fname); +} +static void newfenv(lua_State*L,lua_CFunction cls){ +lua_createtable(L,0,1); +lua_pushcfunction(L,cls); +lua_setfield(L,-2,"__close"); +} +static int luaopen_io(lua_State*L){ +createmeta(L); +newfenv(L,io_fclose); +lua_replace(L,(-10001)); +luaL_register(L,"io",iolib); +newfenv(L,io_noclose); +createstdfile(L,stdin,1,"stdin"); +createstdfile(L,stdout,2,"stdout"); +createstdfile(L,stderr,0,"stderr"); +lua_pop(L,1); +lua_getfield(L,-1,"popen"); +newfenv(L,io_pclose); +lua_setfenv(L,-2); +lua_pop(L,1); +return 1; +} +static int os_pushresult(lua_State*L,int i,const char*filename){ +int en=errno; +if(i){ +lua_pushboolean(L,1); +return 1; +} +else{ +lua_pushnil(L); +lua_pushfstring(L,"%s: %s",filename,strerror(en)); +lua_pushinteger(L,en); +return 3; +} +} +static int os_remove(lua_State*L){ +const char*filename=luaL_checkstring(L,1); +return os_pushresult(L,remove(filename)==0,filename); +} +static int os_exit(lua_State*L){ +exit(luaL_optint(L,1,EXIT_SUCCESS)); +} +static const luaL_Reg syslib[]={ +{"exit",os_exit}, +{"remove",os_remove}, +{NULL,NULL} +}; +static int luaopen_os(lua_State*L){ +luaL_register(L,"os",syslib); +return 1; +} +#define uchar(c)((unsigned char)(c)) +static ptrdiff_t posrelat(ptrdiff_t pos,size_t len){ +if(pos<0)pos+=(ptrdiff_t)len+1; +return(pos>=0)?pos:0; +} +static int str_sub(lua_State*L){ +size_t l; +const char*s=luaL_checklstring(L,1,&l); +ptrdiff_t start=posrelat(luaL_checkinteger(L,2),l); +ptrdiff_t end=posrelat(luaL_optinteger(L,3,-1),l); +if(start<1)start=1; +if(end>(ptrdiff_t)l)end=(ptrdiff_t)l; +if(start<=end) +lua_pushlstring(L,s+start-1,end-start+1); +else lua_pushliteral(L,""); +return 1; +} +static int str_lower(lua_State*L){ +size_t l; +size_t i; +luaL_Buffer b; +const char*s=luaL_checklstring(L,1,&l); +luaL_buffinit(L,&b); +for(i=0;i0) +luaL_addlstring(&b,s,l); +luaL_pushresult(&b); +return 1; +} +static int str_byte(lua_State*L){ +size_t l; +const char*s=luaL_checklstring(L,1,&l); +ptrdiff_t posi=posrelat(luaL_optinteger(L,2,1),l); +ptrdiff_t pose=posrelat(luaL_optinteger(L,3,posi),l); +int n,i; +if(posi<=0)posi=1; +if((size_t)pose>l)pose=l; +if(posi>pose)return 0; +n=(int)(pose-posi+1); +if(posi+n<=pose) +luaL_error(L,"string slice too long"); +luaL_checkstack(L,n,"string slice too long"); +for(i=0;i=ms->level||ms->capture[l].len==(-1)) +return luaL_error(ms->L,"invalid capture index"); +return l; +} +static int capture_to_close(MatchState*ms){ +int level=ms->level; +for(level--;level>=0;level--) +if(ms->capture[level].len==(-1))return level; +return luaL_error(ms->L,"invalid pattern capture"); +} +static const char*classend(MatchState*ms,const char*p){ +switch(*p++){ +case'%':{ +if(*p=='\0') +luaL_error(ms->L,"malformed pattern (ends with "LUA_QL("%%")")"); +return p+1; +} +case'[':{ +if(*p=='^')p++; +do{ +if(*p=='\0') +luaL_error(ms->L,"malformed pattern (missing "LUA_QL("]")")"); +if(*(p++)=='%'&&*p!='\0') +p++; +}while(*p!=']'); +return p+1; +} +default:{ +return p; +} +} +} +static int match_class(int c,int cl){ +int res; +switch(tolower(cl)){ +case'a':res=isalpha(c);break; +case'c':res=iscntrl(c);break; +case'd':res=isdigit(c);break; +case'l':res=islower(c);break; +case'p':res=ispunct(c);break; +case's':res=isspace(c);break; +case'u':res=isupper(c);break; +case'w':res=isalnum(c);break; +case'x':res=isxdigit(c);break; +case'z':res=(c==0);break; +default:return(cl==c); +} +return(islower(cl)?res:!res); +} +static int matchbracketclass(int c,const char*p,const char*ec){ +int sig=1; +if(*(p+1)=='^'){ +sig=0; +p++; +} +while(++pL,"unbalanced pattern"); +if(*s!=*p)return NULL; +else{ +int b=*p; +int e=*(p+1); +int cont=1; +while(++ssrc_end){ +if(*s==e){ +if(--cont==0)return s+1; +} +else if(*s==b)cont++; +} +} +return NULL; +} +static const char*max_expand(MatchState*ms,const char*s, +const char*p,const char*ep){ +ptrdiff_t i=0; +while((s+i)src_end&&singlematch(uchar(*(s+i)),p,ep)) +i++; +while(i>=0){ +const char*res=match(ms,(s+i),ep+1); +if(res)return res; +i--; +} +return NULL; +} +static const char*min_expand(MatchState*ms,const char*s, +const char*p,const char*ep){ +for(;;){ +const char*res=match(ms,s,ep+1); +if(res!=NULL) +return res; +else if(ssrc_end&&singlematch(uchar(*s),p,ep)) +s++; +else return NULL; +} +} +static const char*start_capture(MatchState*ms,const char*s, +const char*p,int what){ +const char*res; +int level=ms->level; +if(level>=32)luaL_error(ms->L,"too many captures"); +ms->capture[level].init=s; +ms->capture[level].len=what; +ms->level=level+1; +if((res=match(ms,s,p))==NULL) +ms->level--; +return res; +} +static const char*end_capture(MatchState*ms,const char*s, +const char*p){ +int l=capture_to_close(ms); +const char*res; +ms->capture[l].len=s-ms->capture[l].init; +if((res=match(ms,s,p))==NULL) +ms->capture[l].len=(-1); +return res; +} +static const char*match_capture(MatchState*ms,const char*s,int l){ +size_t len; +l=check_capture(ms,l); +len=ms->capture[l].len; +if((size_t)(ms->src_end-s)>=len&& +memcmp(ms->capture[l].init,s,len)==0) +return s+len; +else return NULL; +} +static const char*match(MatchState*ms,const char*s,const char*p){ +init: +switch(*p){ +case'(':{ +if(*(p+1)==')') +return start_capture(ms,s,p+2,(-2)); +else +return start_capture(ms,s,p+1,(-1)); +} +case')':{ +return end_capture(ms,s,p+1); +} +case'%':{ +switch(*(p+1)){ +case'b':{ +s=matchbalance(ms,s,p+2); +if(s==NULL)return NULL; +p+=4;goto init; +} +case'f':{ +const char*ep;char previous; +p+=2; +if(*p!='[') +luaL_error(ms->L,"missing "LUA_QL("[")" after " +LUA_QL("%%f")" in pattern"); +ep=classend(ms,p); +previous=(s==ms->src_init)?'\0':*(s-1); +if(matchbracketclass(uchar(previous),p,ep-1)|| +!matchbracketclass(uchar(*s),p,ep-1))return NULL; +p=ep;goto init; +} +default:{ +if(isdigit(uchar(*(p+1)))){ +s=match_capture(ms,s,uchar(*(p+1))); +if(s==NULL)return NULL; +p+=2;goto init; +} +goto dflt; +} +} +} +case'\0':{ +return s; +} +case'$':{ +if(*(p+1)=='\0') +return(s==ms->src_end)?s:NULL; +else goto dflt; +} +default:dflt:{ +const char*ep=classend(ms,p); +int m=ssrc_end&&singlematch(uchar(*s),p,ep); +switch(*ep){ +case'?':{ +const char*res; +if(m&&((res=match(ms,s+1,ep+1))!=NULL)) +return res; +p=ep+1;goto init; +} +case'*':{ +return max_expand(ms,s,p,ep); +} +case'+':{ +return(m?max_expand(ms,s+1,p,ep):NULL); +} +case'-':{ +return min_expand(ms,s,p,ep); +} +default:{ +if(!m)return NULL; +s++;p=ep;goto init; +} +} +} +} +} +static const char*lmemfind(const char*s1,size_t l1, +const char*s2,size_t l2){ +if(l2==0)return s1; +else if(l2>l1)return NULL; +else{ +const char*init; +l2--; +l1=l1-l2; +while(l1>0&&(init=(const char*)memchr(s1,*s2,l1))!=NULL){ +init++; +if(memcmp(init,s2+1,l2)==0) +return init-1; +else{ +l1-=init-s1; +s1=init; +} +} +return NULL; +} +} +static void push_onecapture(MatchState*ms,int i,const char*s, +const char*e){ +if(i>=ms->level){ +if(i==0) +lua_pushlstring(ms->L,s,e-s); +else +luaL_error(ms->L,"invalid capture index"); +} +else{ +ptrdiff_t l=ms->capture[i].len; +if(l==(-1))luaL_error(ms->L,"unfinished capture"); +if(l==(-2)) +lua_pushinteger(ms->L,ms->capture[i].init-ms->src_init+1); +else +lua_pushlstring(ms->L,ms->capture[i].init,l); +} +} +static int push_captures(MatchState*ms,const char*s,const char*e){ +int i; +int nlevels=(ms->level==0&&s)?1:ms->level; +luaL_checkstack(ms->L,nlevels,"too many captures"); +for(i=0;il1)init=(ptrdiff_t)l1; +if(find&&(lua_toboolean(L,4)|| +strpbrk(p,"^$*+?.([%-")==NULL)){ +const char*s2=lmemfind(s+init,l1-init,p,l2); +if(s2){ +lua_pushinteger(L,s2-s+1); +lua_pushinteger(L,s2-s+l2); +return 2; +} +} +else{ +MatchState ms; +int anchor=(*p=='^')?(p++,1):0; +const char*s1=s+init; +ms.L=L; +ms.src_init=s; +ms.src_end=s+l1; +do{ +const char*res; +ms.level=0; +if((res=match(&ms,s1,p))!=NULL){ +if(find){ +lua_pushinteger(L,s1-s+1); +lua_pushinteger(L,res-s); +return push_captures(&ms,NULL,0)+2; +} +else +return push_captures(&ms,s1,res); +} +}while(s1++L,3,&l); +for(i=0;iL; +switch(lua_type(L,3)){ +case 3: +case 4:{ +add_s(ms,b,s,e); +return; +} +case 6:{ +int n; +lua_pushvalue(L,3); +n=push_captures(ms,s,e); +lua_call(L,n,1); +break; +} +case 5:{ +push_onecapture(ms,0,s,e); +lua_gettable(L,3); +break; +} +} +if(!lua_toboolean(L,-1)){ +lua_pop(L,1); +lua_pushlstring(L,s,e-s); +} +else if(!lua_isstring(L,-1)) +luaL_error(L,"invalid replacement value (a %s)",luaL_typename(L,-1)); +luaL_addvalue(b); +} +static int str_gsub(lua_State*L){ +size_t srcl; +const char*src=luaL_checklstring(L,1,&srcl); +const char*p=luaL_checkstring(L,2); +int tr=lua_type(L,3); +int max_s=luaL_optint(L,4,srcl+1); +int anchor=(*p=='^')?(p++,1):0; +int n=0; +MatchState ms; +luaL_Buffer b; +luaL_argcheck(L,tr==3||tr==4|| +tr==6||tr==5,3, +"string/function/table expected"); +luaL_buffinit(L,&b); +ms.L=L; +ms.src_init=src; +ms.src_end=src+srcl; +while(nsrc) +src=e; +else if(src=sizeof("-+ #0")) +luaL_error(L,"invalid format (repeated flags)"); +if(isdigit(uchar(*p)))p++; +if(isdigit(uchar(*p)))p++; +if(*p=='.'){ +p++; +if(isdigit(uchar(*p)))p++; +if(isdigit(uchar(*p)))p++; +} +if(isdigit(uchar(*p))) +luaL_error(L,"invalid format (width or precision too long)"); +*(form++)='%'; +strncpy(form,strfrmt,p-strfrmt+1); +form+=p-strfrmt+1; +*form='\0'; +return p; +} +static void addintlen(char*form){ +size_t l=strlen(form); +char spec=form[l-1]; +strcpy(form+l-1,"l"); +form[l+sizeof("l")-2]=spec; +form[l+sizeof("l")-1]='\0'; +} +static int str_format(lua_State*L){ +int top=lua_gettop(L); +int arg=1; +size_t sfl; +const char*strfrmt=luaL_checklstring(L,arg,&sfl); +const char*strfrmt_end=strfrmt+sfl; +luaL_Buffer b; +luaL_buffinit(L,&b); +while(strfrmttop) +luaL_argerror(L,arg,"no value"); +strfrmt=scanformat(L,strfrmt,form); +switch(*strfrmt++){ +case'c':{ +sprintf(buff,form,(int)luaL_checknumber(L,arg)); +break; +} +case'd':case'i':{ +addintlen(form); +sprintf(buff,form,(long)luaL_checknumber(L,arg)); +break; +} +case'o':case'u':case'x':case'X':{ +addintlen(form); +sprintf(buff,form,(unsigned long)luaL_checknumber(L,arg)); +break; +} +case'e':case'E':case'f': +case'g':case'G':{ +sprintf(buff,form,(double)luaL_checknumber(L,arg)); +break; +} +case'q':{ +addquoted(L,&b,arg); +continue; +} +case's':{ +size_t l; +const char*s=luaL_checklstring(L,arg,&l); +if(!strchr(form,'.')&&l>=100){ +lua_pushvalue(L,arg); +luaL_addvalue(&b); +continue; +} +else{ +sprintf(buff,form,s); +break; +} +} +default:{ +return luaL_error(L,"invalid option "LUA_QL("%%%c")" to " +LUA_QL("format"),*(strfrmt-1)); +} +} +luaL_addlstring(&b,buff,strlen(buff)); +} +} +luaL_pushresult(&b); +return 1; +} +static const luaL_Reg strlib[]={ +{"byte",str_byte}, +{"char",str_char}, +{"find",str_find}, +{"format",str_format}, +{"gmatch",gmatch}, +{"gsub",str_gsub}, +{"lower",str_lower}, +{"match",str_match}, +{"rep",str_rep}, +{"sub",str_sub}, +{"upper",str_upper}, +{NULL,NULL} +}; +static void createmetatable(lua_State*L){ +lua_createtable(L,0,1); +lua_pushliteral(L,""); +lua_pushvalue(L,-2); +lua_setmetatable(L,-2); +lua_pop(L,1); +lua_pushvalue(L,-2); +lua_setfield(L,-2,"__index"); +lua_pop(L,1); +} +static int luaopen_string(lua_State*L){ +luaL_register(L,"string",strlib); +createmetatable(L); +return 1; +} +static const luaL_Reg lualibs[]={ +{"",luaopen_base}, +{"table",luaopen_table}, +{"io",luaopen_io}, +{"os",luaopen_os}, +{"string",luaopen_string}, +{NULL,NULL} +}; +static void luaL_openlibs(lua_State*L){ +const luaL_Reg*lib=lualibs; +for(;lib->func;lib++){ +lua_pushcfunction(L,lib->func); +lua_pushstring(L,lib->name); +lua_call(L,1,0); +} +} +typedef unsigned int UB; +static UB barg(lua_State*L,int idx){ +union{lua_Number n;U64 b;}bn; +bn.n=lua_tonumber(L,idx)+6755399441055744.0; +if(bn.n==0.0&&!lua_isnumber(L,idx))luaL_typerror(L,idx,"number"); +return(UB)bn.b; +} +#define BRET(b)lua_pushnumber(L,(lua_Number)(int)(b));return 1; +static int tobit(lua_State*L){ +BRET(barg(L,1))} +static int bnot(lua_State*L){ +BRET(~barg(L,1))} +static int band(lua_State*L){ +int i;UB b=barg(L,1);for(i=lua_gettop(L);i>1;i--)b&=barg(L,i);BRET(b)} +static int bor(lua_State*L){ +int i;UB b=barg(L,1);for(i=lua_gettop(L);i>1;i--)b|=barg(L,i);BRET(b)} +static int bxor(lua_State*L){ +int i;UB b=barg(L,1);for(i=lua_gettop(L);i>1;i--)b^=barg(L,i);BRET(b)} +static int lshift(lua_State*L){ +UB b=barg(L,1),n=barg(L,2)&31;BRET(b<>n)} +static int arshift(lua_State*L){ +UB b=barg(L,1),n=barg(L,2)&31;BRET((int)b>>n)} +static int rol(lua_State*L){ +UB b=barg(L,1),n=barg(L,2)&31;BRET((b<>(32-n)))} +static int ror(lua_State*L){ +UB b=barg(L,1),n=barg(L,2)&31;BRET((b>>n)|(b<<(32-n)))} +static int bswap(lua_State*L){ +UB b=barg(L,1);b=(b>>24)|((b>>8)&0xff00)|((b&0xff00)<<8)|(b<<24);BRET(b)} +static int tohex(lua_State*L){ +UB b=barg(L,1); +int n=lua_isnone(L,2)?8:(int)barg(L,2); +const char*hexdigits="0123456789abcdef"; +char buf[8]; +int i; +if(n<0){n=-n;hexdigits="0123456789ABCDEF";} +if(n>8)n=8; +for(i=(int)n;--i>=0;){buf[i]=hexdigits[b&15];b>>=4;} +lua_pushlstring(L,buf,(size_t)n); +return 1; +} +static const struct luaL_Reg bitlib[]={ +{"tobit",tobit}, +{"bnot",bnot}, +{"band",band}, +{"bor",bor}, +{"bxor",bxor}, +{"lshift",lshift}, +{"rshift",rshift}, +{"arshift",arshift}, +{"rol",rol}, +{"ror",ror}, +{"bswap",bswap}, +{"tohex",tohex}, +{NULL,NULL} +}; +int main(int argc,char**argv){ +lua_State*L=luaL_newstate(); +int i; +luaL_openlibs(L); +luaL_register(L,"bit",bitlib); +if(argc<2)return sizeof(void*); +lua_createtable(L,0,1); +lua_pushstring(L,argv[1]); +lua_rawseti(L,-2,0); +lua_setglobal(L,"arg"); +if(luaL_loadfile(L,argv[1])) +goto err; +for(i=2;i | + +----------------------------------------------------------------------+ +*/ + +#include +#include "jit/zend_jit.h" + +#ifdef HAVE_JIT + +#include "Optimizer/zend_ssa.h" + +// TODO: define DASM_M_GROW and DASM_M_FREE to use CG(arena) ??? + +#include "dynasm/dasm_proto.h" +#include "dynasm/dasm_x86.h" +#include "jit/zend_jit_x86.c" + +#if _WIN32 +# include +#else +# include +# if !defined(MAP_ANONYMOUS) && defined(MAP_ANON) +# define MAP_ANONYMOUS MAP_ANON +# endif +#endif + +#define DASM_ALIGNMENT 16 + +static void *dasm_buf = NULL; +static void *dasm_ptr = NULL; +static void *dasm_end = NULL; + +static void *dasm_link_and_encode(dasm_State **dasm_state) +{ + size_t size; + int ret; + void *entry; + + if (dasm_link(dasm_state, &size) != DASM_S_OK) { + // TODO: dasm_link() failed ??? + return NULL; + } + + size = ZEND_MM_ALIGNED_SIZE_EX(size, DASM_ALIGNMENT); + + if ((void*)((char*)dasm_ptr + size) > dasm_end) { + // TODO: jit_buffer_size overflow ??? + return NULL; + } + +#ifdef HAVE_MPROTECT +// mprotect(dasm_ptr, size, PROT_READ | PROT_WRITE); +//??? mprotect(dasm_buf, ((char*)dasm_end) - ((char*)dasm_buf), PROT_READ | PROT_WRITE); +#endif + + ret = dasm_encode(dasm_state, dasm_ptr); + +#ifdef HAVE_MPROTECT +// mprotect(dasm_ptr, size, PROT_READ | PROT_EXEC); +//??? mprotect(dasm_buf, ((char*)dasm_end) - ((char*)dasm_buf), PROT_READ | PROT_EXEC); +#endif + + if (ret != DASM_S_OK) { + // TODO: dasm_encode() failed ??? + return NULL; + } + + entry = dasm_ptr; + dasm_ptr = (void*)((char*)dasm_ptr + size); + + return entry; +} + +static size_t jit_page_size(void) +{ +#ifdef _WIN32 + SYSTEM_INFO system_info; + GetSystemInfo(&system_info); + return system_info.dwPageSize; +#else + return getpagesize(); +#endif +} + +static void *jit_alloc(size_t size, int shared) +{ +#ifdef _WIN32 + return VirtualAlloc(0, size, MEM_COMMIT | MEM_RESERVE, PAGE_EXECUTE_READWRITE); +#else + void *p; + +# ifdef MAP_HUGETLB + p = mmap(NULL, size, +//???# ifdef HAVE_MPROTECT +//??? PROT_NONE, +//???# else + PROT_EXEC | PROT_READ | PROT_WRITE, +//???# endif + (shared ? MAP_SHARED : MAP_PRIVATE) | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0); + if (p != MAP_FAILED) { + return (void*)p; + } +# endif + p = mmap(NULL, size, +//???# ifdef HAVE_MPROTECT +//??? PROT_NONE, +//???# else + PROT_EXEC | PROT_READ | PROT_WRITE, +//???# endif + (shared ? MAP_SHARED : MAP_PRIVATE) | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + return NULL; + } + + return (void*)p; +#endif +} + +static void jit_free(void *p, size_t size) +{ +#ifdef _WIN32 + VirtualFree(p, 0, MEM_RELEASE); +#else + munmap(p, size); +#endif +} + +ZEND_API int zend_jit(zend_op_array *op_array, zend_persistent_script* main_persistent_script) +{ + uint32_t flags; + zend_ssa ssa; + void *checkpoint; + int b, i, end; + zend_op *opline; + dasm_State* dasm_state = NULL; + void *handler; + + if (!dasm_buf) { + return FAILURE; + } + + checkpoint = zend_arena_checkpoint(CG(arena)); + + /* Build SSA */ + memset(&ssa, 0, sizeof(zend_ssa)); + + if (zend_build_cfg(&CG(arena), op_array, ZEND_RT_CONSTANTS | ZEND_SSA_RC_INFERENCE, &ssa.cfg, &flags) != SUCCESS) { + goto jit_failure; + } + + if (zend_cfg_build_predecessors(&CG(arena), &ssa.cfg) != SUCCESS) { + goto jit_failure; + } + + /* Compute Dominators Tree */ + if (zend_cfg_compute_dominators_tree(op_array, &ssa.cfg) != SUCCESS) { + goto jit_failure; + } + + /* Identify reducible and irreducible loops */ + if (zend_cfg_identify_loops(op_array, &ssa.cfg, &flags) != SUCCESS) { + goto jit_failure; + } + + dasm_init(&dasm_state, DASM_MAXSECTION); + dasm_setupglobal(&dasm_state, dasm_labels, lbl__MAX); + dasm_setup(&dasm_state, dasm_actions); + + dasm_growpc(&dasm_state, ssa.cfg.blocks_count * 2); + + zend_jit_align_func(&dasm_state); + for (b = 0; b < ssa.cfg.blocks_count; b++) { + if ((ssa.cfg.blocks[b].flags & ZEND_BB_REACHABLE) == 0) { + continue; + } + if (ssa.cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_ENTRY)) { + zend_jit_label(&dasm_state, ssa.cfg.blocks_count + b); + zend_jit_prologue(&dasm_state); + } + zend_jit_label(&dasm_state, b); + if (ssa.cfg.blocks[b].flags & ZEND_BB_LOOP_HEADER) { + if (!zend_jit_check_timeout(&dasm_state)) { + goto jit_failure; + } + } + if (!ssa.cfg.blocks[b].len) { + continue; + } + end = ssa.cfg.blocks[b].start + ssa.cfg.blocks[b].len - 1; + for (i = ssa.cfg.blocks[b].start; i <= end; i++) { + opline = op_array->opcodes + i; + switch (opline->opcode) { + case ZEND_RECV: + /* RECV may be skipped */ + if (!zend_jit_recv(&dasm_state, opline)) { + goto jit_failure; + } + break; + case ZEND_RECV_INIT: + case ZEND_BIND_GLOBAL: + if (opline->opcode != op_array->opcodes[i+1].opcode) { + /* repeatable opcodes */ + if (!zend_jit_handler(&dasm_state, opline)) { + goto jit_failure; + } + } + break; + case ZEND_NOP: + if (!zend_jit_skip_handler(&dasm_state)) { + goto jit_failure; + } + break; + case ZEND_OP_DATA: + break; + case ZEND_JMP: + if (!zend_jit_set_opline(&dasm_state, OP_JMP_ADDR(opline, opline->op1)) || + !zend_jit_jmp(&dasm_state, ssa.cfg.blocks[b].successors[0])) { + goto jit_failure; + } + break; + case ZEND_CATCH: + case ZEND_FAST_CALL: + case ZEND_FAST_RET: + case ZEND_GENERATOR_CREATE: + case ZEND_GENERATOR_RETURN: + case ZEND_RETURN_BY_REF: + case ZEND_RETURN: + case ZEND_EXIT: + /* switch through trampoline */ + case ZEND_YIELD: + case ZEND_YIELD_FROM: + if (!zend_jit_tail_handler(&dasm_state, opline)) { + goto jit_failure; + } + break; + /* stackless execution */ + case ZEND_INCLUDE_OR_EVAL: + case ZEND_DO_FCALL: + case ZEND_DO_UCALL: + case ZEND_DO_FCALL_BY_NAME: + if (!zend_jit_call(&dasm_state, opline)) { + goto jit_failure; + } + break; + case ZEND_JMPZ: + case ZEND_JMPNZ: + if (opline != 0) { + if ((opline-1)->opcode == ZEND_IS_EQUAL || + (opline-1)->opcode == ZEND_IS_NOT_EQUAL || + (opline-1)->opcode == ZEND_IS_SMALLER || + (opline-1)->opcode == ZEND_IS_SMALLER_OR_EQUAL || + (opline-1)->opcode == ZEND_CASE) { + /* might be smart branch */ + if (!zend_jit_smart_branch(&dasm_state, opline, (b + 1), ssa.cfg.blocks[b].successors[0])) { + goto jit_failure; + } + /* break missing intentionally */ + } else if ((opline-1)->opcode == ZEND_IS_IDENTICAL || + (opline-1)->opcode == ZEND_IS_NOT_IDENTICAL || + (opline-1)->opcode == ZEND_ISSET_ISEMPTY_VAR || + (opline-1)->opcode == ZEND_ISSET_ISEMPTY_STATIC_PROP || + (opline-1)->opcode == ZEND_ISSET_ISEMPTY_DIM_OBJ || + (opline-1)->opcode == ZEND_ISSET_ISEMPTY_PROP_OBJ || + (opline-1)->opcode == ZEND_INSTANCEOF || + (opline-1)->opcode == ZEND_TYPE_CHECK || + (opline-1)->opcode == ZEND_DEFINED) { + /* smart branch */ + if (!zend_jit_cond_jmp(&dasm_state, opline + 1, ssa.cfg.blocks[b].successors[0])) { + goto jit_failure; + } + break; + } + } + /* break missing intentionally */ + case ZEND_JMPZ_EX: + case ZEND_JMPNZ_EX: + case ZEND_JMP_SET: + case ZEND_COALESCE: + case ZEND_FE_RESET_R: + case ZEND_FE_RESET_RW: + case ZEND_ASSERT_CHECK: + if (!zend_jit_handler(&dasm_state, opline) || + !zend_jit_cond_jmp(&dasm_state, opline + 1, ssa.cfg.blocks[b].successors[0])) { + goto jit_failure; + } + break; + case ZEND_NEW: + if (!zend_jit_new(&dasm_state, opline)) { + goto jit_failure; + } + if (EXPECTED(opline->extended_value == 0 && (opline+1)->opcode == ZEND_DO_FCALL)) { + i++; + } + break; + case ZEND_JMPZNZ: + if (!zend_jit_handler(&dasm_state, opline) || + !zend_jit_cond_jmp(&dasm_state, OP_JMP_ADDR(opline, opline->op2), ssa.cfg.blocks[b].successors[1]) || + !zend_jit_jmp(&dasm_state, ssa.cfg.blocks[b].successors[0])) { + goto jit_failure; + } + break; + case ZEND_FE_FETCH_R: + case ZEND_FE_FETCH_RW: + case ZEND_DECLARE_ANON_CLASS: + case ZEND_DECLARE_ANON_INHERITED_CLASS: + if (!zend_jit_handler(&dasm_state, opline) || + !zend_jit_cond_jmp(&dasm_state, opline + 1, ssa.cfg.blocks[b].successors[0])) { + goto jit_failure; + } + break; + default: + if (!zend_jit_handler(&dasm_state, opline)) { + goto jit_failure; + } + } + } + } + + handler = dasm_link_and_encode(&dasm_state); + if (!handler) { + goto jit_failure; + } + for (b = 0; b < ssa.cfg.blocks_count; b++) { + if (ssa.cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_ENTRY)) { + opline = op_array->opcodes + ssa.cfg.blocks[b].start; + opline->handler = (void*)(((char*)handler) + + dasm_getpclabel(&dasm_state, ssa.cfg.blocks_count + b)); + } + } + dasm_free(&dasm_state); + zend_arena_release(&CG(arena), checkpoint); + return SUCCESS; + +jit_failure: + if (dasm_state) { + dasm_free(&dasm_state); + } + zend_arena_release(&CG(arena), checkpoint); + return FAILURE; +} + +ZEND_API int zend_jit_startup(size_t size) +{ + size_t page_size = jit_page_size(); + int shared = 1; + void *buf; + + /* Round up to the page size, which should be a power of two. */ + page_size = jit_page_size(); + if (!page_size || (page_size & (page_size - 1))) { + abort(); + } + size = (size + page_size - 1) & ~(page_size - 1); + + buf = jit_alloc(size, shared); + + if (!buf) { + return FAILURE; + } + + dasm_buf = dasm_ptr = buf; + dasm_end = (void*)(((char*)dasm_buf)+size); + + return SUCCESS; +} + +ZEND_API void zend_jit_shutdown(void) +{ + if (dasm_buf) { + jit_free(dasm_buf, ((char*)dasm_end) - ((char*)dasm_buf)); + } +} + +#else /* HAVE_JIT */ + +ZEND_API int zend_jit(zend_op_array *op_array, zend_persistent_script* main_persistent_script) +{ + return FAILURE; +} + +ZEND_API int zend_jit_startup(size_t size) +{ + return FAILURE; +} + +ZEND_API void zend_jit_shutdown(void) +{ +} + +#endif /* HAVE_JIT */ + +/* + * Local variables: + * tab-width: 4 + * c-basic-offset: 4 + * indent-tabs-mode: t + * End: + */ diff --git a/ext/opcache/jit/zend_jit.h b/ext/opcache/jit/zend_jit.h new file mode 100644 index 0000000000000..23cc4d5cc9f1b --- /dev/null +++ b/ext/opcache/jit/zend_jit.h @@ -0,0 +1,36 @@ +/* + +----------------------------------------------------------------------+ + | Zend OPcache | + +----------------------------------------------------------------------+ + | Copyright (c) 1998-2016 The PHP Group | + +----------------------------------------------------------------------+ + | This source file is subject to version 3.01 of the PHP license, | + | that is bundled with this package in the file LICENSE, and is | + | available through the world-wide-web at the following url: | + | http://www.php.net/license/3_01.txt | + | If you did not receive a copy of the PHP license and are unable to | + | obtain it through the world-wide-web, please send a note to | + | license@php.net so we can mail you a copy immediately. | + +----------------------------------------------------------------------+ + | Authors: Dmitry Stogov | + +----------------------------------------------------------------------+ +*/ + +/* $Id:$ */ + +#ifndef HAVE_JIT_H +#define HAVE_JIT_H + +ZEND_API int zend_jit(zend_op_array *op_array, zend_persistent_script* main_persistent_script); +ZEND_API int zend_jit_startup(size_t size); +ZEND_API void zend_jit_shutdown(void); + +#endif /* HAVE_JIT_H */ + +/* + * Local variables: + * tab-width: 4 + * c-basic-offset: 4 + * indent-tabs-mode: t + * End: + */ \ No newline at end of file diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc new file mode 100644 index 0000000000000..e07915e88ba2b --- /dev/null +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -0,0 +1,204 @@ +/* + * +----------------------------------------------------------------------+ + * | Zend OPcache | + * +----------------------------------------------------------------------+ + * | Copyright (c) 1998-2016 The PHP Group | + * +----------------------------------------------------------------------+ + * | This source file is subject to version 3.01 of the PHP license, | + * | that is bundled with this package in the file LICENSE, and is | + * | available through the world-wide-web at the following url: | + * | http://www.php.net/license/3_01.txt | + * | If you did not receive a copy of the PHP license and are unable to | + * | obtain it through the world-wide-web, please send a note to | + * | license@php.net so we can mail you a copy immediately. | + * +----------------------------------------------------------------------+ + * | Authors: Dmitry Stogov | + * +----------------------------------------------------------------------+ + */ + +|.if X64 + |.arch x64 +|.else + |.arch x86 +|.endif + +|.if X64 + |.define FP, r14 + |.define IP, r15 + |.define IPl, r15d +|.else + |.define FP, esi + |.define IP, edi + |.define IPl, edi +|.endif + +|.actionlist dasm_actions + +|.globals lbl_ +static void* dasm_labels[lbl__MAX]; + +|.section code + +static int zend_jit_align_func(dasm_State **Dst) +{ + |.align 16 + return 1; +} + +static int zend_jit_prologue(dasm_State **Dst) +{ + |.if X64 + | //add rsp, 8 // stack alignment + |.endif + return 1; +} + +static int zend_jit_label(dasm_State **Dst, unsigned int label) +{ + |=>label: + return 1; +} + +static int zend_jit_check_timeout(dasm_State **Dst) +{ + // ??? + | cmp byte [&EG(vm_interrupt)], 0 + | jne extern zend_timeout + return 1; +} + +static int zend_jit_check_exception(dasm_State **Dst) +{ + const void *handler = EG(exception_op)->handler; + + | cmp aword [&EG(exception)], 0 + | jne &handler + return 1; +} + +static int zend_jit_handler(dasm_State **Dst, zend_op *opline) +{ + const void *handler = opline->handler; + + |.if X64 + | mov64 rax, ((ptrdiff_t)handler) + | call rax + |.else + | call aword &handler + |.endif + zend_jit_check_exception(Dst); + return 1; +} + +static int zend_jit_tail_handler(dasm_State **Dst, zend_op *opline) +{ + const void *handler = opline->handler; + + |.if X64 + | //sub rsp, 8 // stack alignment + | mov64 rax, ((ptrdiff_t)handler) + | jmp rax + |.else + | jmp aword &handler + |.endif + return 1; +} + +static int zend_jit_skip_handler(dasm_State **Dst) +{ + | add IP, sizeof(zend_op) + return 1; +} + +static int zend_jit_set_opline(dasm_State **Dst, zend_op *target_opline) +{ + |.if X64 + | mov64 IP, ((ptrdiff_t)target_opline) + |.else + | mov IP, target_opline + |.endif + return 1; +} + +static int zend_jit_jmp(dasm_State **Dst, unsigned int target_label) +{ + | jmp =>target_label + return 1; +} + +static int zend_jit_cond_jmp(dasm_State **Dst, zend_op *next_opline, unsigned int target_label) +{ + | cmp IPl, next_opline + | jnz =>target_label + return 1; +} + +static int zend_jit_smart_branch(dasm_State **Dst, zend_op *opline, unsigned int next_label, unsigned int target_label) +{ + zend_op *next_opline = opline + 1; + zend_op *target_opline = OP_JMP_ADDR(opline, opline->op2); + + | cmp IPl, next_opline + | jz =>next_label + | cmp IPl, target_opline + | jz =>target_label + return 1; +} + +static int zend_jit_recv(dasm_State **Dst, zend_op *opline) +{ + | cmp IPl, opline + | jnz >1 + zend_jit_handler(Dst, opline); + |1: + return 1; +} + +#ifdef CONTEXT_THREDED_JIT +static int zend_jit_context_threaded_call(dasm_State **Dst, zend_op *opline) +{ + if (!zend_jit_handler(Dst, opline)) return 0; + if (opline->opcode == ZEND_DO_UCALL) { + | call aword [IP] + } else { + zend_op *next_opline = opline + 1; + + | cmp IPl, next_opline + | jz >1 + | call aword [IP] + |1: + } + return 1; +} +#endif + +static int zend_jit_call(dasm_State **Dst, zend_op *opline) +{ +#ifdef CONTEXT_THREDED_JIT + return zend_jit_context_threaded_call(Dst, opline); +#else + return zend_jit_tail_handler(Dst, opline); +#endif +} + +static int zend_jit_new(dasm_State **Dst, zend_op *opline) +{ + if (!zend_jit_handler(Dst, opline)) return 0; + if (EXPECTED(opline->extended_value == 0 && (opline+1)->opcode == ZEND_DO_FCALL)) { + zend_op *next_opline = opline + 1; + + | cmp IPl, next_opline + | jnz >1 + zend_jit_call(Dst, next_opline); + |1: + } + return 1; +} + +/* + * Local variables: + * tab-width: 4 + * c-basic-offset: 4 + * indent-tabs-mode: t + * End: + */ diff --git a/ext/opcache/zend_accelerator_module.c b/ext/opcache/zend_accelerator_module.c index b8a34108957c9..9b8863da28c9f 100644 --- a/ext/opcache/zend_accelerator_module.c +++ b/ext/opcache/zend_accelerator_module.c @@ -317,6 +317,10 @@ ZEND_INI_BEGIN() #ifdef HAVE_HUGE_CODE_PAGES STD_PHP_INI_BOOLEAN("opcache.huge_code_pages" , "0" , PHP_INI_SYSTEM, OnUpdateBool, accel_directives.huge_code_pages, zend_accel_globals, accel_globals) #endif +#ifdef HAVE_JIT + STD_PHP_INI_ENTRY("opcache.jit_buffer_size" , "0" , PHP_INI_SYSTEM, OnUpdateLong, accel_directives.jit_buffer_size, zend_accel_globals, accel_globals) + STD_PHP_INI_ENTRY("opcache.jit_debug" , "0" , PHP_INI_SYSTEM, OnUpdateLong, accel_directives.jit_debug, zend_accel_globals, accel_globals) +#endif ZEND_INI_END() static int filename_is_in_cache(zend_string *filename) diff --git a/ext/opcache/zend_persist.c b/ext/opcache/zend_persist.c index 24a3d21ca2799..94751a6f3d55a 100644 --- a/ext/opcache/zend_persist.c +++ b/ext/opcache/zend_persist.c @@ -28,6 +28,10 @@ #include "zend_constants.h" #include "zend_operators.h" +#ifdef HAVE_JIT +# include "jit/zend_jit.h" +#endif + #define zend_accel_store(p, size) \ (p = _zend_shared_memdup((void*)p, size, 1)) #define zend_accel_memdup(p, size) \ @@ -332,6 +336,9 @@ static void zend_persist_op_array_ex(zend_op_array *op_array, zend_persistent_sc int already_stored = 0; zend_op *persist_ptr; zval *orig_literals = NULL; +#ifdef HAVE_JIT + int do_jit = 1; +#endif if (op_array->type != ZEND_USER_FUNCTION) { return; @@ -398,6 +405,9 @@ static void zend_persist_op_array_ex(zend_op_array *op_array, zend_persistent_sc persist_ptr = zend_shared_alloc_get_xlat_entry(op_array->opcodes); ZEND_ASSERT(persist_ptr != NULL); op_array->opcodes = persist_ptr; +#ifdef HAVE_JIT + do_jit = 0; +#endif } else { zend_op *new_opcodes = zend_accel_memdup(op_array->opcodes, sizeof(zend_op) * op_array->last); #if ZEND_USE_ABS_CONST_ADDR || ZEND_USE_ABS_JMP_ADDR @@ -558,6 +568,12 @@ static void zend_persist_op_array_ex(zend_op_array *op_array, zend_persistent_sc } ZCG(mem) = (void*)((char*)ZCG(mem) + ZEND_ALIGNED_SIZE(zend_extensions_op_array_persist(op_array, ZCG(mem)))); + +#ifdef HAVE_JIT + if (do_jit && ZCG(accel_directives).jit_buffer_size) { + zend_jit(op_array, main_persistent_script TSRMLS_CC); + } +#endif } static void zend_persist_op_array(zval *zv) From f452c60a2e92a508c80ad15bb8f4070ea6245515 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 18 Aug 2016 12:16:59 +0300 Subject: [PATCH 002/569] Autodetect 32/64-bit target --- ext/opcache/Makefile.frag | 2 +- ext/opcache/config.m4 | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/ext/opcache/Makefile.frag b/ext/opcache/Makefile.frag index 9273e8595aa0a..2fc85bdbb8de7 100644 --- a/ext/opcache/Makefile.frag +++ b/ext/opcache/Makefile.frag @@ -3,7 +3,7 @@ $(builddir)/minilua: $(srcdir)/dynasm/minilua.c $(CC) $(srcdir)/dynasm/minilua.c -lm -o $@ $(builddir)/jit/zend_jit_x86.c: $(srcdir)/jit/zend_jit_x86.dasc $(srcdir)/dynasm/*.lua $(builddir)/minilua - $(builddir)/minilua $(srcdir)/dynasm/dynasm.lua -o $@ $(srcdir)/jit/zend_jit_x86.dasc + $(builddir)/minilua $(srcdir)/dynasm/dynasm.lua $(DASM_FLAGS) -o $@ $(srcdir)/jit/zend_jit_x86.dasc $(builddir)/jit/zend_jit.lo: $(builddir)/jit/zend_jit_x86.c diff --git a/ext/opcache/config.m4 b/ext/opcache/config.m4 index 45a44f544a10e..8cd8ea9371387 100644 --- a/ext/opcache/config.m4 +++ b/ext/opcache/config.m4 @@ -27,6 +27,17 @@ if test "$PHP_OPCACHE" != "no"; then if test "$PHP_OPCACHE_JIT" = "yes"; then AC_DEFINE(HAVE_JIT, 1, [Define to enable JIT]) + # Find out which ABI we are using. + echo 'int i;' > conftest.$ac_ext + if AC_TRY_EVAL(ac_compile); then + case `/usr/bin/file conftest.o` in + *64-bit*) + DASM_FLAGS="-D X64=1" + PHP_SUBST(DASM_FLAGS) + ;; + esac + fi + rm -rf conftest* fi AC_CHECK_FUNC(mprotect,[ From d5885945d2f64ab15f2c564dad42fd14b9abdcb6 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 18 Aug 2016 14:33:10 +0300 Subject: [PATCH 003/569] Disable writing into JIT buffer during execution --- ext/opcache/jit/zend_jit.c | 38 ++++++++++++++++++++++++++++++-------- ext/opcache/jit/zend_jit.h | 2 ++ ext/opcache/zend_persist.c | 8 ++++++++ 3 files changed, 40 insertions(+), 8 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 1e614f056d074..c5c57bda0add0 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -105,22 +105,22 @@ static void *jit_alloc(size_t size, int shared) # ifdef MAP_HUGETLB p = mmap(NULL, size, -//???# ifdef HAVE_MPROTECT -//??? PROT_NONE, -//???# else +# ifdef HAVE_MPROTECT + PROT_NONE, +# else PROT_EXEC | PROT_READ | PROT_WRITE, -//???# endif +# endif (shared ? MAP_SHARED : MAP_PRIVATE) | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0); if (p != MAP_FAILED) { return (void*)p; } # endif p = mmap(NULL, size, -//???# ifdef HAVE_MPROTECT -//??? PROT_NONE, -//???# else +# ifdef HAVE_MPROTECT + PROT_NONE, +# else PROT_EXEC | PROT_READ | PROT_WRITE, -//???# endif +# endif (shared ? MAP_SHARED : MAP_PRIVATE) | MAP_ANONYMOUS, -1, 0); if (p == MAP_FAILED) { return NULL; @@ -353,6 +353,20 @@ ZEND_API int zend_jit(zend_op_array *op_array, zend_persistent_script* main_pers return FAILURE; } +ZEND_API void zend_jit_unprotect(void) +{ +#ifdef HAVE_MPROTECT + mprotect(dasm_buf, ((char*)dasm_end) - ((char*)dasm_buf), PROT_READ | PROT_WRITE); +#endif +} + +ZEND_API void zend_jit_protect(void) +{ +#ifdef HAVE_MPROTECT + mprotect(dasm_buf, ((char*)dasm_end) - ((char*)dasm_buf), PROT_READ | PROT_EXEC); +#endif +} + ZEND_API int zend_jit_startup(size_t size) { size_t page_size = jit_page_size(); @@ -392,6 +406,14 @@ ZEND_API int zend_jit(zend_op_array *op_array, zend_persistent_script* main_pers return FAILURE; } +ZEND_API void zend_jit_unprotect(void) +{ +} + +ZEND_API void zend_jit_protect(void) +{ +} + ZEND_API int zend_jit_startup(size_t size) { return FAILURE; diff --git a/ext/opcache/jit/zend_jit.h b/ext/opcache/jit/zend_jit.h index 23cc4d5cc9f1b..9e93f456dc695 100644 --- a/ext/opcache/jit/zend_jit.h +++ b/ext/opcache/jit/zend_jit.h @@ -22,6 +22,8 @@ #define HAVE_JIT_H ZEND_API int zend_jit(zend_op_array *op_array, zend_persistent_script* main_persistent_script); +ZEND_API void zend_jit_unprotect(void); +ZEND_API void zend_jit_protect(void); ZEND_API int zend_jit_startup(size_t size); ZEND_API void zend_jit_shutdown(void); diff --git a/ext/opcache/zend_persist.c b/ext/opcache/zend_persist.c index 94751a6f3d55a..ce9d9ceebcdc8 100644 --- a/ext/opcache/zend_persist.c +++ b/ext/opcache/zend_persist.c @@ -856,10 +856,18 @@ zend_persistent_script *zend_accel_script_persist(zend_persistent_script *script script->arena_mem = ZCG(arena_mem) = ZCG(mem); ZCG(mem) = (void*)((char*)ZCG(mem) + script->arena_size); +#ifdef HAVE_JIT + zend_jit_unprotect(); +#endif + zend_accel_persist_class_table(&script->script.class_table); zend_hash_persist(&script->script.function_table, zend_persist_op_array); zend_persist_op_array_ex(&script->script.main_op_array, script); +#ifdef HAVE_JIT + zend_jit_protect(); +#endif + return script; } From ab6c3abac475784c2199a3d0e9cee87a6890e919 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 18 Aug 2016 15:34:03 +0300 Subject: [PATCH 004/569] Fixed stack alignment for 64-bit JIT --- ext/opcache/jit/zend_jit_x86.dasc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index e07915e88ba2b..7f825b6ae9740 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -48,7 +48,7 @@ static int zend_jit_align_func(dasm_State **Dst) static int zend_jit_prologue(dasm_State **Dst) { |.if X64 - | //add rsp, 8 // stack alignment + | sub rsp, 8 // stack alignment |.endif return 1; } @@ -95,7 +95,7 @@ static int zend_jit_tail_handler(dasm_State **Dst, zend_op *opline) const void *handler = opline->handler; |.if X64 - | //sub rsp, 8 // stack alignment + | add rsp, 8 // stack alignment | mov64 rax, ((ptrdiff_t)handler) | jmp rax |.else From 363f82d883f82d8e0eed06f5e4ec5b6ea9c37916 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 18 Aug 2016 16:21:29 +0300 Subject: [PATCH 005/569] Fixed stack alignment in 32-bit JIT and exception handling in 64-bit JIT --- ext/opcache/jit/zend_jit.c | 17 ++++++++++++++ ext/opcache/jit/zend_jit_x86.dasc | 38 +++++++++++++++++++++++++++---- 2 files changed, 50 insertions(+), 5 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index c5c57bda0add0..82ff86d1c83b0 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -17,6 +17,7 @@ */ #include +#include "Zend/zend_execute.h" #include "jit/zend_jit.h" #ifdef HAVE_JIT @@ -372,6 +373,8 @@ ZEND_API int zend_jit_startup(size_t size) size_t page_size = jit_page_size(); int shared = 1; void *buf; + dasm_State* dasm_state = NULL; + int ret; /* Round up to the page size, which should be a power of two. */ page_size = jit_page_size(); @@ -389,6 +392,20 @@ ZEND_API int zend_jit_startup(size_t size) dasm_buf = dasm_ptr = buf; dasm_end = (void*)(((char*)dasm_buf)+size); + zend_jit_unprotect(); + + dasm_init(&dasm_state, DASM_MAXSECTION); + dasm_setupglobal(&dasm_state, dasm_labels, lbl__MAX); + dasm_setup(&dasm_state, dasm_actions); + ret = zend_jit_stubs(&dasm_state) && dasm_link_and_encode(&dasm_state); + dasm_free(&dasm_state); + + zend_jit_protect(); + + if (!ret) { + return FAILURE; + } + return SUCCESS; } diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 7f825b6ae9740..f38a2f756bdcb 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -39,6 +39,34 @@ static void* dasm_labels[lbl__MAX]; |.section code +static int zend_jit_stubs(dasm_State **Dst) +{ + // TODO: check interrupt_handler ??? + |.align 16 + |->interrupt_handler: + |.if X64 + | add rsp, 8 // stack alignment + | mov64 rax, ((ptrdiff_t)zend_timeout) + | jmp rax + |.else + | add esp, 12 // stack alignment + | jmp extern zend_timeout + |.endif + | + |.align 16 + |->exception_handler: + |.if X64 + | add rsp, 8 // stack alignment + | mov64 rax, ((ptrdiff_t)EG(exception_op)->handler) + | jmp rax + |.else + | add esp, 12 // stack alignment + | jmp &(EG(exception_op)->handler) + |.endif + + return 1; +} + static int zend_jit_align_func(dasm_State **Dst) { |.align 16 @@ -49,6 +77,8 @@ static int zend_jit_prologue(dasm_State **Dst) { |.if X64 | sub rsp, 8 // stack alignment + |.else + | sub esp, 12 // stack alignment |.endif return 1; } @@ -61,18 +91,15 @@ static int zend_jit_label(dasm_State **Dst, unsigned int label) static int zend_jit_check_timeout(dasm_State **Dst) { - // ??? | cmp byte [&EG(vm_interrupt)], 0 - | jne extern zend_timeout + | jne ->interrupt_handler return 1; } static int zend_jit_check_exception(dasm_State **Dst) { - const void *handler = EG(exception_op)->handler; - | cmp aword [&EG(exception)], 0 - | jne &handler + | jne ->exception_handler return 1; } @@ -99,6 +126,7 @@ static int zend_jit_tail_handler(dasm_State **Dst, zend_op *opline) | mov64 rax, ((ptrdiff_t)handler) | jmp rax |.else + | add esp, 12 // stack alignment | jmp aword &handler |.endif return 1; From ee0e4af0d983ab0c7577426f5906347cbf28b426 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 18 Aug 2016 16:27:39 +0300 Subject: [PATCH 006/569] Removed deprecated code --- ext/opcache/jit/zend_jit.c | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 82ff86d1c83b0..06876c857d7a2 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -63,18 +63,8 @@ static void *dasm_link_and_encode(dasm_State **dasm_state) return NULL; } -#ifdef HAVE_MPROTECT -// mprotect(dasm_ptr, size, PROT_READ | PROT_WRITE); -//??? mprotect(dasm_buf, ((char*)dasm_end) - ((char*)dasm_buf), PROT_READ | PROT_WRITE); -#endif - ret = dasm_encode(dasm_state, dasm_ptr); -#ifdef HAVE_MPROTECT -// mprotect(dasm_ptr, size, PROT_READ | PROT_EXEC); -//??? mprotect(dasm_buf, ((char*)dasm_end) - ((char*)dasm_buf), PROT_READ | PROT_EXEC); -#endif - if (ret != DASM_S_OK) { // TODO: dasm_encode() failed ??? return NULL; From f1234ff035c92f95cbabfcf196231f98c3c6501c Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 18 Aug 2016 17:53:58 +0300 Subject: [PATCH 007/569] Initial GDB JIT interface (incomplete) --- ext/opcache/jit/zend_jit_gdb.c | 76 ++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 ext/opcache/jit/zend_jit_gdb.c diff --git a/ext/opcache/jit/zend_jit_gdb.c b/ext/opcache/jit/zend_jit_gdb.c new file mode 100644 index 0000000000000..db66b3cd193b4 --- /dev/null +++ b/ext/opcache/jit/zend_jit_gdb.c @@ -0,0 +1,76 @@ +/* + +----------------------------------------------------------------------+ + | Zend OPcache | + +----------------------------------------------------------------------+ + | Copyright (c) 1998-2016 The PHP Group | + +----------------------------------------------------------------------+ + | This source file is subject to version 3.01 of the PHP license, | + | that is bundled with this package in the file LICENSE, and is | + | available through the world-wide-web at the following url: | + | http://www.php.net/license/3_01.txt | + | If you did not receive a copy of the PHP license and are unable to | + | obtain it through the world-wide-web, please send a note to | + | license@php.net so we can mail you a copy immediately. | + +----------------------------------------------------------------------+ + | Authors: Dmitry Stogov | + +----------------------------------------------------------------------+ +*/ + +#define GDB_JIT_NOACTION 0 +#define GDB_JIT_REGISTER 1 +#define GDB_JIT_UNREGISTYER 2 + +struct _gdb_jit_code_entry { + struct _gdb_jit_code_entry *next_entry; + struct _gdb_jit_code_entry *prev_entry; + const char *symfile_addr; + uint64_t symfile_size; +}; + +struct _gdb_jit_descriptor { + uint32_t version; + uint32_t action_flag; + struct _gdb_jit_code_entry *relevant_entry; + struct _gdb_jit_code_entry *first_entry; +}; + +struct _gdb_jit_descriptor __jit_debug_descriptor = { + 1, GDBJIT_NOACTION, NULL, NULL +}; + +zend_never_inline void __jit_debug_register_code() +{ + __asm__ __volatile__(""); +} + +int gdb_jit_register(const char*symfile_addr, uint64_t symfile_size) +{ + struct _gdb_jit_code_entry *entry; + + entry = (struct _gdb_jit_code_entry*) malloc(sizeof(struct _gdb_jit_code_entry)); + if (!entry) { + return -1; + } + entry->symfile_addr = symfile_addr; + entry->symfile_size = symfile_size; + entry->prev_entry = NULL; + entry->next_entry = __jit_debug_descriptor.first_entry; + if (__jit_debug_descriptor.first_entry) { + __jit_debug_descriptor.first_entry->prev_entry = entry; + } + __jit_debug_descriptor.first_entry = entry; + __jit_debug_descriptor.relevant_entry = entry; + + __jit_debug_descriptor.action_flag = GDB_JIT_REGISTER; + __jit_debug_register_code(); + + return 0; +} + +/* + * Local variables: + * tab-width: 4 + * c-basic-offset: 4 + * indent-tabs-mode: t + * End: + */ From bc3bb3f43cf580fcc2c662fc347d9248885e79a2 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 19 Aug 2016 11:49:50 +0300 Subject: [PATCH 008/569] Implemented x86 disassembler and slight refactoring. --- ext/opcache/Makefile.frag | 9 - ext/opcache/config.m4 | 9 +- ext/opcache/jit/Makefile.frag | 12 + ext/opcache/{ => jit}/dynasm/dasm_arm.h | 0 ext/opcache/{ => jit}/dynasm/dasm_arm.lua | 0 ext/opcache/{ => jit}/dynasm/dasm_arm64.h | 0 ext/opcache/{ => jit}/dynasm/dasm_arm64.lua | 0 ext/opcache/{ => jit}/dynasm/dasm_mips.h | 0 ext/opcache/{ => jit}/dynasm/dasm_mips.lua | 0 ext/opcache/{ => jit}/dynasm/dasm_mips64.lua | 0 ext/opcache/{ => jit}/dynasm/dasm_ppc.h | 0 ext/opcache/{ => jit}/dynasm/dasm_ppc.lua | 0 ext/opcache/{ => jit}/dynasm/dasm_proto.h | 0 ext/opcache/{ => jit}/dynasm/dasm_x64.lua | 0 ext/opcache/{ => jit}/dynasm/dasm_x86.h | 0 ext/opcache/{ => jit}/dynasm/dasm_x86.lua | 0 ext/opcache/{ => jit}/dynasm/dynasm.lua | 0 ext/opcache/{ => jit}/dynasm/minilua.c | 0 ext/opcache/jit/libudis86/LICENSE | 22 + ext/opcache/jit/libudis86/Makefile.am | 51 + ext/opcache/jit/libudis86/decode.c | 1266 ++++ ext/opcache/jit/libudis86/decode.h | 197 + ext/opcache/jit/libudis86/extern.h | 113 + ext/opcache/jit/libudis86/itab.c | 5946 ++++++++++++++++++ ext/opcache/jit/libudis86/itab.h | 939 +++ ext/opcache/jit/libudis86/syn-att.c | 228 + ext/opcache/jit/libudis86/syn-intel.c | 224 + ext/opcache/jit/libudis86/syn.c | 212 + ext/opcache/jit/libudis86/syn.h | 53 + ext/opcache/jit/libudis86/types.h | 260 + ext/opcache/jit/libudis86/udint.h | 99 + ext/opcache/jit/libudis86/udis86.c | 458 ++ ext/opcache/jit/zend_jit.c | 32 +- ext/opcache/jit/zend_jit.h | 6 +- ext/opcache/jit/zend_jit_disasm_x86.c | 105 + ext/opcache/jit/zend_jit_gdb.c | 2 +- ext/opcache/jit/zend_jit_x86.dasc | 2 +- ext/opcache/zend_persist.c | 2 +- 38 files changed, 10227 insertions(+), 20 deletions(-) delete mode 100644 ext/opcache/Makefile.frag create mode 100644 ext/opcache/jit/Makefile.frag rename ext/opcache/{ => jit}/dynasm/dasm_arm.h (100%) rename ext/opcache/{ => jit}/dynasm/dasm_arm.lua (100%) rename ext/opcache/{ => jit}/dynasm/dasm_arm64.h (100%) rename ext/opcache/{ => jit}/dynasm/dasm_arm64.lua (100%) rename ext/opcache/{ => jit}/dynasm/dasm_mips.h (100%) rename ext/opcache/{ => jit}/dynasm/dasm_mips.lua (100%) rename ext/opcache/{ => jit}/dynasm/dasm_mips64.lua (100%) rename ext/opcache/{ => jit}/dynasm/dasm_ppc.h (100%) rename ext/opcache/{ => jit}/dynasm/dasm_ppc.lua (100%) rename ext/opcache/{ => jit}/dynasm/dasm_proto.h (100%) rename ext/opcache/{ => jit}/dynasm/dasm_x64.lua (100%) rename ext/opcache/{ => jit}/dynasm/dasm_x86.h (100%) rename ext/opcache/{ => jit}/dynasm/dasm_x86.lua (100%) rename ext/opcache/{ => jit}/dynasm/dynasm.lua (100%) rename ext/opcache/{ => jit}/dynasm/minilua.c (100%) create mode 100644 ext/opcache/jit/libudis86/LICENSE create mode 100644 ext/opcache/jit/libudis86/Makefile.am create mode 100644 ext/opcache/jit/libudis86/decode.c create mode 100644 ext/opcache/jit/libudis86/decode.h create mode 100644 ext/opcache/jit/libudis86/extern.h create mode 100644 ext/opcache/jit/libudis86/itab.c create mode 100644 ext/opcache/jit/libudis86/itab.h create mode 100644 ext/opcache/jit/libudis86/syn-att.c create mode 100644 ext/opcache/jit/libudis86/syn-intel.c create mode 100644 ext/opcache/jit/libudis86/syn.c create mode 100644 ext/opcache/jit/libudis86/syn.h create mode 100644 ext/opcache/jit/libudis86/types.h create mode 100644 ext/opcache/jit/libudis86/udint.h create mode 100644 ext/opcache/jit/libudis86/udis86.c create mode 100644 ext/opcache/jit/zend_jit_disasm_x86.c diff --git a/ext/opcache/Makefile.frag b/ext/opcache/Makefile.frag deleted file mode 100644 index 2fc85bdbb8de7..0000000000000 --- a/ext/opcache/Makefile.frag +++ /dev/null @@ -1,9 +0,0 @@ - -$(builddir)/minilua: $(srcdir)/dynasm/minilua.c - $(CC) $(srcdir)/dynasm/minilua.c -lm -o $@ - -$(builddir)/jit/zend_jit_x86.c: $(srcdir)/jit/zend_jit_x86.dasc $(srcdir)/dynasm/*.lua $(builddir)/minilua - $(builddir)/minilua $(srcdir)/dynasm/dynasm.lua $(DASM_FLAGS) -o $@ $(srcdir)/jit/zend_jit_x86.dasc - -$(builddir)/jit/zend_jit.lo: $(builddir)/jit/zend_jit_x86.c - diff --git a/ext/opcache/config.m4 b/ext/opcache/config.m4 index 8cd8ea9371387..ec59369752ce3 100644 --- a/ext/opcache/config.m4 +++ b/ext/opcache/config.m4 @@ -27,6 +27,7 @@ if test "$PHP_OPCACHE" != "no"; then if test "$PHP_OPCACHE_JIT" = "yes"; then AC_DEFINE(HAVE_JIT, 1, [Define to enable JIT]) + ZEND_JIT_SRC=jit/zend_jit.c # Find out which ABI we are using. echo 'int i;' > conftest.$ac_ext if AC_TRY_EVAL(ac_compile); then @@ -429,12 +430,14 @@ fi Optimizer/zend_func_info.c \ Optimizer/zend_call_graph.c \ Optimizer/zend_dump.c \ - jit/zend_jit.c, + $ZEND_JIT_SRC, shared,,-DZEND_ENABLE_STATIC_TSRMLS_CACHE=1,,yes) PHP_ADD_BUILD_DIR([$ext_builddir/Optimizer], 1) - PHP_ADD_BUILD_DIR([$ext_builddir/jit], 1) PHP_ADD_EXTENSION_DEP(opcache, pcre) - PHP_ADD_MAKEFILE_FRAGMENT + if test "$PHP_OPCACHE_JIT" = "yes"; then + PHP_ADD_BUILD_DIR([$ext_builddir/jit], 1) + PHP_ADD_MAKEFILE_FRAGMENT($ext_srcdir/jit/Makefile.frag) + fi fi diff --git a/ext/opcache/jit/Makefile.frag b/ext/opcache/jit/Makefile.frag new file mode 100644 index 0000000000000..a9e175406a6f4 --- /dev/null +++ b/ext/opcache/jit/Makefile.frag @@ -0,0 +1,12 @@ + +$(builddir)/minilua: $(srcdir)/jit/dynasm/minilua.c + $(CC) $(srcdir)/jit/dynasm/minilua.c -lm -o $@ + +$(builddir)/jit/zend_jit_x86.c: $(srcdir)/jit/zend_jit_x86.dasc $(srcdir)/jit/dynasm/*.lua $(builddir)/minilua + $(builddir)/minilua $(srcdir)/jit/dynasm/dynasm.lua $(DASM_FLAGS) -o $@ $(srcdir)/jit/zend_jit_x86.dasc + +$(builddir)/jit/zend_jit.lo: \ + $(builddir)/jit/zend_jit_x86.c \ + $(srcdir)/jit/zend_jit_disasm_x86.c \ + $(srcdir)/jit/zend_jit_gdb.c + diff --git a/ext/opcache/dynasm/dasm_arm.h b/ext/opcache/jit/dynasm/dasm_arm.h similarity index 100% rename from ext/opcache/dynasm/dasm_arm.h rename to ext/opcache/jit/dynasm/dasm_arm.h diff --git a/ext/opcache/dynasm/dasm_arm.lua b/ext/opcache/jit/dynasm/dasm_arm.lua similarity index 100% rename from ext/opcache/dynasm/dasm_arm.lua rename to ext/opcache/jit/dynasm/dasm_arm.lua diff --git a/ext/opcache/dynasm/dasm_arm64.h b/ext/opcache/jit/dynasm/dasm_arm64.h similarity index 100% rename from ext/opcache/dynasm/dasm_arm64.h rename to ext/opcache/jit/dynasm/dasm_arm64.h diff --git a/ext/opcache/dynasm/dasm_arm64.lua b/ext/opcache/jit/dynasm/dasm_arm64.lua similarity index 100% rename from ext/opcache/dynasm/dasm_arm64.lua rename to ext/opcache/jit/dynasm/dasm_arm64.lua diff --git a/ext/opcache/dynasm/dasm_mips.h b/ext/opcache/jit/dynasm/dasm_mips.h similarity index 100% rename from ext/opcache/dynasm/dasm_mips.h rename to ext/opcache/jit/dynasm/dasm_mips.h diff --git a/ext/opcache/dynasm/dasm_mips.lua b/ext/opcache/jit/dynasm/dasm_mips.lua similarity index 100% rename from ext/opcache/dynasm/dasm_mips.lua rename to ext/opcache/jit/dynasm/dasm_mips.lua diff --git a/ext/opcache/dynasm/dasm_mips64.lua b/ext/opcache/jit/dynasm/dasm_mips64.lua similarity index 100% rename from ext/opcache/dynasm/dasm_mips64.lua rename to ext/opcache/jit/dynasm/dasm_mips64.lua diff --git a/ext/opcache/dynasm/dasm_ppc.h b/ext/opcache/jit/dynasm/dasm_ppc.h similarity index 100% rename from ext/opcache/dynasm/dasm_ppc.h rename to ext/opcache/jit/dynasm/dasm_ppc.h diff --git a/ext/opcache/dynasm/dasm_ppc.lua b/ext/opcache/jit/dynasm/dasm_ppc.lua similarity index 100% rename from ext/opcache/dynasm/dasm_ppc.lua rename to ext/opcache/jit/dynasm/dasm_ppc.lua diff --git a/ext/opcache/dynasm/dasm_proto.h b/ext/opcache/jit/dynasm/dasm_proto.h similarity index 100% rename from ext/opcache/dynasm/dasm_proto.h rename to ext/opcache/jit/dynasm/dasm_proto.h diff --git a/ext/opcache/dynasm/dasm_x64.lua b/ext/opcache/jit/dynasm/dasm_x64.lua similarity index 100% rename from ext/opcache/dynasm/dasm_x64.lua rename to ext/opcache/jit/dynasm/dasm_x64.lua diff --git a/ext/opcache/dynasm/dasm_x86.h b/ext/opcache/jit/dynasm/dasm_x86.h similarity index 100% rename from ext/opcache/dynasm/dasm_x86.h rename to ext/opcache/jit/dynasm/dasm_x86.h diff --git a/ext/opcache/dynasm/dasm_x86.lua b/ext/opcache/jit/dynasm/dasm_x86.lua similarity index 100% rename from ext/opcache/dynasm/dasm_x86.lua rename to ext/opcache/jit/dynasm/dasm_x86.lua diff --git a/ext/opcache/dynasm/dynasm.lua b/ext/opcache/jit/dynasm/dynasm.lua similarity index 100% rename from ext/opcache/dynasm/dynasm.lua rename to ext/opcache/jit/dynasm/dynasm.lua diff --git a/ext/opcache/dynasm/minilua.c b/ext/opcache/jit/dynasm/minilua.c similarity index 100% rename from ext/opcache/dynasm/minilua.c rename to ext/opcache/jit/dynasm/minilua.c diff --git a/ext/opcache/jit/libudis86/LICENSE b/ext/opcache/jit/libudis86/LICENSE new file mode 100644 index 0000000000000..580f35987f661 --- /dev/null +++ b/ext/opcache/jit/libudis86/LICENSE @@ -0,0 +1,22 @@ +Copyright (c) 2002-2012, Vivek Thampi +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/ext/opcache/jit/libudis86/Makefile.am b/ext/opcache/jit/libudis86/Makefile.am new file mode 100644 index 0000000000000..c61cf9d04ff0d --- /dev/null +++ b/ext/opcache/jit/libudis86/Makefile.am @@ -0,0 +1,51 @@ +# +# -- udis86/libudis86 +# + +PYTHON = @PYTHON@ +OPTABLE = @top_srcdir@/docs/x86/optable.xml + +MAINTAINERCLEANFILES = Makefile.in + +lib_LTLIBRARIES = libudis86.la + +libudis86_la_SOURCES = \ + itab.c \ + decode.c \ + syn.c \ + syn-intel.c \ + syn-att.c \ + udis86.c \ + udint.h \ + syn.h \ + decode.h + +include_ladir = ${includedir}/libudis86 +include_la_HEADERS = \ + types.h \ + extern.h \ + itab.h + + +BUILT_SOURCES = \ + itab.c \ + itab.h + +# +# DLLs may not contain undefined symbol references. +# We have the linker check this explicitly. +# +if TARGET_WINDOWS +libudis86_la_LDFLAGS = -no-undefined -version-info 0:0:0 +endif + +itab.c itab.h: $(OPTABLE) \ + $(top_srcdir)/scripts/ud_itab.py \ + $(top_srcdir)/scripts/ud_opcode.py + $(PYTHON) $(top_srcdir)/scripts/ud_itab.py $(OPTABLE) $(srcdir) + + +clean-local: + rm -rf $(BUILT_SOURCES) + +maintainer-clean-local: diff --git a/ext/opcache/jit/libudis86/decode.c b/ext/opcache/jit/libudis86/decode.c new file mode 100644 index 0000000000000..036b9ed01291d --- /dev/null +++ b/ext/opcache/jit/libudis86/decode.c @@ -0,0 +1,1266 @@ +/* udis86 - libudis86/decode.c + * + * Copyright (c) 2002-2009 Vivek Thampi + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#include "udint.h" +#include "types.h" +#include "extern.h" +#include "decode.h" + +#ifndef __UD_STANDALONE__ +# include +#endif /* __UD_STANDALONE__ */ + +/* The max number of prefixes to an instruction */ +#define MAX_PREFIXES 15 + +/* rex prefix bits */ +#define REX_W(r) ( ( 0xF & ( r ) ) >> 3 ) +#define REX_R(r) ( ( 0x7 & ( r ) ) >> 2 ) +#define REX_X(r) ( ( 0x3 & ( r ) ) >> 1 ) +#define REX_B(r) ( ( 0x1 & ( r ) ) >> 0 ) +#define REX_PFX_MASK(n) ( ( P_REXW(n) << 3 ) | \ + ( P_REXR(n) << 2 ) | \ + ( P_REXX(n) << 1 ) | \ + ( P_REXB(n) << 0 ) ) + +/* scable-index-base bits */ +#define SIB_S(b) ( ( b ) >> 6 ) +#define SIB_I(b) ( ( ( b ) >> 3 ) & 7 ) +#define SIB_B(b) ( ( b ) & 7 ) + +/* modrm bits */ +#define MODRM_REG(b) ( ( ( b ) >> 3 ) & 7 ) +#define MODRM_NNN(b) ( ( ( b ) >> 3 ) & 7 ) +#define MODRM_MOD(b) ( ( ( b ) >> 6 ) & 3 ) +#define MODRM_RM(b) ( ( b ) & 7 ) + +static int decode_ext(struct ud *u, uint16_t ptr); +static int decode_opcode(struct ud *u); + +enum reg_class { /* register classes */ + REGCLASS_GPR, + REGCLASS_MMX, + REGCLASS_CR, + REGCLASS_DB, + REGCLASS_SEG, + REGCLASS_XMM +}; + + /* + * inp_start + * Should be called before each de-code operation. + */ +static void +inp_start(struct ud *u) +{ + u->inp_ctr = 0; +} + +static uint8_t +inp_peek(struct ud *u) +{ + if (u->inp_end == 0) { + if (u->inp_buf != NULL) { + if (u->inp_buf_index < u->inp_buf_size) { + return u->inp_buf[u->inp_buf_index]; + } + } else if (u->inp_peek != UD_EOI) { + return u->inp_peek; + } else { + int c; + if ((c = u->inp_hook(u)) != UD_EOI) { + u->inp_peek = c; + return u->inp_peek; + } + } + } + u->inp_end = 1; + UDERR(u, "byte expected, eoi received\n"); + return 0; +} + +static uint8_t +inp_next(struct ud *u) +{ + if (u->inp_end == 0) { + if (u->inp_buf != NULL) { + if (u->inp_buf_index < u->inp_buf_size) { + u->inp_ctr++; + return (u->inp_curr = u->inp_buf[u->inp_buf_index++]); + } + } else { + int c = u->inp_peek; + if (c != UD_EOI || (c = u->inp_hook(u)) != UD_EOI) { + u->inp_peek = UD_EOI; + u->inp_curr = c; + u->inp_sess[u->inp_ctr++] = u->inp_curr; + return u->inp_curr; + } + } + } + u->inp_end = 1; + UDERR(u, "byte expected, eoi received\n"); + return 0; +} + +static uint8_t +inp_curr(struct ud *u) +{ + return u->inp_curr; +} + + +/* + * inp_uint8 + * int_uint16 + * int_uint32 + * int_uint64 + * Load little-endian values from input + */ +static uint8_t +inp_uint8(struct ud* u) +{ + return inp_next(u); +} + +static uint16_t +inp_uint16(struct ud* u) +{ + uint16_t r, ret; + + ret = inp_next(u); + r = inp_next(u); + return ret | (r << 8); +} + +static uint32_t +inp_uint32(struct ud* u) +{ + uint32_t r, ret; + + ret = inp_next(u); + r = inp_next(u); + ret = ret | (r << 8); + r = inp_next(u); + ret = ret | (r << 16); + r = inp_next(u); + return ret | (r << 24); +} + +static uint64_t +inp_uint64(struct ud* u) +{ + uint64_t r, ret; + + ret = inp_next(u); + r = inp_next(u); + ret = ret | (r << 8); + r = inp_next(u); + ret = ret | (r << 16); + r = inp_next(u); + ret = ret | (r << 24); + r = inp_next(u); + ret = ret | (r << 32); + r = inp_next(u); + ret = ret | (r << 40); + r = inp_next(u); + ret = ret | (r << 48); + r = inp_next(u); + return ret | (r << 56); +} + + +static UD_INLINE int +eff_opr_mode(int dis_mode, int rex_w, int pfx_opr) +{ + if (dis_mode == 64) { + return rex_w ? 64 : (pfx_opr ? 16 : 32); + } else if (dis_mode == 32) { + return pfx_opr ? 16 : 32; + } else { + UD_ASSERT(dis_mode == 16); + return pfx_opr ? 32 : 16; + } +} + + +static UD_INLINE int +eff_adr_mode(int dis_mode, int pfx_adr) +{ + if (dis_mode == 64) { + return pfx_adr ? 32 : 64; + } else if (dis_mode == 32) { + return pfx_adr ? 16 : 32; + } else { + UD_ASSERT(dis_mode == 16); + return pfx_adr ? 32 : 16; + } +} + + +/* + * decode_prefixes + * + * Extracts instruction prefixes. + */ +static int +decode_prefixes(struct ud *u) +{ + int done = 0; + uint8_t curr = 0, last = 0; + UD_RETURN_ON_ERROR(u); + + do { + last = curr; + curr = inp_next(u); + UD_RETURN_ON_ERROR(u); + if (u->inp_ctr == MAX_INSN_LENGTH) { + UD_RETURN_WITH_ERROR(u, "max instruction length"); + } + + switch (curr) + { + case 0x2E: + u->pfx_seg = UD_R_CS; + break; + case 0x36: + u->pfx_seg = UD_R_SS; + break; + case 0x3E: + u->pfx_seg = UD_R_DS; + break; + case 0x26: + u->pfx_seg = UD_R_ES; + break; + case 0x64: + u->pfx_seg = UD_R_FS; + break; + case 0x65: + u->pfx_seg = UD_R_GS; + break; + case 0x67: /* adress-size override prefix */ + u->pfx_adr = 0x67; + break; + case 0xF0: + u->pfx_lock = 0xF0; + break; + case 0x66: + u->pfx_opr = 0x66; + break; + case 0xF2: + u->pfx_str = 0xf2; + break; + case 0xF3: + u->pfx_str = 0xf3; + break; + default: + /* consume if rex */ + done = (u->dis_mode == 64 && (curr & 0xF0) == 0x40) ? 0 : 1; + break; + } + } while (!done); + /* rex prefixes in 64bit mode, must be the last prefix */ + if (u->dis_mode == 64 && (last & 0xF0) == 0x40) { + u->pfx_rex = last; + } + return 0; +} + + +/* + * vex_l, vex_w + * Return the vex.L and vex.W bits + */ +static UD_INLINE uint8_t +vex_l(const struct ud *u) +{ + UD_ASSERT(u->vex_op != 0); + return ((u->vex_op == 0xc4 ? u->vex_b2 : u->vex_b1) >> 2) & 1; +} + +static UD_INLINE uint8_t +vex_w(const struct ud *u) +{ + UD_ASSERT(u->vex_op != 0); + return u->vex_op == 0xc4 ? ((u->vex_b2 >> 7) & 1) : 0; +} + + +static UD_INLINE uint8_t +modrm(struct ud * u) +{ + if ( !u->have_modrm ) { + u->modrm = inp_next( u ); + u->modrm_offset = (uint8_t) (u->inp_ctr - 1); + u->have_modrm = 1; + } + return u->modrm; +} + + +static unsigned int +resolve_operand_size(const struct ud* u, ud_operand_size_t osize) +{ + switch (osize) { + case SZ_V: + return u->opr_mode; + case SZ_Z: + return u->opr_mode == 16 ? 16 : 32; + case SZ_Y: + return u->opr_mode == 16 ? 32 : u->opr_mode; + case SZ_RDQ: + return u->dis_mode == 64 ? 64 : 32; + case SZ_X: + UD_ASSERT(u->vex_op != 0); + return (P_VEXL(u->itab_entry->prefix) && vex_l(u)) ? SZ_QQ : SZ_DQ; + default: + return osize; + } +} + + +static int resolve_mnemonic( struct ud* u ) +{ + /* resolve 3dnow weirdness. */ + if ( u->mnemonic == UD_I3dnow ) { + u->mnemonic = ud_itab[ u->le->table[ inp_curr( u ) ] ].mnemonic; + } + /* SWAPGS is only valid in 64bits mode */ + if ( u->mnemonic == UD_Iswapgs && u->dis_mode != 64 ) { + UDERR(u, "swapgs invalid in 64bits mode\n"); + return -1; + } + + if (u->mnemonic == UD_Ixchg) { + if ((u->operand[0].type == UD_OP_REG && u->operand[0].base == UD_R_AX && + u->operand[1].type == UD_OP_REG && u->operand[1].base == UD_R_AX) || + (u->operand[0].type == UD_OP_REG && u->operand[0].base == UD_R_EAX && + u->operand[1].type == UD_OP_REG && u->operand[1].base == UD_R_EAX)) { + u->operand[0].type = UD_NONE; + u->operand[1].type = UD_NONE; + u->mnemonic = UD_Inop; + } + } + + if (u->mnemonic == UD_Inop && u->pfx_repe) { + u->pfx_repe = 0; + u->mnemonic = UD_Ipause; + } + return 0; +} + + +/* ----------------------------------------------------------------------------- + * decode_a()- Decodes operands of the type seg:offset + * ----------------------------------------------------------------------------- + */ +static void +decode_a(struct ud* u, struct ud_operand *op) +{ + if (u->opr_mode == 16) { + /* seg16:off16 */ + op->type = UD_OP_PTR; + op->size = 32; + op->lval.ptr.off = inp_uint16(u); + op->lval.ptr.seg = inp_uint16(u); + } else { + /* seg16:off32 */ + op->type = UD_OP_PTR; + op->size = 48; + op->lval.ptr.off = inp_uint32(u); + op->lval.ptr.seg = inp_uint16(u); + } +} + +/* ----------------------------------------------------------------------------- + * decode_gpr() - Returns decoded General Purpose Register + * ----------------------------------------------------------------------------- + */ +static enum ud_type +decode_gpr(register struct ud* u, unsigned int s, unsigned char rm) +{ + switch (s) { + case 64: + return UD_R_RAX + rm; + case 32: + return UD_R_EAX + rm; + case 16: + return UD_R_AX + rm; + case 8: + if (u->dis_mode == 64 && u->pfx_rex) { + if (rm >= 4) + return UD_R_SPL + (rm-4); + return UD_R_AL + rm; + } else return UD_R_AL + rm; + case 0: + /* invalid size in case of a decode error */ + UD_ASSERT(u->error); + return UD_NONE; + default: + UD_ASSERT(!"invalid operand size"); + return UD_NONE; + } +} + +static void +decode_reg(struct ud *u, + struct ud_operand *opr, + int type, + int num, + int size) +{ + int reg; + size = resolve_operand_size(u, size); + switch (type) { + case REGCLASS_GPR : reg = decode_gpr(u, size, num); break; + case REGCLASS_MMX : reg = UD_R_MM0 + (num & 7); break; + case REGCLASS_XMM : + reg = num + (size == SZ_QQ ? UD_R_YMM0 : UD_R_XMM0); + break; + case REGCLASS_CR : reg = UD_R_CR0 + num; break; + case REGCLASS_DB : reg = UD_R_DR0 + num; break; + case REGCLASS_SEG : { + /* + * Only 6 segment registers, anything else is an error. + */ + if ((num & 7) > 5) { + UDERR(u, "invalid segment register value\n"); + return; + } else { + reg = UD_R_ES + (num & 7); + } + break; + } + default: + UD_ASSERT(!"invalid register type"); + return; + } + opr->type = UD_OP_REG; + opr->base = reg; + opr->size = size; +} + + +/* + * decode_imm + * + * Decode Immediate values. + */ +static void +decode_imm(struct ud* u, unsigned int size, struct ud_operand *op) +{ + op->size = resolve_operand_size(u, size); + op->type = UD_OP_IMM; + + switch (op->size) { + case 8: op->lval.sbyte = inp_uint8(u); break; + case 16: op->lval.uword = inp_uint16(u); break; + case 32: op->lval.udword = inp_uint32(u); break; + case 64: op->lval.uqword = inp_uint64(u); break; + default: return; + } +} + + +/* + * decode_mem_disp + * + * Decode mem address displacement. + */ +static void +decode_mem_disp(struct ud* u, unsigned int size, struct ud_operand *op) +{ + switch (size) { + case 8: + op->offset = 8; + op->lval.ubyte = inp_uint8(u); + break; + case 16: + op->offset = 16; + op->lval.uword = inp_uint16(u); + break; + case 32: + op->offset = 32; + op->lval.udword = inp_uint32(u); + break; + case 64: + op->offset = 64; + op->lval.uqword = inp_uint64(u); + break; + default: + return; + } +} + + +/* + * decode_modrm_reg + * + * Decodes reg field of mod/rm byte + * + */ +static UD_INLINE void +decode_modrm_reg(struct ud *u, + struct ud_operand *operand, + unsigned int type, + unsigned int size) +{ + uint8_t reg = (REX_R(u->_rex) << 3) | MODRM_REG(modrm(u)); + decode_reg(u, operand, type, reg, size); +} + + +/* + * decode_modrm_rm + * + * Decodes rm field of mod/rm byte + * + */ +static void +decode_modrm_rm(struct ud *u, + struct ud_operand *op, + unsigned char type, /* register type */ + unsigned int size) /* operand size */ + +{ + size_t offset = 0; + unsigned char mod, rm; + + /* get mod, r/m and reg fields */ + mod = MODRM_MOD(modrm(u)); + rm = (REX_B(u->_rex) << 3) | MODRM_RM(modrm(u)); + + /* + * If mod is 11b, then the modrm.rm specifies a register. + * + */ + if (mod == 3) { + decode_reg(u, op, type, rm, size); + return; + } + + /* + * !11b => Memory Address + */ + op->type = UD_OP_MEM; + op->size = resolve_operand_size(u, size); + + if (u->adr_mode == 64) { + op->base = UD_R_RAX + rm; + if (mod == 1) { + offset = 8; + } else if (mod == 2) { + offset = 32; + } else if (mod == 0 && (rm & 7) == 5) { + op->base = UD_R_RIP; + offset = 32; + } else { + offset = 0; + } + /* + * Scale-Index-Base (SIB) + */ + if ((rm & 7) == 4) { + inp_next(u); + + op->base = UD_R_RAX + (SIB_B(inp_curr(u)) | (REX_B(u->_rex) << 3)); + op->index = UD_R_RAX + (SIB_I(inp_curr(u)) | (REX_X(u->_rex) << 3)); + /* special conditions for base reference */ + if (op->index == UD_R_RSP) { + op->index = UD_NONE; + op->scale = UD_NONE; + } else { + op->scale = (1 << SIB_S(inp_curr(u))) & ~1; + } + + if (op->base == UD_R_RBP || op->base == UD_R_R13) { + if (mod == 0) { + op->base = UD_NONE; + } + if (mod == 1) { + offset = 8; + } else { + offset = 32; + } + } + } else { + op->scale = UD_NONE; + op->index = UD_NONE; + } + } else if (u->adr_mode == 32) { + op->base = UD_R_EAX + rm; + if (mod == 1) { + offset = 8; + } else if (mod == 2) { + offset = 32; + } else if (mod == 0 && rm == 5) { + op->base = UD_NONE; + offset = 32; + } else { + offset = 0; + } + + /* Scale-Index-Base (SIB) */ + if ((rm & 7) == 4) { + inp_next(u); + + op->scale = (1 << SIB_S(inp_curr(u))) & ~1; + op->index = UD_R_EAX + (SIB_I(inp_curr(u)) | (REX_X(u->pfx_rex) << 3)); + op->base = UD_R_EAX + (SIB_B(inp_curr(u)) | (REX_B(u->pfx_rex) << 3)); + + if (op->index == UD_R_ESP) { + op->index = UD_NONE; + op->scale = UD_NONE; + } + + /* special condition for base reference */ + if (op->base == UD_R_EBP) { + if (mod == 0) { + op->base = UD_NONE; + } + if (mod == 1) { + offset = 8; + } else { + offset = 32; + } + } + } else { + op->scale = UD_NONE; + op->index = UD_NONE; + } + } else { + const unsigned int bases[] = { UD_R_BX, UD_R_BX, UD_R_BP, UD_R_BP, + UD_R_SI, UD_R_DI, UD_R_BP, UD_R_BX }; + const unsigned int indices[] = { UD_R_SI, UD_R_DI, UD_R_SI, UD_R_DI, + UD_NONE, UD_NONE, UD_NONE, UD_NONE }; + op->base = bases[rm & 7]; + op->index = indices[rm & 7]; + op->scale = UD_NONE; + if (mod == 0 && rm == 6) { + offset = 16; + op->base = UD_NONE; + } else if (mod == 1) { + offset = 8; + } else if (mod == 2) { + offset = 16; + } + } + + if (offset) { + decode_mem_disp(u, offset, op); + } else { + op->offset = 0; + } +} + + +/* + * decode_moffset + * Decode offset-only memory operand + */ +static void +decode_moffset(struct ud *u, unsigned int size, struct ud_operand *opr) +{ + opr->type = UD_OP_MEM; + opr->base = UD_NONE; + opr->index = UD_NONE; + opr->scale = UD_NONE; + opr->size = resolve_operand_size(u, size); + decode_mem_disp(u, u->adr_mode, opr); +} + + +static void +decode_vex_vvvv(struct ud *u, struct ud_operand *opr, unsigned size) +{ + uint8_t vvvv; + UD_ASSERT(u->vex_op != 0); + vvvv = ((u->vex_op == 0xc4 ? u->vex_b2 : u->vex_b1) >> 3) & 0xf; + decode_reg(u, opr, REGCLASS_XMM, (0xf & ~vvvv), size); +} + + +/* + * decode_vex_immreg + * Decode source operand encoded in immediate byte [7:4] + */ +static int +decode_vex_immreg(struct ud *u, struct ud_operand *opr, unsigned size) +{ + uint8_t imm = inp_next(u); + uint8_t mask = u->dis_mode == 64 ? 0xf : 0x7; + UD_RETURN_ON_ERROR(u); + UD_ASSERT(u->vex_op != 0); + decode_reg(u, opr, REGCLASS_XMM, mask & (imm >> 4), size); + return 0; +} + + +/* + * decode_operand + * + * Decodes a single operand. + * Returns the type of the operand (UD_NONE if none) + */ +static int +decode_operand(struct ud *u, + struct ud_operand *operand, + enum ud_operand_code type, + unsigned int size) +{ + operand->type = UD_NONE; + operand->_oprcode = type; + + switch (type) { + case OP_A : + decode_a(u, operand); + break; + case OP_MR: + decode_modrm_rm(u, operand, REGCLASS_GPR, + MODRM_MOD(modrm(u)) == 3 ? + Mx_reg_size(size) : Mx_mem_size(size)); + break; + case OP_F: + u->br_far = 1; + /* intended fall through */ + case OP_M: + if (MODRM_MOD(modrm(u)) == 3) { + UDERR(u, "expected modrm.mod != 3\n"); + } + /* intended fall through */ + case OP_E: + decode_modrm_rm(u, operand, REGCLASS_GPR, size); + break; + case OP_G: + decode_modrm_reg(u, operand, REGCLASS_GPR, size); + break; + case OP_sI: + case OP_I: + decode_imm(u, size, operand); + break; + case OP_I1: + operand->type = UD_OP_CONST; + operand->lval.udword = 1; + break; + case OP_N: + if (MODRM_MOD(modrm(u)) != 3) { + UDERR(u, "expected modrm.mod == 3\n"); + } + /* intended fall through */ + case OP_Q: + decode_modrm_rm(u, operand, REGCLASS_MMX, size); + break; + case OP_P: + decode_modrm_reg(u, operand, REGCLASS_MMX, size); + break; + case OP_U: + if (MODRM_MOD(modrm(u)) != 3) { + UDERR(u, "expected modrm.mod == 3\n"); + } + /* intended fall through */ + case OP_W: + decode_modrm_rm(u, operand, REGCLASS_XMM, size); + break; + case OP_V: + decode_modrm_reg(u, operand, REGCLASS_XMM, size); + break; + case OP_H: + decode_vex_vvvv(u, operand, size); + break; + case OP_MU: + decode_modrm_rm(u, operand, REGCLASS_XMM, + MODRM_MOD(modrm(u)) == 3 ? + Mx_reg_size(size) : Mx_mem_size(size)); + break; + case OP_S: + decode_modrm_reg(u, operand, REGCLASS_SEG, size); + break; + case OP_O: + decode_moffset(u, size, operand); + break; + case OP_R0: + case OP_R1: + case OP_R2: + case OP_R3: + case OP_R4: + case OP_R5: + case OP_R6: + case OP_R7: + decode_reg(u, operand, REGCLASS_GPR, + (REX_B(u->_rex) << 3) | (type - OP_R0), size); + break; + case OP_AL: + case OP_AX: + case OP_eAX: + case OP_rAX: + decode_reg(u, operand, REGCLASS_GPR, 0, size); + break; + case OP_CL: + case OP_CX: + case OP_eCX: + decode_reg(u, operand, REGCLASS_GPR, 1, size); + break; + case OP_DL: + case OP_DX: + case OP_eDX: + decode_reg(u, operand, REGCLASS_GPR, 2, size); + break; + case OP_ES: + case OP_CS: + case OP_DS: + case OP_SS: + case OP_FS: + case OP_GS: + /* in 64bits mode, only fs and gs are allowed */ + if (u->dis_mode == 64) { + if (type != OP_FS && type != OP_GS) { + UDERR(u, "invalid segment register in 64bits\n"); + } + } + operand->type = UD_OP_REG; + operand->base = (type - OP_ES) + UD_R_ES; + operand->size = 16; + break; + case OP_J : + decode_imm(u, size, operand); + operand->type = UD_OP_JIMM; + break ; + case OP_R : + if (MODRM_MOD(modrm(u)) != 3) { + UDERR(u, "expected modrm.mod == 3\n"); + } + decode_modrm_rm(u, operand, REGCLASS_GPR, size); + break; + case OP_C: + decode_modrm_reg(u, operand, REGCLASS_CR, size); + break; + case OP_D: + decode_modrm_reg(u, operand, REGCLASS_DB, size); + break; + case OP_I3 : + operand->type = UD_OP_CONST; + operand->lval.sbyte = 3; + break; + case OP_ST0: + case OP_ST1: + case OP_ST2: + case OP_ST3: + case OP_ST4: + case OP_ST5: + case OP_ST6: + case OP_ST7: + operand->type = UD_OP_REG; + operand->base = (type - OP_ST0) + UD_R_ST0; + operand->size = 80; + break; + case OP_L: + decode_vex_immreg(u, operand, size); + break; + default : + operand->type = UD_NONE; + break; + } + return operand->type; +} + + +/* + * decode_operands + * + * Disassemble upto 3 operands of the current instruction being + * disassembled. By the end of the function, the operand fields + * of the ud structure will have been filled. + */ +static int +decode_operands(struct ud* u) +{ + decode_operand(u, &u->operand[0], + u->itab_entry->operand1.type, + u->itab_entry->operand1.size); + if (u->operand[0].type != UD_NONE) { + decode_operand(u, &u->operand[1], + u->itab_entry->operand2.type, + u->itab_entry->operand2.size); + } + if (u->operand[1].type != UD_NONE) { + decode_operand(u, &u->operand[2], + u->itab_entry->operand3.type, + u->itab_entry->operand3.size); + } + if (u->operand[2].type != UD_NONE) { + decode_operand(u, &u->operand[3], + u->itab_entry->operand4.type, + u->itab_entry->operand4.size); + } + return 0; +} + +/* ----------------------------------------------------------------------------- + * clear_insn() - clear instruction structure + * ----------------------------------------------------------------------------- + */ +static void +clear_insn(register struct ud* u) +{ + u->error = 0; + u->pfx_seg = 0; + u->pfx_opr = 0; + u->pfx_adr = 0; + u->pfx_lock = 0; + u->pfx_repne = 0; + u->pfx_rep = 0; + u->pfx_repe = 0; + u->pfx_rex = 0; + u->pfx_str = 0; + u->mnemonic = UD_Inone; + u->itab_entry = NULL; + u->have_modrm = 0; + u->br_far = 0; + u->vex_op = 0; + u->_rex = 0; + u->operand[0].type = UD_NONE; + u->operand[1].type = UD_NONE; + u->operand[2].type = UD_NONE; + u->operand[3].type = UD_NONE; +} + + +static UD_INLINE int +resolve_pfx_str(struct ud* u) +{ + if (u->pfx_str == 0xf3) { + if (P_STR(u->itab_entry->prefix)) { + u->pfx_rep = 0xf3; + } else { + u->pfx_repe = 0xf3; + } + } else if (u->pfx_str == 0xf2) { + u->pfx_repne = 0xf3; + } + return 0; +} + + +static int +resolve_mode( struct ud* u ) +{ + int default64; + /* if in error state, bail out */ + if ( u->error ) return -1; + + /* propagate prefix effects */ + if ( u->dis_mode == 64 ) { /* set 64bit-mode flags */ + + /* Check validity of instruction m64 */ + if ( P_INV64( u->itab_entry->prefix ) ) { + UDERR(u, "instruction invalid in 64bits\n"); + return -1; + } + + /* compute effective rex based on, + * - vex prefix (if any) + * - rex prefix (if any, and not vex) + * - allowed prefixes specified by the opcode map + */ + if (u->vex_op == 0xc4) { + /* vex has rex.rxb in 1's complement */ + u->_rex = ((~(u->vex_b1 >> 5) & 0x7) /* rex.0rxb */ | + ((u->vex_b2 >> 4) & 0x8) /* rex.w000 */); + } else if (u->vex_op == 0xc5) { + /* vex has rex.r in 1's complement */ + u->_rex = (~(u->vex_b1 >> 5)) & 4; + } else { + UD_ASSERT(u->vex_op == 0); + u->_rex = u->pfx_rex; + } + u->_rex &= REX_PFX_MASK(u->itab_entry->prefix); + + /* whether this instruction has a default operand size of + * 64bit, also hardcoded into the opcode map. + */ + default64 = P_DEF64( u->itab_entry->prefix ); + /* calculate effective operand size */ + if (REX_W(u->_rex)) { + u->opr_mode = 64; + } else if ( u->pfx_opr ) { + u->opr_mode = 16; + } else { + /* unless the default opr size of instruction is 64, + * the effective operand size in the absence of rex.w + * prefix is 32. + */ + u->opr_mode = default64 ? 64 : 32; + } + + /* calculate effective address size */ + u->adr_mode = (u->pfx_adr) ? 32 : 64; + } else if ( u->dis_mode == 32 ) { /* set 32bit-mode flags */ + u->opr_mode = ( u->pfx_opr ) ? 16 : 32; + u->adr_mode = ( u->pfx_adr ) ? 16 : 32; + } else if ( u->dis_mode == 16 ) { /* set 16bit-mode flags */ + u->opr_mode = ( u->pfx_opr ) ? 32 : 16; + u->adr_mode = ( u->pfx_adr ) ? 32 : 16; + } + + return 0; +} + + +static UD_INLINE int +decode_insn(struct ud *u, uint16_t ptr) +{ + UD_ASSERT((ptr & 0x8000) == 0); + u->itab_entry = &ud_itab[ ptr ]; + u->mnemonic = u->itab_entry->mnemonic; + return (resolve_pfx_str(u) == 0 && + resolve_mode(u) == 0 && + decode_operands(u) == 0 && + resolve_mnemonic(u) == 0) ? 0 : -1; +} + + +/* + * decode_3dnow() + * + * Decoding 3dnow is a little tricky because of its strange opcode + * structure. The final opcode disambiguation depends on the last + * byte that comes after the operands have been decoded. Fortunately, + * all 3dnow instructions have the same set of operand types. So we + * go ahead and decode the instruction by picking an arbitrarily chosen + * valid entry in the table, decode the operands, and read the final + * byte to resolve the menmonic. + */ +static UD_INLINE int +decode_3dnow(struct ud* u) +{ + uint16_t ptr; + UD_ASSERT(u->le->type == UD_TAB__OPC_3DNOW); + UD_ASSERT(u->le->table[0xc] != 0); + decode_insn(u, u->le->table[0xc]); + inp_next(u); + if (u->error) { + return -1; + } + ptr = u->le->table[inp_curr(u)]; + UD_ASSERT((ptr & 0x8000) == 0); + u->mnemonic = ud_itab[ptr].mnemonic; + return 0; +} + + +static int +decode_ssepfx(struct ud *u) +{ + uint8_t idx; + uint8_t pfx; + + /* + * String prefixes (f2, f3) take precedence over operand + * size prefix (66). + */ + pfx = u->pfx_str; + if (pfx == 0) { + pfx = u->pfx_opr; + } + idx = ((pfx & 0xf) + 1) / 2; + if (u->le->table[idx] == 0) { + idx = 0; + } + if (idx && u->le->table[idx] != 0) { + /* + * "Consume" the prefix as a part of the opcode, so it is no + * longer exported as an instruction prefix. + */ + u->pfx_str = 0; + if (pfx == 0x66) { + /* + * consume "66" only if it was used for decoding, leaving + * it to be used as an operands size override for some + * simd instructions. + */ + u->pfx_opr = 0; + } + } + return decode_ext(u, u->le->table[idx]); +} + + +static int +decode_vex(struct ud *u) +{ + uint8_t index; + if (u->dis_mode != 64 && MODRM_MOD(inp_peek(u)) != 0x3) { + index = 0; + } else { + u->vex_op = inp_curr(u); + u->vex_b1 = inp_next(u); + if (u->vex_op == 0xc4) { + uint8_t pp, m; + /* 3-byte vex */ + u->vex_b2 = inp_next(u); + UD_RETURN_ON_ERROR(u); + m = u->vex_b1 & 0x1f; + if (m == 0 || m > 3) { + UD_RETURN_WITH_ERROR(u, "reserved vex.m-mmmm value"); + } + pp = u->vex_b2 & 0x3; + index = (pp << 2) | m; + } else { + /* 2-byte vex */ + UD_ASSERT(u->vex_op == 0xc5); + index = 0x1 | ((u->vex_b1 & 0x3) << 2); + } + } + return decode_ext(u, u->le->table[index]); +} + + +/* + * decode_ext() + * + * Decode opcode extensions (if any) + */ +static int +decode_ext(struct ud *u, uint16_t ptr) +{ + uint8_t idx = 0; + if ((ptr & 0x8000) == 0) { + return decode_insn(u, ptr); + } + u->le = &ud_lookup_table_list[(~0x8000 & ptr)]; + if (u->le->type == UD_TAB__OPC_3DNOW) { + return decode_3dnow(u); + } + + switch (u->le->type) { + case UD_TAB__OPC_MOD: + /* !11 = 0, 11 = 1 */ + idx = (MODRM_MOD(modrm(u)) + 1) / 4; + break; + /* disassembly mode/operand size/address size based tables. + * 16 = 0,, 32 = 1, 64 = 2 + */ + case UD_TAB__OPC_MODE: + idx = u->dis_mode != 64 ? 0 : 1; + break; + case UD_TAB__OPC_OSIZE: + idx = eff_opr_mode(u->dis_mode, REX_W(u->pfx_rex), u->pfx_opr) / 32; + break; + case UD_TAB__OPC_ASIZE: + idx = eff_adr_mode(u->dis_mode, u->pfx_adr) / 32; + break; + case UD_TAB__OPC_X87: + idx = modrm(u) - 0xC0; + break; + case UD_TAB__OPC_VENDOR: + if (u->vendor == UD_VENDOR_ANY) { + /* choose a valid entry */ + idx = (u->le->table[idx] != 0) ? 0 : 1; + } else if (u->vendor == UD_VENDOR_AMD) { + idx = 0; + } else { + idx = 1; + } + break; + case UD_TAB__OPC_RM: + idx = MODRM_RM(modrm(u)); + break; + case UD_TAB__OPC_REG: + idx = MODRM_REG(modrm(u)); + break; + case UD_TAB__OPC_SSE: + return decode_ssepfx(u); + case UD_TAB__OPC_VEX: + return decode_vex(u); + case UD_TAB__OPC_VEX_W: + idx = vex_w(u); + break; + case UD_TAB__OPC_VEX_L: + idx = vex_l(u); + break; + case UD_TAB__OPC_TABLE: + inp_next(u); + return decode_opcode(u); + default: + UD_ASSERT(!"not reached"); + break; + } + + return decode_ext(u, u->le->table[idx]); +} + + +static int +decode_opcode(struct ud *u) +{ + uint16_t ptr; + UD_ASSERT(u->le->type == UD_TAB__OPC_TABLE); + UD_RETURN_ON_ERROR(u); + ptr = u->le->table[inp_curr(u)]; + return decode_ext(u, ptr); +} + + +/* ============================================================================= + * ud_decode() - Instruction decoder. Returns the number of bytes decoded. + * ============================================================================= + */ +unsigned int +ud_decode(struct ud *u) +{ + inp_start(u); + clear_insn(u); + u->le = &ud_lookup_table_list[0]; + u->error = decode_prefixes(u) == -1 || + decode_opcode(u) == -1 || + u->error; + /* Handle decode error. */ + if (u->error) { + /* clear out the decode data. */ + clear_insn(u); + /* mark the sequence of bytes as invalid. */ + u->itab_entry = &ud_itab[0]; /* entry 0 is invalid */ + u->mnemonic = u->itab_entry->mnemonic; + } + + /* maybe this stray segment override byte + * should be spewed out? + */ + if ( !P_SEG( u->itab_entry->prefix ) && + u->operand[0].type != UD_OP_MEM && + u->operand[1].type != UD_OP_MEM ) + u->pfx_seg = 0; + + u->insn_offset = u->pc; /* set offset of instruction */ + u->asm_buf_fill = 0; /* set translation buffer index to 0 */ + u->pc += u->inp_ctr; /* move program counter by bytes decoded */ + + /* return number of bytes disassembled. */ + return u->inp_ctr; +} + +/* +vim: set ts=2 sw=2 expandtab +*/ diff --git a/ext/opcache/jit/libudis86/decode.h b/ext/opcache/jit/libudis86/decode.h new file mode 100644 index 0000000000000..3949c4e269bff --- /dev/null +++ b/ext/opcache/jit/libudis86/decode.h @@ -0,0 +1,197 @@ +/* udis86 - libudis86/decode.h + * + * Copyright (c) 2002-2009 Vivek Thampi + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef UD_DECODE_H +#define UD_DECODE_H + +#include "types.h" +#include "udint.h" +#include "itab.h" + +#define MAX_INSN_LENGTH 15 + +/* itab prefix bits */ +#define P_none ( 0 ) + +#define P_inv64 ( 1 << 0 ) +#define P_INV64(n) ( ( n >> 0 ) & 1 ) +#define P_def64 ( 1 << 1 ) +#define P_DEF64(n) ( ( n >> 1 ) & 1 ) + +#define P_oso ( 1 << 2 ) +#define P_OSO(n) ( ( n >> 2 ) & 1 ) +#define P_aso ( 1 << 3 ) +#define P_ASO(n) ( ( n >> 3 ) & 1 ) + +#define P_rexb ( 1 << 4 ) +#define P_REXB(n) ( ( n >> 4 ) & 1 ) +#define P_rexw ( 1 << 5 ) +#define P_REXW(n) ( ( n >> 5 ) & 1 ) +#define P_rexr ( 1 << 6 ) +#define P_REXR(n) ( ( n >> 6 ) & 1 ) +#define P_rexx ( 1 << 7 ) +#define P_REXX(n) ( ( n >> 7 ) & 1 ) + +#define P_seg ( 1 << 8 ) +#define P_SEG(n) ( ( n >> 8 ) & 1 ) + +#define P_vexl ( 1 << 9 ) +#define P_VEXL(n) ( ( n >> 9 ) & 1 ) +#define P_vexw ( 1 << 10 ) +#define P_VEXW(n) ( ( n >> 10 ) & 1 ) + +#define P_str ( 1 << 11 ) +#define P_STR(n) ( ( n >> 11 ) & 1 ) +#define P_strz ( 1 << 12 ) +#define P_STR_ZF(n) ( ( n >> 12 ) & 1 ) + +/* operand type constants -- order is important! */ + +enum ud_operand_code { + OP_NONE, + + OP_A, OP_E, OP_M, OP_G, + OP_I, OP_F, + + OP_R0, OP_R1, OP_R2, OP_R3, + OP_R4, OP_R5, OP_R6, OP_R7, + + OP_AL, OP_CL, OP_DL, + OP_AX, OP_CX, OP_DX, + OP_eAX, OP_eCX, OP_eDX, + OP_rAX, OP_rCX, OP_rDX, + + OP_ES, OP_CS, OP_SS, OP_DS, + OP_FS, OP_GS, + + OP_ST0, OP_ST1, OP_ST2, OP_ST3, + OP_ST4, OP_ST5, OP_ST6, OP_ST7, + + OP_J, OP_S, OP_O, + OP_I1, OP_I3, OP_sI, + + OP_V, OP_W, OP_Q, OP_P, + OP_U, OP_N, OP_MU, OP_H, + OP_L, + + OP_R, OP_C, OP_D, + + OP_MR +} UD_ATTR_PACKED; + + +/* + * Operand size constants + * + * Symbolic constants for various operand sizes. Some of these constants + * are given a value equal to the width of the data (SZ_B == 8), such + * that they maybe used interchangeably in the internals. Modifying them + * will most certainly break things! + */ +typedef uint16_t ud_operand_size_t; + +#define SZ_NA 0 +#define SZ_Z 1 +#define SZ_V 2 +#define SZ_Y 3 +#define SZ_X 4 +#define SZ_RDQ 7 +#define SZ_B 8 +#define SZ_W 16 +#define SZ_D 32 +#define SZ_Q 64 +#define SZ_T 80 +#define SZ_O 12 +#define SZ_DQ 128 /* double quad */ +#define SZ_QQ 256 /* quad quad */ + +/* + * Complex size types; that encode sizes for operands of type MR (memory or + * register); for internal use only. Id space above 256. + */ +#define SZ_BD ((SZ_B << 8) | SZ_D) +#define SZ_BV ((SZ_B << 8) | SZ_V) +#define SZ_WD ((SZ_W << 8) | SZ_D) +#define SZ_WV ((SZ_W << 8) | SZ_V) +#define SZ_WY ((SZ_W << 8) | SZ_Y) +#define SZ_DY ((SZ_D << 8) | SZ_Y) +#define SZ_WO ((SZ_W << 8) | SZ_O) +#define SZ_DO ((SZ_D << 8) | SZ_O) +#define SZ_QO ((SZ_Q << 8) | SZ_O) + + +/* resolve complex size type. + */ +static UD_INLINE ud_operand_size_t +Mx_mem_size(ud_operand_size_t size) +{ + return (size >> 8) & 0xff; +} + +static UD_INLINE ud_operand_size_t +Mx_reg_size(ud_operand_size_t size) +{ + return size & 0xff; +} + +/* A single operand of an entry in the instruction table. + * (internal use only) + */ +struct ud_itab_entry_operand +{ + enum ud_operand_code type; + ud_operand_size_t size; +}; + + +/* A single entry in an instruction table. + *(internal use only) + */ +struct ud_itab_entry +{ + enum ud_mnemonic_code mnemonic; + struct ud_itab_entry_operand operand1; + struct ud_itab_entry_operand operand2; + struct ud_itab_entry_operand operand3; + struct ud_itab_entry_operand operand4; + uint32_t prefix; +}; + +struct ud_lookup_table_list_entry { + const uint16_t *table; + enum ud_table_type type; + const char *meta; +}; + +extern struct ud_itab_entry ud_itab[]; +extern struct ud_lookup_table_list_entry ud_lookup_table_list[]; + +#endif /* UD_DECODE_H */ + +/* vim:cindent + * vim:expandtab + * vim:ts=4 + * vim:sw=4 + */ diff --git a/ext/opcache/jit/libudis86/extern.h b/ext/opcache/jit/libudis86/extern.h new file mode 100644 index 0000000000000..71a01fd9b4d85 --- /dev/null +++ b/ext/opcache/jit/libudis86/extern.h @@ -0,0 +1,113 @@ +/* udis86 - libudis86/extern.h + * + * Copyright (c) 2002-2009, 2013 Vivek Thampi + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef UD_EXTERN_H +#define UD_EXTERN_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "types.h" + +#if defined(_MSC_VER) && defined(_USRDLL) +# ifdef LIBUDIS86_EXPORTS +# define LIBUDIS86_DLLEXTERN __declspec(dllexport) +# else +# define LIBUDIS86_DLLEXTERN __declspec(dllimport) +# endif +#else +# define LIBUDIS86_DLLEXTERN +#endif + +/* ============================= PUBLIC API ================================= */ + +extern LIBUDIS86_DLLEXTERN void ud_init(struct ud*); + +extern LIBUDIS86_DLLEXTERN void ud_set_mode(struct ud*, uint8_t); + +extern LIBUDIS86_DLLEXTERN void ud_set_pc(struct ud*, uint64_t); + +extern LIBUDIS86_DLLEXTERN void ud_set_input_hook(struct ud*, int (*)(struct ud*)); + +extern LIBUDIS86_DLLEXTERN void ud_set_input_buffer(struct ud*, const uint8_t*, size_t); + +#ifndef __UD_STANDALONE__ +extern LIBUDIS86_DLLEXTERN void ud_set_input_file(struct ud*, FILE*); +#endif /* __UD_STANDALONE__ */ + +extern LIBUDIS86_DLLEXTERN void ud_set_vendor(struct ud*, unsigned); + +extern LIBUDIS86_DLLEXTERN void ud_set_syntax(struct ud*, void (*)(struct ud*)); + +extern LIBUDIS86_DLLEXTERN void ud_input_skip(struct ud*, size_t); + +extern LIBUDIS86_DLLEXTERN int ud_input_end(const struct ud*); + +extern LIBUDIS86_DLLEXTERN unsigned int ud_decode(struct ud*); + +extern LIBUDIS86_DLLEXTERN unsigned int ud_disassemble(struct ud*); + +extern LIBUDIS86_DLLEXTERN void ud_translate_intel(struct ud*); + +extern LIBUDIS86_DLLEXTERN void ud_translate_att(struct ud*); + +extern LIBUDIS86_DLLEXTERN const char* ud_insn_asm(const struct ud* u); + +extern LIBUDIS86_DLLEXTERN const uint8_t* ud_insn_ptr(const struct ud* u); + +extern LIBUDIS86_DLLEXTERN uint64_t ud_insn_off(const struct ud*); + +extern LIBUDIS86_DLLEXTERN const char* ud_insn_hex(struct ud*); + +extern LIBUDIS86_DLLEXTERN unsigned int ud_insn_len(const struct ud* u); + +extern LIBUDIS86_DLLEXTERN const struct ud_operand* ud_insn_opr(const struct ud *u, unsigned int n); + +extern LIBUDIS86_DLLEXTERN int ud_opr_is_sreg(const struct ud_operand *opr); + +extern LIBUDIS86_DLLEXTERN int ud_opr_is_gpr(const struct ud_operand *opr); + +extern LIBUDIS86_DLLEXTERN enum ud_mnemonic_code ud_insn_mnemonic(const struct ud *u); + +extern LIBUDIS86_DLLEXTERN const char* ud_lookup_mnemonic(enum ud_mnemonic_code c); + +extern LIBUDIS86_DLLEXTERN void ud_set_user_opaque_data(struct ud*, void*); + +extern LIBUDIS86_DLLEXTERN void* ud_get_user_opaque_data(const struct ud*); + +extern LIBUDIS86_DLLEXTERN void ud_set_asm_buffer(struct ud *u, char *buf, size_t size); + +extern LIBUDIS86_DLLEXTERN void ud_set_sym_resolver(struct ud *u, + const char* (*resolver)(struct ud*, + uint64_t addr, + int64_t *offset)); + +/* ========================================================================== */ + +#ifdef __cplusplus +} +#endif +#endif /* UD_EXTERN_H */ diff --git a/ext/opcache/jit/libudis86/itab.c b/ext/opcache/jit/libudis86/itab.c new file mode 100644 index 0000000000000..953f3e5227875 --- /dev/null +++ b/ext/opcache/jit/libudis86/itab.c @@ -0,0 +1,5946 @@ +/* itab.c -- generated by udis86:scripts/ud_itab.py, do no edit */ +#include "decode.h" + +#define GROUP(n) (0x8000 | (n)) +#define INVALID 0 + + +const uint16_t ud_itab__0[] = { + /* 0 */ 15, 16, 17, 18, + /* 4 */ 19, 20, GROUP(1), GROUP(2), + /* 8 */ 964, 965, 966, 967, + /* c */ 968, 969, GROUP(3), GROUP(4), + /* 10 */ 5, 6, 7, 8, + /* 14 */ 9, 10, GROUP(284), GROUP(285), + /* 18 */ 1336, 1337, 1338, 1339, + /* 1c */ 1340, 1341, GROUP(286), GROUP(287), + /* 20 */ 49, 50, 51, 52, + /* 24 */ 53, 54, INVALID, GROUP(288), + /* 28 */ 1407, 1408, 1409, 1410, + /* 2c */ 1411, 1412, INVALID, GROUP(289), + /* 30 */ 1487, 1488, 1489, 1490, + /* 34 */ 1491, 1492, INVALID, GROUP(290), + /* 38 */ 100, 101, 102, 103, + /* 3c */ 104, 105, INVALID, GROUP(291), + /* 40 */ 699, 700, 701, 702, + /* 44 */ 703, 704, 705, 706, + /* 48 */ 175, 176, 177, 178, + /* 4c */ 179, 180, 181, 182, + /* 50 */ 1246, 1247, 1248, 1249, + /* 54 */ 1250, 1251, 1252, 1253, + /* 58 */ 1101, 1102, 1103, 1104, + /* 5c */ 1105, 1106, 1107, 1108, + /* 60 */ GROUP(292), GROUP(295), GROUP(298), GROUP(299), + /* 64 */ INVALID, INVALID, INVALID, INVALID, + /* 68 */ 1254, 697, 1256, 698, + /* 6c */ 709, GROUP(300), 982, GROUP(301), + /* 70 */ 726, 728, 730, 732, + /* 74 */ 734, 736, 738, 740, + /* 78 */ 742, 744, 746, 748, + /* 7c */ 750, 752, 754, 756, + /* 80 */ GROUP(302), GROUP(303), GROUP(304), GROUP(313), + /* 84 */ 1433, 1434, 1475, 1476, + /* 88 */ 828, 829, 830, 831, + /* 8c */ 832, 770, 833, GROUP(314), + /* 90 */ 1477, 1478, 1479, 1480, + /* 94 */ 1481, 1482, 1483, 1484, + /* 98 */ GROUP(315), GROUP(316), GROUP(317), 1470, + /* 9c */ GROUP(318), GROUP(322), 1310, 766, + /* a0 */ 834, 835, 836, 837, + /* a4 */ 922, GROUP(326), 114, GROUP(327), + /* a8 */ 1435, 1436, 1402, GROUP(328), + /* ac */ 790, GROUP(329), 1346, GROUP(330), + /* b0 */ 838, 839, 840, 841, + /* b4 */ 842, 843, 844, 845, + /* b8 */ 846, 847, 848, 849, + /* bc */ 850, 851, 852, 853, + /* c0 */ GROUP(331), GROUP(332), 1301, 1302, + /* c4 */ GROUP(333), GROUP(403), GROUP(405), GROUP(406), + /* c8 */ 200, 776, 1303, 1304, + /* cc */ 713, 714, GROUP(407), GROUP(408), + /* d0 */ GROUP(409), GROUP(410), GROUP(411), GROUP(412), + /* d4 */ GROUP(413), GROUP(414), GROUP(415), 1486, + /* d8 */ GROUP(416), GROUP(419), GROUP(422), GROUP(425), + /* dc */ GROUP(428), GROUP(431), GROUP(434), GROUP(437), + /* e0 */ 794, 795, 796, GROUP(440), + /* e4 */ 690, 691, 978, 979, + /* e8 */ 72, 763, GROUP(441), 765, + /* ec */ 692, 693, 980, 981, + /* f0 */ 789, 712, 1299, 1300, + /* f4 */ 687, 83, GROUP(442), GROUP(443), + /* f8 */ 77, 1395, 81, 1398, + /* fc */ 78, 1396, GROUP(444), GROUP(445), +}; + +static const uint16_t ud_itab__1[] = { + /* 0 */ 1240, INVALID, +}; + +static const uint16_t ud_itab__2[] = { + /* 0 */ 1096, INVALID, +}; + +static const uint16_t ud_itab__3[] = { + /* 0 */ 1241, INVALID, +}; + +static const uint16_t ud_itab__4[] = { + /* 0 */ GROUP(5), GROUP(6), 767, 797, + /* 4 */ INVALID, 1426, 82, 1431, + /* 8 */ 716, 1471, INVALID, 1444, + /* c */ INVALID, GROUP(27), 430, GROUP(28), + /* 10 */ GROUP(29), GROUP(30), GROUP(31), GROUP(34), + /* 14 */ GROUP(35), GROUP(36), GROUP(37), GROUP(40), + /* 18 */ GROUP(41), 955, 956, 957, + /* 1c */ 958, 959, 960, 961, + /* 20 */ 854, 855, 856, 857, + /* 24 */ INVALID, INVALID, INVALID, INVALID, + /* 28 */ GROUP(42), GROUP(43), GROUP(44), GROUP(45), + /* 2c */ GROUP(46), GROUP(47), GROUP(48), GROUP(49), + /* 30 */ 1472, 1297, 1295, 1296, + /* 34 */ GROUP(50), GROUP(52), INVALID, 1514, + /* 38 */ GROUP(54), INVALID, GROUP(116), INVALID, + /* 3c */ INVALID, INVALID, INVALID, INVALID, + /* 40 */ 84, 85, 86, 87, + /* 44 */ 88, 89, 90, 91, + /* 48 */ 92, 93, 94, 95, + /* 4c */ 96, 97, 98, 99, + /* 50 */ GROUP(143), GROUP(144), GROUP(145), GROUP(146), + /* 54 */ GROUP(147), GROUP(148), GROUP(149), GROUP(150), + /* 58 */ GROUP(151), GROUP(152), GROUP(153), GROUP(154), + /* 5c */ GROUP(155), GROUP(156), GROUP(157), GROUP(158), + /* 60 */ GROUP(159), GROUP(160), GROUP(161), GROUP(162), + /* 64 */ GROUP(163), GROUP(164), GROUP(165), GROUP(166), + /* 68 */ GROUP(167), GROUP(168), GROUP(169), GROUP(170), + /* 6c */ GROUP(171), GROUP(172), GROUP(173), GROUP(176), + /* 70 */ GROUP(177), GROUP(178), GROUP(182), GROUP(186), + /* 74 */ GROUP(191), GROUP(192), GROUP(193), 199, + /* 78 */ GROUP(194), GROUP(195), INVALID, INVALID, + /* 7c */ GROUP(196), GROUP(197), GROUP(198), GROUP(201), + /* 80 */ 727, 729, 731, 733, + /* 84 */ 735, 737, 739, 741, + /* 88 */ 743, 745, 747, 749, + /* 8c */ 751, 753, 755, 757, + /* 90 */ 1350, 1351, 1352, 1353, + /* 94 */ 1354, 1355, 1356, 1357, + /* 98 */ 1358, 1359, 1360, 1361, + /* 9c */ 1362, 1363, 1364, 1365, + /* a0 */ 1245, 1100, 131, 1670, + /* a4 */ 1375, 1376, GROUP(202), GROUP(207), + /* a8 */ 1244, 1099, 1305, 1675, + /* ac */ 1377, 1378, GROUP(215), 694, + /* b0 */ 122, 123, 775, 1673, + /* b4 */ 772, 773, 940, 941, + /* b8 */ GROUP(221), INVALID, GROUP(222), 1671, + /* bc */ 1659, 1660, 930, 931, + /* c0 */ 1473, 1474, GROUP(223), 904, + /* c4 */ GROUP(224), GROUP(225), GROUP(226), GROUP(227), + /* c8 */ 1661, 1662, 1663, 1664, + /* cc */ 1665, 1666, 1667, 1668, + /* d0 */ GROUP(236), GROUP(237), GROUP(238), GROUP(239), + /* d4 */ GROUP(240), GROUP(241), GROUP(242), GROUP(243), + /* d8 */ GROUP(244), GROUP(245), GROUP(246), GROUP(247), + /* dc */ GROUP(248), GROUP(249), GROUP(250), GROUP(251), + /* e0 */ GROUP(252), GROUP(253), GROUP(254), GROUP(255), + /* e4 */ GROUP(256), GROUP(257), GROUP(258), GROUP(259), + /* e8 */ GROUP(260), GROUP(261), GROUP(262), GROUP(263), + /* ec */ GROUP(264), GROUP(265), GROUP(266), GROUP(267), + /* f0 */ GROUP(268), GROUP(269), GROUP(270), GROUP(271), + /* f4 */ GROUP(272), GROUP(273), GROUP(274), GROUP(275), + /* f8 */ GROUP(277), GROUP(278), GROUP(279), GROUP(280), + /* fc */ GROUP(281), GROUP(282), GROUP(283), INVALID, +}; + +static const uint16_t ud_itab__5[] = { + /* 0 */ 1384, 1406, 786, 798, + /* 4 */ 1453, 1454, INVALID, INVALID, +}; + +static const uint16_t ud_itab__6[] = { + /* 0 */ GROUP(7), GROUP(8), +}; + +static const uint16_t ud_itab__7[] = { + /* 0 */ 1374, 1383, 785, 774, + /* 4 */ 1385, INVALID, 787, 719, +}; + +static const uint16_t ud_itab__8[] = { + /* 0 */ GROUP(9), GROUP(14), GROUP(15), GROUP(16), + /* 4 */ 1386, INVALID, 788, GROUP(25), +}; + +static const uint16_t ud_itab__9[] = { + /* 0 */ INVALID, GROUP(10), GROUP(11), GROUP(12), + /* 4 */ GROUP(13), INVALID, INVALID, INVALID, +}; + +static const uint16_t ud_itab__10[] = { + /* 0 */ INVALID, 1455, INVALID, +}; + +static const uint16_t ud_itab__11[] = { + /* 0 */ INVALID, 1461, INVALID, +}; + +static const uint16_t ud_itab__12[] = { + /* 0 */ INVALID, 1462, INVALID, +}; + +static const uint16_t ud_itab__13[] = { + /* 0 */ INVALID, 1463, INVALID, +}; + +static const uint16_t ud_itab__14[] = { + /* 0 */ 824, 952, INVALID, INVALID, + /* 4 */ INVALID, INVALID, INVALID, INVALID, +}; + +static const uint16_t ud_itab__15[] = { + /* 0 */ 1485, 1508, INVALID, INVALID, + /* 4 */ INVALID, INVALID, INVALID, INVALID, +}; + +static const uint16_t ud_itab__16[] = { + /* 0 */ GROUP(17), GROUP(18), GROUP(19), GROUP(20), + /* 4 */ GROUP(21), GROUP(22), GROUP(23), GROUP(24), +}; + +static const uint16_t ud_itab__17[] = { + /* 0 */ 1466, INVALID, INVALID, +}; + +static const uint16_t ud_itab__18[] = { + /* 0 */ 1467, INVALID, INVALID, +}; + +static const uint16_t ud_itab__19[] = { + /* 0 */ 1468, INVALID, INVALID, +}; + +static const uint16_t ud_itab__20[] = { + /* 0 */ 1469, INVALID, INVALID, +}; + +static const uint16_t ud_itab__21[] = { + /* 0 */ 1397, INVALID, INVALID, +}; + +static const uint16_t ud_itab__22[] = { + /* 0 */ 80, INVALID, INVALID, +}; + +static const uint16_t ud_itab__23[] = { + /* 0 */ 1399, INVALID, INVALID, +}; + +static const uint16_t ud_itab__24[] = { + /* 0 */ 720, INVALID, INVALID, +}; + +static const uint16_t ud_itab__25[] = { + /* 0 */ 1425, GROUP(26), INVALID, INVALID, + /* 4 */ INVALID, INVALID, INVALID, INVALID, +}; + +static const uint16_t ud_itab__26[] = { + /* 0 */ 1298, INVALID, INVALID, +}; + +static const uint16_t ud_itab__27[] = { + /* 0 */ 1119, 1120, 1121, 1122, + /* 4 */ 1123, 1124, 1125, 1126, +}; + +static const uint16_t ud_itab__28[] = { + /* 0 */ INVALID, INVALID, INVALID, INVALID, + /* 4 */ INVALID, INVALID, INVALID, INVALID, + /* 8 */ INVALID, INVALID, INVALID, INVALID, + /* c */ 1216, 1217, INVALID, INVALID, + /* 10 */ INVALID, INVALID, INVALID, INVALID, + /* 14 */ INVALID, INVALID, INVALID, INVALID, + /* 18 */ INVALID, INVALID, INVALID, INVALID, + /* 1c */ 1218, 1219, INVALID, INVALID, + /* 20 */ INVALID, INVALID, INVALID, INVALID, + /* 24 */ INVALID, INVALID, INVALID, INVALID, + /* 28 */ INVALID, INVALID, INVALID, INVALID, + /* 2c */ INVALID, INVALID, INVALID, INVALID, + /* 30 */ INVALID, INVALID, INVALID, INVALID, + /* 34 */ INVALID, INVALID, INVALID, INVALID, + /* 38 */ INVALID, INVALID, INVALID, INVALID, + /* 3c */ INVALID, INVALID, INVALID, INVALID, + /* 40 */ INVALID, INVALID, INVALID, INVALID, + /* 44 */ INVALID, INVALID, INVALID, INVALID, + /* 48 */ INVALID, INVALID, INVALID, INVALID, + /* 4c */ INVALID, INVALID, INVALID, INVALID, + /* 50 */ INVALID, INVALID, INVALID, INVALID, + /* 54 */ INVALID, INVALID, INVALID, INVALID, + /* 58 */ INVALID, INVALID, INVALID, INVALID, + /* 5c */ INVALID, INVALID, INVALID, INVALID, + /* 60 */ INVALID, INVALID, INVALID, INVALID, + /* 64 */ INVALID, INVALID, INVALID, INVALID, + /* 68 */ INVALID, INVALID, INVALID, INVALID, + /* 6c */ INVALID, INVALID, INVALID, INVALID, + /* 70 */ INVALID, INVALID, INVALID, INVALID, + /* 74 */ INVALID, INVALID, INVALID, INVALID, + /* 78 */ INVALID, INVALID, INVALID, INVALID, + /* 7c */ INVALID, INVALID, INVALID, INVALID, + /* 80 */ INVALID, INVALID, INVALID, INVALID, + /* 84 */ INVALID, INVALID, INVALID, INVALID, + /* 88 */ INVALID, INVALID, 1220, INVALID, + /* 8c */ INVALID, INVALID, 1221, INVALID, + /* 90 */ 1222, INVALID, INVALID, INVALID, + /* 94 */ 1223, INVALID, 1224, 1225, + /* 98 */ INVALID, INVALID, 1226, INVALID, + /* 9c */ INVALID, INVALID, 1227, INVALID, + /* a0 */ 1228, INVALID, INVALID, INVALID, + /* a4 */ 1229, INVALID, 1230, 1231, + /* a8 */ INVALID, INVALID, 1232, INVALID, + /* ac */ INVALID, INVALID, 1233, INVALID, + /* b0 */ 1234, INVALID, INVALID, INVALID, + /* b4 */ 1235, INVALID, 1236, 1237, + /* b8 */ INVALID, INVALID, INVALID, 1238, + /* bc */ INVALID, INVALID, INVALID, 1239, + /* c0 */ INVALID, INVALID, INVALID, INVALID, + /* c4 */ INVALID, INVALID, INVALID, INVALID, + /* c8 */ INVALID, INVALID, INVALID, INVALID, + /* cc */ INVALID, INVALID, INVALID, INVALID, + /* d0 */ INVALID, INVALID, INVALID, INVALID, + /* d4 */ INVALID, INVALID, INVALID, INVALID, + /* d8 */ INVALID, INVALID, INVALID, INVALID, + /* dc */ INVALID, INVALID, INVALID, INVALID, + /* e0 */ INVALID, INVALID, INVALID, INVALID, + /* e4 */ INVALID, INVALID, INVALID, INVALID, + /* e8 */ INVALID, INVALID, INVALID, INVALID, + /* ec */ INVALID, INVALID, INVALID, INVALID, + /* f0 */ INVALID, INVALID, INVALID, INVALID, + /* f4 */ INVALID, INVALID, INVALID, INVALID, + /* f8 */ INVALID, INVALID, INVALID, INVALID, + /* fc */ INVALID, INVALID, INVALID, INVALID, +}; + +static const uint16_t ud_itab__29[] = { + /* 0 */ 936, 925, 928, 932, +}; + +static const uint16_t ud_itab__30[] = { + /* 0 */ 938, 926, 929, 934, +}; + +static const uint16_t ud_itab__31[] = { + /* 0 */ GROUP(32), GROUP(33), +}; + +static const uint16_t ud_itab__32[] = { + /* 0 */ 892, 1563, 1571, 888, +}; + +static const uint16_t ud_itab__33[] = { + /* 0 */ 896, 1561, 1569, INVALID, +}; + +static const uint16_t ud_itab__34[] = { + /* 0 */ 894, INVALID, INVALID, 890, +}; + +static const uint16_t ud_itab__35[] = { + /* 0 */ 1449, INVALID, INVALID, 1451, +}; + +static const uint16_t ud_itab__36[] = { + /* 0 */ 1447, INVALID, INVALID, 1445, +}; + +static const uint16_t ud_itab__37[] = { + /* 0 */ GROUP(38), GROUP(39), +}; + +static const uint16_t ud_itab__38[] = { + /* 0 */ 882, INVALID, 1567, 878, +}; + +static const uint16_t ud_itab__39[] = { + /* 0 */ 886, INVALID, 1565, INVALID, +}; + +static const uint16_t ud_itab__40[] = { + /* 0 */ 884, INVALID, INVALID, 880, +}; + +static const uint16_t ud_itab__41[] = { + /* 0 */ 1127, 1128, 1129, 1130, + /* 4 */ INVALID, INVALID, INVALID, INVALID, +}; + +static const uint16_t ud_itab__42[] = { + /* 0 */ 862, INVALID, INVALID, 858, +}; + +static const uint16_t ud_itab__43[] = { + /* 0 */ 864, INVALID, INVALID, 860, +}; + +static const uint16_t ud_itab__44[] = { + /* 0 */ 141, 152, 154, 142, +}; + +static const uint16_t ud_itab__45[] = { + /* 0 */ 907, INVALID, INVALID, 905, +}; + +static const uint16_t ud_itab__46[] = { + /* 0 */ 165, 166, 168, 162, +}; + +static const uint16_t ud_itab__47[] = { + /* 0 */ 147, 148, 158, 138, +}; + +static const uint16_t ud_itab__48[] = { + /* 0 */ 1442, INVALID, INVALID, 1440, +}; + +static const uint16_t ud_itab__49[] = { + /* 0 */ 129, INVALID, INVALID, 127, +}; + +static const uint16_t ud_itab__50[] = { + /* 0 */ 1427, GROUP(51), +}; + +static const uint16_t ud_itab__51[] = { + /* 0 */ INVALID, 1428, INVALID, +}; + +static const uint16_t ud_itab__52[] = { + /* 0 */ 1429, GROUP(53), +}; + +static const uint16_t ud_itab__53[] = { + /* 0 */ INVALID, 1430, INVALID, +}; + +static const uint16_t ud_itab__54[] = { + /* 0 */ GROUP(67), GROUP(68), GROUP(63), GROUP(64), + /* 4 */ GROUP(65), GROUP(66), GROUP(86), GROUP(90), + /* 8 */ GROUP(69), GROUP(70), GROUP(71), GROUP(72), + /* c */ INVALID, INVALID, INVALID, INVALID, + /* 10 */ GROUP(73), INVALID, INVALID, INVALID, + /* 14 */ GROUP(75), GROUP(76), INVALID, GROUP(77), + /* 18 */ INVALID, INVALID, INVALID, INVALID, + /* 1c */ GROUP(78), GROUP(79), GROUP(80), INVALID, + /* 20 */ GROUP(81), GROUP(82), GROUP(83), GROUP(84), + /* 24 */ GROUP(85), GROUP(108), INVALID, INVALID, + /* 28 */ GROUP(87), GROUP(88), GROUP(89), GROUP(74), + /* 2c */ INVALID, INVALID, INVALID, INVALID, + /* 30 */ GROUP(91), GROUP(92), GROUP(93), GROUP(94), + /* 34 */ GROUP(95), GROUP(96), INVALID, GROUP(97), + /* 38 */ GROUP(98), GROUP(99), GROUP(100), GROUP(101), + /* 3c */ GROUP(102), GROUP(103), GROUP(104), GROUP(105), + /* 40 */ GROUP(106), GROUP(107), INVALID, INVALID, + /* 44 */ INVALID, INVALID, INVALID, INVALID, + /* 48 */ INVALID, INVALID, INVALID, INVALID, + /* 4c */ INVALID, INVALID, INVALID, INVALID, + /* 50 */ INVALID, INVALID, INVALID, INVALID, + /* 54 */ INVALID, INVALID, INVALID, INVALID, + /* 58 */ INVALID, INVALID, INVALID, INVALID, + /* 5c */ INVALID, INVALID, INVALID, INVALID, + /* 60 */ INVALID, INVALID, INVALID, INVALID, + /* 64 */ INVALID, INVALID, INVALID, INVALID, + /* 68 */ INVALID, INVALID, INVALID, INVALID, + /* 6c */ INVALID, INVALID, INVALID, INVALID, + /* 70 */ INVALID, INVALID, INVALID, INVALID, + /* 74 */ INVALID, INVALID, INVALID, INVALID, + /* 78 */ INVALID, INVALID, INVALID, INVALID, + /* 7c */ INVALID, INVALID, INVALID, INVALID, + /* 80 */ GROUP(55), GROUP(59), INVALID, INVALID, + /* 84 */ INVALID, INVALID, INVALID, INVALID, + /* 88 */ INVALID, INVALID, INVALID, INVALID, + /* 8c */ INVALID, INVALID, INVALID, INVALID, + /* 90 */ INVALID, INVALID, INVALID, INVALID, + /* 94 */ INVALID, INVALID, INVALID, INVALID, + /* 98 */ INVALID, INVALID, INVALID, INVALID, + /* 9c */ INVALID, INVALID, INVALID, INVALID, + /* a0 */ INVALID, INVALID, INVALID, INVALID, + /* a4 */ INVALID, INVALID, INVALID, INVALID, + /* a8 */ INVALID, INVALID, INVALID, INVALID, + /* ac */ INVALID, INVALID, INVALID, INVALID, + /* b0 */ INVALID, INVALID, INVALID, INVALID, + /* b4 */ INVALID, INVALID, INVALID, INVALID, + /* b8 */ INVALID, INVALID, INVALID, INVALID, + /* bc */ INVALID, INVALID, INVALID, INVALID, + /* c0 */ INVALID, INVALID, INVALID, INVALID, + /* c4 */ INVALID, INVALID, INVALID, INVALID, + /* c8 */ INVALID, INVALID, INVALID, INVALID, + /* cc */ INVALID, INVALID, INVALID, INVALID, + /* d0 */ INVALID, INVALID, INVALID, INVALID, + /* d4 */ INVALID, INVALID, INVALID, INVALID, + /* d8 */ INVALID, INVALID, INVALID, GROUP(109), + /* dc */ GROUP(110), GROUP(111), GROUP(112), GROUP(113), + /* e0 */ INVALID, INVALID, INVALID, INVALID, + /* e4 */ INVALID, INVALID, INVALID, INVALID, + /* e8 */ INVALID, INVALID, INVALID, INVALID, + /* ec */ INVALID, INVALID, INVALID, INVALID, + /* f0 */ GROUP(114), GROUP(115), INVALID, INVALID, + /* f4 */ INVALID, INVALID, INVALID, INVALID, + /* f8 */ INVALID, INVALID, INVALID, INVALID, + /* fc */ INVALID, INVALID, INVALID, INVALID, +}; + +static const uint16_t ud_itab__55[] = { + /* 0 */ INVALID, INVALID, INVALID, GROUP(56), +}; + +static const uint16_t ud_itab__56[] = { + /* 0 */ GROUP(57), GROUP(58), +}; + +static const uint16_t ud_itab__57[] = { + /* 0 */ INVALID, 717, INVALID, +}; + +static const uint16_t ud_itab__58[] = { + /* 0 */ INVALID, 718, INVALID, +}; + +static const uint16_t ud_itab__59[] = { + /* 0 */ INVALID, INVALID, INVALID, GROUP(60), +}; + +static const uint16_t ud_itab__60[] = { + /* 0 */ GROUP(61), GROUP(62), +}; + +static const uint16_t ud_itab__61[] = { + /* 0 */ INVALID, 721, INVALID, +}; + +static const uint16_t ud_itab__62[] = { + /* 0 */ INVALID, 722, INVALID, +}; + +static const uint16_t ud_itab__63[] = { + /* 0 */ 1588, INVALID, INVALID, 1589, +}; + +static const uint16_t ud_itab__64[] = { + /* 0 */ 1591, INVALID, INVALID, 1592, +}; + +static const uint16_t ud_itab__65[] = { + /* 0 */ 1594, INVALID, INVALID, 1595, +}; + +static const uint16_t ud_itab__66[] = { + /* 0 */ 1597, INVALID, INVALID, 1598, +}; + +static const uint16_t ud_itab__67[] = { + /* 0 */ 1582, INVALID, INVALID, 1583, +}; + +static const uint16_t ud_itab__68[] = { + /* 0 */ 1585, INVALID, INVALID, 1586, +}; + +static const uint16_t ud_itab__69[] = { + /* 0 */ 1606, INVALID, INVALID, 1607, +}; + +static const uint16_t ud_itab__70[] = { + /* 0 */ 1612, INVALID, INVALID, 1613, +}; + +static const uint16_t ud_itab__71[] = { + /* 0 */ 1609, INVALID, INVALID, 1610, +}; + +static const uint16_t ud_itab__72[] = { + /* 0 */ 1615, INVALID, INVALID, 1616, +}; + +static const uint16_t ud_itab__73[] = { + /* 0 */ INVALID, INVALID, INVALID, 1621, +}; + +static const uint16_t ud_itab__74[] = { + /* 0 */ INVALID, INVALID, INVALID, 1683, +}; + +static const uint16_t ud_itab__75[] = { + /* 0 */ INVALID, INVALID, INVALID, 1657, +}; + +static const uint16_t ud_itab__76[] = { + /* 0 */ INVALID, INVALID, INVALID, 1656, +}; + +static const uint16_t ud_itab__77[] = { + /* 0 */ INVALID, INVALID, INVALID, 1711, +}; + +static const uint16_t ud_itab__78[] = { + /* 0 */ 1573, INVALID, INVALID, 1574, +}; + +static const uint16_t ud_itab__79[] = { + /* 0 */ 1576, INVALID, INVALID, 1577, +}; + +static const uint16_t ud_itab__80[] = { + /* 0 */ 1579, INVALID, INVALID, 1580, +}; + +static const uint16_t ud_itab__81[] = { + /* 0 */ INVALID, INVALID, INVALID, 1685, +}; + +static const uint16_t ud_itab__82[] = { + /* 0 */ INVALID, INVALID, INVALID, 1687, +}; + +static const uint16_t ud_itab__83[] = { + /* 0 */ INVALID, INVALID, INVALID, 1689, +}; + +static const uint16_t ud_itab__84[] = { + /* 0 */ INVALID, INVALID, INVALID, 1691, +}; + +static const uint16_t ud_itab__85[] = { + /* 0 */ INVALID, INVALID, INVALID, 1693, +}; + +static const uint16_t ud_itab__86[] = { + /* 0 */ 1600, INVALID, INVALID, 1601, +}; + +static const uint16_t ud_itab__87[] = { + /* 0 */ INVALID, INVALID, INVALID, 1622, +}; + +static const uint16_t ud_itab__88[] = { + /* 0 */ INVALID, INVALID, INVALID, 1708, +}; + +static const uint16_t ud_itab__89[] = { + /* 0 */ INVALID, INVALID, INVALID, 1681, +}; + +static const uint16_t ud_itab__90[] = { + /* 0 */ 1603, INVALID, INVALID, 1604, +}; + +static const uint16_t ud_itab__91[] = { + /* 0 */ INVALID, INVALID, INVALID, 1696, +}; + +static const uint16_t ud_itab__92[] = { + /* 0 */ INVALID, INVALID, INVALID, 1698, +}; + +static const uint16_t ud_itab__93[] = { + /* 0 */ INVALID, INVALID, INVALID, 1700, +}; + +static const uint16_t ud_itab__94[] = { + /* 0 */ INVALID, INVALID, INVALID, 1702, +}; + +static const uint16_t ud_itab__95[] = { + /* 0 */ INVALID, INVALID, INVALID, 1704, +}; + +static const uint16_t ud_itab__96[] = { + /* 0 */ INVALID, INVALID, INVALID, 1706, +}; + +static const uint16_t ud_itab__97[] = { + /* 0 */ INVALID, INVALID, INVALID, 1717, +}; + +static const uint16_t ud_itab__98[] = { + /* 0 */ INVALID, INVALID, INVALID, 1624, +}; + +static const uint16_t ud_itab__99[] = { + /* 0 */ INVALID, INVALID, INVALID, 1626, +}; + +static const uint16_t ud_itab__100[] = { + /* 0 */ INVALID, INVALID, INVALID, 1628, +}; + +static const uint16_t ud_itab__101[] = { + /* 0 */ INVALID, INVALID, INVALID, 1630, +}; + +static const uint16_t ud_itab__102[] = { + /* 0 */ INVALID, INVALID, INVALID, 1632, +}; + +static const uint16_t ud_itab__103[] = { + /* 0 */ INVALID, INVALID, INVALID, 1634, +}; + +static const uint16_t ud_itab__104[] = { + /* 0 */ INVALID, INVALID, INVALID, 1638, +}; + +static const uint16_t ud_itab__105[] = { + /* 0 */ INVALID, INVALID, INVALID, 1636, +}; + +static const uint16_t ud_itab__106[] = { + /* 0 */ INVALID, INVALID, INVALID, 1640, +}; + +static const uint16_t ud_itab__107[] = { + /* 0 */ INVALID, INVALID, INVALID, 1642, +}; + +static const uint16_t ud_itab__108[] = { + /* 0 */ INVALID, INVALID, INVALID, 1695, +}; + +static const uint16_t ud_itab__109[] = { + /* 0 */ INVALID, INVALID, INVALID, 45, +}; + +static const uint16_t ud_itab__110[] = { + /* 0 */ INVALID, INVALID, INVALID, 41, +}; + +static const uint16_t ud_itab__111[] = { + /* 0 */ INVALID, INVALID, INVALID, 43, +}; + +static const uint16_t ud_itab__112[] = { + /* 0 */ INVALID, INVALID, INVALID, 37, +}; + +static const uint16_t ud_itab__113[] = { + /* 0 */ INVALID, INVALID, INVALID, 39, +}; + +static const uint16_t ud_itab__114[] = { + /* 0 */ 1723, 1725, INVALID, INVALID, +}; + +static const uint16_t ud_itab__115[] = { + /* 0 */ 1724, 1726, INVALID, INVALID, +}; + +static const uint16_t ud_itab__116[] = { + /* 0 */ INVALID, INVALID, INVALID, INVALID, + /* 4 */ INVALID, INVALID, INVALID, INVALID, + /* 8 */ GROUP(117), GROUP(118), GROUP(119), GROUP(120), + /* c */ GROUP(121), GROUP(122), GROUP(123), GROUP(124), + /* 10 */ INVALID, INVALID, INVALID, INVALID, + /* 14 */ GROUP(125), GROUP(126), GROUP(127), GROUP(129), + /* 18 */ INVALID, INVALID, INVALID, INVALID, + /* 1c */ INVALID, INVALID, INVALID, INVALID, + /* 20 */ GROUP(130), GROUP(131), GROUP(132), INVALID, + /* 24 */ INVALID, INVALID, INVALID, INVALID, + /* 28 */ INVALID, INVALID, INVALID, INVALID, + /* 2c */ INVALID, INVALID, INVALID, INVALID, + /* 30 */ INVALID, INVALID, INVALID, INVALID, + /* 34 */ INVALID, INVALID, INVALID, INVALID, + /* 38 */ INVALID, INVALID, INVALID, INVALID, + /* 3c */ INVALID, INVALID, INVALID, INVALID, + /* 40 */ GROUP(134), GROUP(135), GROUP(136), INVALID, + /* 44 */ GROUP(137), INVALID, INVALID, INVALID, + /* 48 */ INVALID, INVALID, INVALID, INVALID, + /* 4c */ INVALID, INVALID, INVALID, INVALID, + /* 50 */ INVALID, INVALID, INVALID, INVALID, + /* 54 */ INVALID, INVALID, INVALID, INVALID, + /* 58 */ INVALID, INVALID, INVALID, INVALID, + /* 5c */ INVALID, INVALID, INVALID, INVALID, + /* 60 */ GROUP(139), GROUP(140), GROUP(141), GROUP(142), + /* 64 */ INVALID, INVALID, INVALID, INVALID, + /* 68 */ INVALID, INVALID, INVALID, INVALID, + /* 6c */ INVALID, INVALID, INVALID, INVALID, + /* 70 */ INVALID, INVALID, INVALID, INVALID, + /* 74 */ INVALID, INVALID, INVALID, INVALID, + /* 78 */ INVALID, INVALID, INVALID, INVALID, + /* 7c */ INVALID, INVALID, INVALID, INVALID, + /* 80 */ INVALID, INVALID, INVALID, INVALID, + /* 84 */ INVALID, INVALID, INVALID, INVALID, + /* 88 */ INVALID, INVALID, INVALID, INVALID, + /* 8c */ INVALID, INVALID, INVALID, INVALID, + /* 90 */ INVALID, INVALID, INVALID, INVALID, + /* 94 */ INVALID, INVALID, INVALID, INVALID, + /* 98 */ INVALID, INVALID, INVALID, INVALID, + /* 9c */ INVALID, INVALID, INVALID, INVALID, + /* a0 */ INVALID, INVALID, INVALID, INVALID, + /* a4 */ INVALID, INVALID, INVALID, INVALID, + /* a8 */ INVALID, INVALID, INVALID, INVALID, + /* ac */ INVALID, INVALID, INVALID, INVALID, + /* b0 */ INVALID, INVALID, INVALID, INVALID, + /* b4 */ INVALID, INVALID, INVALID, INVALID, + /* b8 */ INVALID, INVALID, INVALID, INVALID, + /* bc */ INVALID, INVALID, INVALID, INVALID, + /* c0 */ INVALID, INVALID, INVALID, INVALID, + /* c4 */ INVALID, INVALID, INVALID, INVALID, + /* c8 */ INVALID, INVALID, INVALID, INVALID, + /* cc */ INVALID, INVALID, INVALID, INVALID, + /* d0 */ INVALID, INVALID, INVALID, INVALID, + /* d4 */ INVALID, INVALID, INVALID, INVALID, + /* d8 */ INVALID, INVALID, INVALID, INVALID, + /* dc */ INVALID, INVALID, INVALID, GROUP(138), + /* e0 */ INVALID, INVALID, INVALID, INVALID, + /* e4 */ INVALID, INVALID, INVALID, INVALID, + /* e8 */ INVALID, INVALID, INVALID, INVALID, + /* ec */ INVALID, INVALID, INVALID, INVALID, + /* f0 */ INVALID, INVALID, INVALID, INVALID, + /* f4 */ INVALID, INVALID, INVALID, INVALID, + /* f8 */ INVALID, INVALID, INVALID, INVALID, + /* fc */ INVALID, INVALID, INVALID, INVALID, +}; + +static const uint16_t ud_itab__117[] = { + /* 0 */ INVALID, INVALID, INVALID, 1644, +}; + +static const uint16_t ud_itab__118[] = { + /* 0 */ INVALID, INVALID, INVALID, 1646, +}; + +static const uint16_t ud_itab__119[] = { + /* 0 */ INVALID, INVALID, INVALID, 1648, +}; + +static const uint16_t ud_itab__120[] = { + /* 0 */ INVALID, INVALID, INVALID, 1650, +}; + +static const uint16_t ud_itab__121[] = { + /* 0 */ INVALID, INVALID, INVALID, 1654, +}; + +static const uint16_t ud_itab__122[] = { + /* 0 */ INVALID, INVALID, INVALID, 1652, +}; + +static const uint16_t ud_itab__123[] = { + /* 0 */ INVALID, INVALID, INVALID, 1677, +}; + +static const uint16_t ud_itab__124[] = { + /* 0 */ 1618, INVALID, INVALID, 1619, +}; + +static const uint16_t ud_itab__125[] = { + /* 0 */ INVALID, INVALID, INVALID, 1045, +}; + +static const uint16_t ud_itab__126[] = { + /* 0 */ INVALID, INVALID, INVALID, 1056, +}; + +static const uint16_t ud_itab__127[] = { + /* 0 */ INVALID, INVALID, INVALID, GROUP(128), +}; + +static const uint16_t ud_itab__128[] = { + /* 0 */ 1047, 1049, 1051, +}; + +static const uint16_t ud_itab__129[] = { + /* 0 */ INVALID, INVALID, INVALID, 201, +}; + +static const uint16_t ud_itab__130[] = { + /* 0 */ INVALID, INVALID, INVALID, 1058, +}; + +static const uint16_t ud_itab__131[] = { + /* 0 */ INVALID, INVALID, INVALID, 1557, +}; + +static const uint16_t ud_itab__132[] = { + /* 0 */ INVALID, INVALID, INVALID, GROUP(133), +}; + +static const uint16_t ud_itab__133[] = { + /* 0 */ 1062, 1063, 1064, +}; + +static const uint16_t ud_itab__134[] = { + /* 0 */ INVALID, INVALID, INVALID, 197, +}; + +static const uint16_t ud_itab__135[] = { + /* 0 */ INVALID, INVALID, INVALID, 195, +}; + +static const uint16_t ud_itab__136[] = { + /* 0 */ INVALID, INVALID, INVALID, 1679, +}; + +static const uint16_t ud_itab__137[] = { + /* 0 */ INVALID, INVALID, INVALID, 1512, +}; + +static const uint16_t ud_itab__138[] = { + /* 0 */ INVALID, INVALID, INVALID, 47, +}; + +static const uint16_t ud_itab__139[] = { + /* 0 */ INVALID, INVALID, INVALID, 1715, +}; + +static const uint16_t ud_itab__140[] = { + /* 0 */ INVALID, INVALID, INVALID, 1713, +}; + +static const uint16_t ud_itab__141[] = { + /* 0 */ INVALID, INVALID, INVALID, 1721, +}; + +static const uint16_t ud_itab__142[] = { + /* 0 */ INVALID, INVALID, INVALID, 1719, +}; + +static const uint16_t ud_itab__143[] = { + /* 0 */ 900, INVALID, INVALID, 898, +}; + +static const uint16_t ud_itab__144[] = { + /* 0 */ 1387, 1391, 1393, 1389, +}; + +static const uint16_t ud_itab__145[] = { + /* 0 */ 1306, INVALID, 1308, INVALID, +}; + +static const uint16_t ud_itab__146[] = { + /* 0 */ 1291, INVALID, 1293, INVALID, +}; + +static const uint16_t ud_itab__147[] = { + /* 0 */ 61, INVALID, INVALID, 59, +}; + +static const uint16_t ud_itab__148[] = { + /* 0 */ 65, INVALID, INVALID, 63, +}; + +static const uint16_t ud_itab__149[] = { + /* 0 */ 976, INVALID, INVALID, 974, +}; + +static const uint16_t ud_itab__150[] = { + /* 0 */ 1499, INVALID, INVALID, 1497, +}; + +static const uint16_t ud_itab__151[] = { + /* 0 */ 27, 29, 31, 25, +}; + +static const uint16_t ud_itab__152[] = { + /* 0 */ 946, 948, 950, 944, +}; + +static const uint16_t ud_itab__153[] = { + /* 0 */ 145, 150, 156, 139, +}; + +static const uint16_t ud_itab__154[] = { + /* 0 */ 134, INVALID, 163, 143, +}; + +static const uint16_t ud_itab__155[] = { + /* 0 */ 1419, 1421, 1423, 1417, +}; + +static const uint16_t ud_itab__156[] = { + /* 0 */ 818, 820, 822, 816, +}; + +static const uint16_t ud_itab__157[] = { + /* 0 */ 189, 191, 193, 187, +}; + +static const uint16_t ud_itab__158[] = { + /* 0 */ 802, 804, 806, 800, +}; + +static const uint16_t ud_itab__159[] = { + /* 0 */ 1209, INVALID, INVALID, 1207, +}; + +static const uint16_t ud_itab__160[] = { + /* 0 */ 1212, INVALID, INVALID, 1210, +}; + +static const uint16_t ud_itab__161[] = { + /* 0 */ 1215, INVALID, INVALID, 1213, +}; + +static const uint16_t ud_itab__162[] = { + /* 0 */ 987, INVALID, INVALID, 985, +}; + +static const uint16_t ud_itab__163[] = { + /* 0 */ 1038, INVALID, INVALID, 1036, +}; + +static const uint16_t ud_itab__164[] = { + /* 0 */ 1041, INVALID, INVALID, 1039, +}; + +static const uint16_t ud_itab__165[] = { + /* 0 */ 1044, INVALID, INVALID, 1042, +}; + +static const uint16_t ud_itab__166[] = { + /* 0 */ 993, INVALID, INVALID, 991, +}; + +static const uint16_t ud_itab__167[] = { + /* 0 */ 1200, INVALID, INVALID, 1198, +}; + +static const uint16_t ud_itab__168[] = { + /* 0 */ 1203, INVALID, INVALID, 1201, +}; + +static const uint16_t ud_itab__169[] = { + /* 0 */ 1206, INVALID, INVALID, 1204, +}; + +static const uint16_t ud_itab__170[] = { + /* 0 */ 990, INVALID, INVALID, 988, +}; + +static const uint16_t ud_itab__171[] = { + /* 0 */ INVALID, INVALID, INVALID, 1547, +}; + +static const uint16_t ud_itab__172[] = { + /* 0 */ INVALID, INVALID, INVALID, 1545, +}; + +static const uint16_t ud_itab__173[] = { + /* 0 */ GROUP(174), INVALID, INVALID, GROUP(175), +}; + +static const uint16_t ud_itab__174[] = { + /* 0 */ 866, 867, 910, +}; + +static const uint16_t ud_itab__175[] = { + /* 0 */ 868, 870, 911, +}; + +static const uint16_t ud_itab__176[] = { + /* 0 */ 920, INVALID, 1522, 1517, +}; + +static const uint16_t ud_itab__177[] = { + /* 0 */ 1134, 1537, 1535, 1539, +}; + +static const uint16_t ud_itab__178[] = { + /* 0 */ INVALID, INVALID, GROUP(179), INVALID, + /* 4 */ GROUP(180), INVALID, GROUP(181), INVALID, +}; + +static const uint16_t ud_itab__179[] = { + /* 0 */ 1159, INVALID, INVALID, 1163, +}; + +static const uint16_t ud_itab__180[] = { + /* 0 */ 1152, INVALID, INVALID, 1150, +}; + +static const uint16_t ud_itab__181[] = { + /* 0 */ 1138, INVALID, INVALID, 1137, +}; + +static const uint16_t ud_itab__182[] = { + /* 0 */ INVALID, INVALID, GROUP(183), INVALID, + /* 4 */ GROUP(184), INVALID, GROUP(185), INVALID, +}; + +static const uint16_t ud_itab__183[] = { + /* 0 */ 1165, INVALID, INVALID, 1169, +}; + +static const uint16_t ud_itab__184[] = { + /* 0 */ 1153, INVALID, INVALID, 1157, +}; + +static const uint16_t ud_itab__185[] = { + /* 0 */ 1142, INVALID, INVALID, 1141, +}; + +static const uint16_t ud_itab__186[] = { + /* 0 */ INVALID, INVALID, GROUP(187), GROUP(188), + /* 4 */ INVALID, INVALID, GROUP(189), GROUP(190), +}; + +static const uint16_t ud_itab__187[] = { + /* 0 */ 1171, INVALID, INVALID, 1175, +}; + +static const uint16_t ud_itab__188[] = { + /* 0 */ INVALID, INVALID, INVALID, 1543, +}; + +static const uint16_t ud_itab__189[] = { + /* 0 */ 1146, INVALID, INVALID, 1145, +}; + +static const uint16_t ud_itab__190[] = { + /* 0 */ INVALID, INVALID, INVALID, 1541, +}; + +static const uint16_t ud_itab__191[] = { + /* 0 */ 1027, INVALID, INVALID, 1028, +}; + +static const uint16_t ud_itab__192[] = { + /* 0 */ 1030, INVALID, INVALID, 1031, +}; + +static const uint16_t ud_itab__193[] = { + /* 0 */ 1033, INVALID, INVALID, 1034, +}; + +static const uint16_t ud_itab__194[] = { + /* 0 */ INVALID, 1464, INVALID, +}; + +static const uint16_t ud_itab__195[] = { + /* 0 */ INVALID, 1465, INVALID, +}; + +static const uint16_t ud_itab__196[] = { + /* 0 */ INVALID, 1551, INVALID, 1549, +}; + +static const uint16_t ud_itab__197[] = { + /* 0 */ INVALID, 1555, INVALID, 1553, +}; + +static const uint16_t ud_itab__198[] = { + /* 0 */ GROUP(199), INVALID, 916, GROUP(200), +}; + +static const uint16_t ud_itab__199[] = { + /* 0 */ 872, 873, 913, +}; + +static const uint16_t ud_itab__200[] = { + /* 0 */ 874, 876, 914, +}; + +static const uint16_t ud_itab__201[] = { + /* 0 */ 921, INVALID, 1524, 1515, +}; + +static const uint16_t ud_itab__202[] = { + /* 0 */ INVALID, GROUP(203), +}; + +static const uint16_t ud_itab__203[] = { + /* 0 */ GROUP(204), GROUP(205), GROUP(206), INVALID, + /* 4 */ INVALID, INVALID, INVALID, INVALID, +}; + +static const uint16_t ud_itab__204[] = { + /* 0 */ 825, INVALID, INVALID, INVALID, + /* 4 */ INVALID, INVALID, INVALID, INVALID, +}; + +static const uint16_t ud_itab__205[] = { + /* 0 */ 1509, INVALID, INVALID, INVALID, + /* 4 */ INVALID, INVALID, INVALID, INVALID, +}; + +static const uint16_t ud_itab__206[] = { + /* 0 */ 1510, INVALID, INVALID, INVALID, + /* 4 */ INVALID, INVALID, INVALID, INVALID, +}; + +static const uint16_t ud_itab__207[] = { + /* 0 */ INVALID, GROUP(208), +}; + +static const uint16_t ud_itab__208[] = { + /* 0 */ GROUP(209), GROUP(210), GROUP(211), GROUP(212), + /* 4 */ GROUP(213), GROUP(214), INVALID, INVALID, +}; + +static const uint16_t ud_itab__209[] = { + /* 0 */ 1511, INVALID, INVALID, INVALID, + /* 4 */ INVALID, INVALID, INVALID, INVALID, +}; + +static const uint16_t ud_itab__210[] = { + /* 0 */ 1501, INVALID, INVALID, INVALID, + /* 4 */ INVALID, INVALID, INVALID, INVALID, +}; + +static const uint16_t ud_itab__211[] = { + /* 0 */ 1502, INVALID, INVALID, INVALID, + /* 4 */ INVALID, INVALID, INVALID, INVALID, +}; + +static const uint16_t ud_itab__212[] = { + /* 0 */ 1503, INVALID, INVALID, INVALID, + /* 4 */ INVALID, INVALID, INVALID, INVALID, +}; + +static const uint16_t ud_itab__213[] = { + /* 0 */ 1504, INVALID, INVALID, INVALID, + /* 4 */ INVALID, INVALID, INVALID, INVALID, +}; + +static const uint16_t ud_itab__214[] = { + /* 0 */ 1505, INVALID, INVALID, INVALID, + /* 4 */ INVALID, INVALID, INVALID, INVALID, +}; + +static const uint16_t ud_itab__215[] = { + /* 0 */ GROUP(216), GROUP(217), +}; + +static const uint16_t ud_itab__216[] = { + /* 0 */ 683, 682, 768, 1400, + /* 4 */ 1507, 1506, INVALID, 79, +}; + +static const uint16_t ud_itab__217[] = { + /* 0 */ INVALID, INVALID, INVALID, INVALID, + /* 4 */ INVALID, GROUP(218), GROUP(219), GROUP(220), +}; + +static const uint16_t ud_itab__218[] = { + /* 0 */ 777, 778, 779, 780, + /* 4 */ 781, 782, 783, 784, +}; + +static const uint16_t ud_itab__219[] = { + /* 0 */ 808, 809, 810, 811, + /* 4 */ 812, 813, 814, 815, +}; + +static const uint16_t ud_itab__220[] = { + /* 0 */ 1366, 1367, 1368, 1369, + /* 4 */ 1370, 1371, 1372, 1373, +}; + +static const uint16_t ud_itab__221[] = { + /* 0 */ INVALID, INVALID, 1710, INVALID, +}; + +static const uint16_t ud_itab__222[] = { + /* 0 */ INVALID, INVALID, INVALID, INVALID, + /* 4 */ 1669, 1676, 1674, 1672, +}; + +static const uint16_t ud_itab__223[] = { + /* 0 */ 112, 117, 120, 110, +}; + +static const uint16_t ud_itab__224[] = { + /* 0 */ 1059, INVALID, INVALID, 1060, +}; + +static const uint16_t ud_itab__225[] = { + /* 0 */ 1055, INVALID, INVALID, 1053, +}; + +static const uint16_t ud_itab__226[] = { + /* 0 */ 1381, INVALID, INVALID, 1379, +}; + +static const uint16_t ud_itab__227[] = { + /* 0 */ GROUP(228), GROUP(235), +}; + +static const uint16_t ud_itab__228[] = { + /* 0 */ INVALID, GROUP(229), INVALID, INVALID, + /* 4 */ INVALID, INVALID, GROUP(230), GROUP(234), +}; + +static const uint16_t ud_itab__229[] = { + /* 0 */ 124, 125, 126, +}; + +static const uint16_t ud_itab__230[] = { + /* 0 */ GROUP(231), INVALID, GROUP(232), GROUP(233), +}; + +static const uint16_t ud_itab__231[] = { + /* 0 */ INVALID, 1459, INVALID, +}; + +static const uint16_t ud_itab__232[] = { + /* 0 */ INVALID, 1458, INVALID, +}; + +static const uint16_t ud_itab__233[] = { + /* 0 */ INVALID, 1457, INVALID, +}; + +static const uint16_t ud_itab__234[] = { + /* 0 */ INVALID, 1460, INVALID, +}; + +static const uint16_t ud_itab__235[] = { + /* 0 */ INVALID, INVALID, INVALID, INVALID, + /* 4 */ INVALID, INVALID, 1456, INVALID, +}; + +static const uint16_t ud_itab__236[] = { + /* 0 */ INVALID, 35, INVALID, 33, +}; + +static const uint16_t ud_itab__237[] = { + /* 0 */ 1160, INVALID, INVALID, 1161, +}; + +static const uint16_t ud_itab__238[] = { + /* 0 */ 1166, INVALID, INVALID, 1167, +}; + +static const uint16_t ud_itab__239[] = { + /* 0 */ 1172, INVALID, INVALID, 1173, +}; + +static const uint16_t ud_itab__240[] = { + /* 0 */ 1527, INVALID, INVALID, 1528, +}; + +static const uint16_t ud_itab__241[] = { + /* 0 */ 1093, INVALID, INVALID, 1094, +}; + +static const uint16_t ud_itab__242[] = { + /* 0 */ INVALID, 1521, 1526, 918, +}; + +static const uint16_t ud_itab__243[] = { + /* 0 */ 1086, INVALID, INVALID, 1084, +}; + +static const uint16_t ud_itab__244[] = { + /* 0 */ 1192, INVALID, INVALID, 1193, +}; + +static const uint16_t ud_itab__245[] = { + /* 0 */ 1195, INVALID, INVALID, 1196, +}; + +static const uint16_t ud_itab__246[] = { + /* 0 */ 1083, INVALID, INVALID, 1081, +}; + +static const uint16_t ud_itab__247[] = { + /* 0 */ 1017, INVALID, INVALID, 1015, +}; + +static const uint16_t ud_itab__248[] = { + /* 0 */ 1009, INVALID, INVALID, 1010, +}; + +static const uint16_t ud_itab__249[] = { + /* 0 */ 1012, INVALID, INVALID, 1013, +}; + +static const uint16_t ud_itab__250[] = { + /* 0 */ 1075, INVALID, INVALID, 1076, +}; + +static const uint16_t ud_itab__251[] = { + /* 0 */ 1020, INVALID, INVALID, 1018, +}; + +static const uint16_t ud_itab__252[] = { + /* 0 */ 1023, INVALID, INVALID, 1021, +}; + +static const uint16_t ud_itab__253[] = { + /* 0 */ 1147, INVALID, INVALID, 1148, +}; + +static const uint16_t ud_itab__254[] = { + /* 0 */ 1156, INVALID, INVALID, 1154, +}; + +static const uint16_t ud_itab__255[] = { + /* 0 */ 1026, INVALID, INVALID, 1024, +}; + +static const uint16_t ud_itab__256[] = { + /* 0 */ 1087, INVALID, INVALID, 1088, +}; + +static const uint16_t ud_itab__257[] = { + /* 0 */ 1092, INVALID, INVALID, 1090, +}; + +static const uint16_t ud_itab__258[] = { + /* 0 */ INVALID, 136, 132, 160, +}; + +static const uint16_t ud_itab__259[] = { + /* 0 */ 909, INVALID, INVALID, 902, +}; + +static const uint16_t ud_itab__260[] = { + /* 0 */ 1186, INVALID, INVALID, 1187, +}; + +static const uint16_t ud_itab__261[] = { + /* 0 */ 1189, INVALID, INVALID, 1190, +}; + +static const uint16_t ud_itab__262[] = { + /* 0 */ 1080, INVALID, INVALID, 1078, +}; + +static const uint16_t ud_itab__263[] = { + /* 0 */ 1118, INVALID, INVALID, 1116, +}; + +static const uint16_t ud_itab__264[] = { + /* 0 */ 1003, INVALID, INVALID, 1004, +}; + +static const uint16_t ud_itab__265[] = { + /* 0 */ 1006, INVALID, INVALID, 1007, +}; + +static const uint16_t ud_itab__266[] = { + /* 0 */ 1074, INVALID, INVALID, 1072, +}; + +static const uint16_t ud_itab__267[] = { + /* 0 */ 1266, INVALID, INVALID, 1264, +}; + +static const uint16_t ud_itab__268[] = { + /* 0 */ INVALID, 1559, INVALID, INVALID, +}; + +static const uint16_t ud_itab__269[] = { + /* 0 */ 1136, INVALID, INVALID, 1135, +}; + +static const uint16_t ud_itab__270[] = { + /* 0 */ 1140, INVALID, INVALID, 1139, +}; + +static const uint16_t ud_itab__271[] = { + /* 0 */ 1144, INVALID, INVALID, 1143, +}; + +static const uint16_t ud_itab__272[] = { + /* 0 */ 1533, INVALID, INVALID, 1534, +}; + +static const uint16_t ud_itab__273[] = { + /* 0 */ 1069, INVALID, INVALID, 1070, +}; + +static const uint16_t ud_itab__274[] = { + /* 0 */ 1133, INVALID, INVALID, 1131, +}; + +static const uint16_t ud_itab__275[] = { + /* 0 */ INVALID, GROUP(276), +}; + +static const uint16_t ud_itab__276[] = { + /* 0 */ 799, INVALID, INVALID, 1519, +}; + +static const uint16_t ud_itab__277[] = { + /* 0 */ 1179, INVALID, INVALID, 1177, +}; + +static const uint16_t ud_itab__278[] = { + /* 0 */ 1182, INVALID, INVALID, 1180, +}; + +static const uint16_t ud_itab__279[] = { + /* 0 */ 1183, INVALID, INVALID, 1184, +}; + +static const uint16_t ud_itab__280[] = { + /* 0 */ 1532, INVALID, INVALID, 1530, +}; + +static const uint16_t ud_itab__281[] = { + /* 0 */ 996, INVALID, INVALID, 994, +}; + +static const uint16_t ud_itab__282[] = { + /* 0 */ 997, INVALID, INVALID, 998, +}; + +static const uint16_t ud_itab__283[] = { + /* 0 */ 1000, INVALID, INVALID, 1001, +}; + +static const uint16_t ud_itab__284[] = { + /* 0 */ 1242, INVALID, +}; + +static const uint16_t ud_itab__285[] = { + /* 0 */ 1097, INVALID, +}; + +static const uint16_t ud_itab__286[] = { + /* 0 */ 1243, INVALID, +}; + +static const uint16_t ud_itab__287[] = { + /* 0 */ 1098, INVALID, +}; + +static const uint16_t ud_itab__288[] = { + /* 0 */ 173, INVALID, +}; + +static const uint16_t ud_itab__289[] = { + /* 0 */ 174, INVALID, +}; + +static const uint16_t ud_itab__290[] = { + /* 0 */ 1, INVALID, +}; + +static const uint16_t ud_itab__291[] = { + /* 0 */ 4, INVALID, +}; + +static const uint16_t ud_itab__292[] = { + /* 0 */ GROUP(293), GROUP(294), INVALID, +}; + +static const uint16_t ud_itab__293[] = { + /* 0 */ 1257, INVALID, +}; + +static const uint16_t ud_itab__294[] = { + /* 0 */ 1258, INVALID, +}; + +static const uint16_t ud_itab__295[] = { + /* 0 */ GROUP(296), GROUP(297), INVALID, +}; + +static const uint16_t ud_itab__296[] = { + /* 0 */ 1110, INVALID, +}; + +static const uint16_t ud_itab__297[] = { + /* 0 */ 1111, INVALID, +}; + +static const uint16_t ud_itab__298[] = { + /* 0 */ 1658, INVALID, +}; + +static const uint16_t ud_itab__299[] = { + /* 0 */ 67, 68, +}; + +static const uint16_t ud_itab__300[] = { + /* 0 */ 710, 711, INVALID, +}; + +static const uint16_t ud_itab__301[] = { + /* 0 */ 983, 984, INVALID, +}; + +static const uint16_t ud_itab__302[] = { + /* 0 */ 21, 970, 11, 1342, + /* 4 */ 55, 1413, 1493, 106, +}; + +static const uint16_t ud_itab__303[] = { + /* 0 */ 23, 971, 13, 1343, + /* 4 */ 57, 1414, 1494, 108, +}; + +static const uint16_t ud_itab__304[] = { + /* 0 */ GROUP(305), GROUP(306), GROUP(307), GROUP(308), + /* 4 */ GROUP(309), GROUP(310), GROUP(311), GROUP(312), +}; + +static const uint16_t ud_itab__305[] = { + /* 0 */ 22, INVALID, +}; + +static const uint16_t ud_itab__306[] = { + /* 0 */ 972, INVALID, +}; + +static const uint16_t ud_itab__307[] = { + /* 0 */ 12, INVALID, +}; + +static const uint16_t ud_itab__308[] = { + /* 0 */ 1344, INVALID, +}; + +static const uint16_t ud_itab__309[] = { + /* 0 */ 56, INVALID, +}; + +static const uint16_t ud_itab__310[] = { + /* 0 */ 1415, INVALID, +}; + +static const uint16_t ud_itab__311[] = { + /* 0 */ 1495, INVALID, +}; + +static const uint16_t ud_itab__312[] = { + /* 0 */ 107, INVALID, +}; + +static const uint16_t ud_itab__313[] = { + /* 0 */ 24, 973, 14, 1345, + /* 4 */ 58, 1416, 1496, 109, +}; + +static const uint16_t ud_itab__314[] = { + /* 0 */ 1109, INVALID, INVALID, INVALID, + /* 4 */ INVALID, INVALID, INVALID, INVALID, +}; + +static const uint16_t ud_itab__315[] = { + /* 0 */ 74, 75, 76, +}; + +static const uint16_t ud_itab__316[] = { + /* 0 */ 170, 171, 172, +}; + +static const uint16_t ud_itab__317[] = { + /* 0 */ 73, INVALID, +}; + +static const uint16_t ud_itab__318[] = { + /* 0 */ GROUP(319), GROUP(320), GROUP(321), +}; + +static const uint16_t ud_itab__319[] = { + /* 0 */ 1259, 1260, +}; + +static const uint16_t ud_itab__320[] = { + /* 0 */ 1261, 1262, +}; + +static const uint16_t ud_itab__321[] = { + /* 0 */ INVALID, 1263, +}; + +static const uint16_t ud_itab__322[] = { + /* 0 */ GROUP(323), GROUP(324), GROUP(325), +}; + +static const uint16_t ud_itab__323[] = { + /* 0 */ 1112, INVALID, +}; + +static const uint16_t ud_itab__324[] = { + /* 0 */ 1113, 1114, +}; + +static const uint16_t ud_itab__325[] = { + /* 0 */ INVALID, 1115, +}; + +static const uint16_t ud_itab__326[] = { + /* 0 */ 923, 924, 927, +}; + +static const uint16_t ud_itab__327[] = { + /* 0 */ 115, 116, 119, +}; + +static const uint16_t ud_itab__328[] = { + /* 0 */ 1403, 1404, 1405, +}; + +static const uint16_t ud_itab__329[] = { + /* 0 */ 791, 792, 793, +}; + +static const uint16_t ud_itab__330[] = { + /* 0 */ 1347, 1348, 1349, +}; + +static const uint16_t ud_itab__331[] = { + /* 0 */ 1279, 1286, 1267, 1275, + /* 4 */ 1327, 1334, 1318, 1313, +}; + +static const uint16_t ud_itab__332[] = { + /* 0 */ 1284, 1287, 1268, 1274, + /* 4 */ 1323, 1330, 1319, 1315, +}; + +static const uint16_t ud_itab__333[] = { + /* 0 */ GROUP(334), GROUP(335), INVALID, INVALID, + /* 4 */ INVALID, GROUP(341), GROUP(357), GROUP(369), + /* 8 */ INVALID, GROUP(394), INVALID, INVALID, + /* c */ INVALID, GROUP(399), INVALID, INVALID, +}; + +static const uint16_t ud_itab__334[] = { + /* 0 */ 771, INVALID, +}; + +static const uint16_t ud_itab__335[] = { + /* 0 */ INVALID, INVALID, INVALID, INVALID, + /* 4 */ INVALID, INVALID, INVALID, INVALID, + /* 8 */ INVALID, INVALID, INVALID, INVALID, + /* c */ INVALID, INVALID, INVALID, INVALID, + /* 10 */ 937, 939, GROUP(336), 895, + /* 14 */ 1450, 1448, GROUP(337), 885, + /* 18 */ INVALID, INVALID, INVALID, INVALID, + /* 1c */ INVALID, INVALID, INVALID, INVALID, + /* 20 */ INVALID, INVALID, INVALID, INVALID, + /* 24 */ INVALID, INVALID, INVALID, INVALID, + /* 28 */ 863, 865, INVALID, 908, + /* 2c */ INVALID, INVALID, 1443, 130, + /* 30 */ INVALID, INVALID, INVALID, INVALID, + /* 34 */ INVALID, INVALID, INVALID, INVALID, + /* 38 */ INVALID, INVALID, INVALID, INVALID, + /* 3c */ INVALID, INVALID, INVALID, INVALID, + /* 40 */ INVALID, INVALID, INVALID, INVALID, + /* 44 */ INVALID, INVALID, INVALID, INVALID, + /* 48 */ INVALID, INVALID, INVALID, INVALID, + /* 4c */ INVALID, INVALID, INVALID, INVALID, + /* 50 */ 901, 1388, 1307, 1292, + /* 54 */ 62, 66, 977, 1500, + /* 58 */ 28, 947, 146, 135, + /* 5c */ 1420, 819, 190, 803, + /* 60 */ INVALID, INVALID, INVALID, INVALID, + /* 64 */ INVALID, INVALID, INVALID, INVALID, + /* 68 */ INVALID, INVALID, INVALID, INVALID, + /* 6c */ INVALID, INVALID, INVALID, INVALID, + /* 70 */ INVALID, INVALID, INVALID, INVALID, + /* 74 */ INVALID, INVALID, INVALID, GROUP(340), + /* 78 */ INVALID, INVALID, INVALID, INVALID, + /* 7c */ INVALID, INVALID, INVALID, INVALID, + /* 80 */ INVALID, INVALID, INVALID, INVALID, + /* 84 */ INVALID, INVALID, INVALID, INVALID, + /* 88 */ INVALID, INVALID, INVALID, INVALID, + /* 8c */ INVALID, INVALID, INVALID, INVALID, + /* 90 */ INVALID, INVALID, INVALID, INVALID, + /* 94 */ INVALID, INVALID, INVALID, INVALID, + /* 98 */ INVALID, INVALID, INVALID, INVALID, + /* 9c */ INVALID, INVALID, INVALID, INVALID, + /* a0 */ INVALID, INVALID, INVALID, INVALID, + /* a4 */ INVALID, INVALID, INVALID, INVALID, + /* a8 */ INVALID, INVALID, INVALID, INVALID, + /* ac */ INVALID, INVALID, GROUP(338), INVALID, + /* b0 */ INVALID, INVALID, INVALID, INVALID, + /* b4 */ INVALID, INVALID, INVALID, INVALID, + /* b8 */ INVALID, INVALID, INVALID, INVALID, + /* bc */ INVALID, INVALID, INVALID, INVALID, + /* c0 */ INVALID, INVALID, 113, INVALID, + /* c4 */ INVALID, INVALID, 1382, INVALID, + /* c8 */ INVALID, INVALID, INVALID, INVALID, + /* cc */ INVALID, INVALID, INVALID, INVALID, + /* d0 */ INVALID, INVALID, INVALID, INVALID, + /* d4 */ INVALID, INVALID, INVALID, INVALID, + /* d8 */ INVALID, INVALID, INVALID, INVALID, + /* dc */ INVALID, INVALID, INVALID, INVALID, + /* e0 */ INVALID, INVALID, INVALID, INVALID, + /* e4 */ INVALID, INVALID, INVALID, INVALID, + /* e8 */ INVALID, INVALID, INVALID, INVALID, + /* ec */ INVALID, INVALID, INVALID, INVALID, + /* f0 */ INVALID, INVALID, INVALID, INVALID, + /* f4 */ INVALID, INVALID, INVALID, INVALID, + /* f8 */ INVALID, INVALID, INVALID, INVALID, + /* fc */ INVALID, INVALID, INVALID, INVALID, +}; + +static const uint16_t ud_itab__336[] = { + /* 0 */ 893, 897, +}; + +static const uint16_t ud_itab__337[] = { + /* 0 */ 883, 887, +}; + +static const uint16_t ud_itab__338[] = { + /* 0 */ GROUP(339), INVALID, +}; + +static const uint16_t ud_itab__339[] = { + /* 0 */ INVALID, INVALID, INVALID, 1401, + /* 4 */ INVALID, INVALID, INVALID, INVALID, +}; + +static const uint16_t ud_itab__340[] = { + /* 0 */ 1742, 1743, +}; + +static const uint16_t ud_itab__341[] = { + /* 0 */ INVALID, INVALID, INVALID, INVALID, + /* 4 */ INVALID, INVALID, INVALID, INVALID, + /* 8 */ INVALID, INVALID, INVALID, INVALID, + /* c */ INVALID, INVALID, INVALID, INVALID, + /* 10 */ 933, 935, GROUP(342), 891, + /* 14 */ 1452, 1446, GROUP(343), 881, + /* 18 */ INVALID, INVALID, INVALID, INVALID, + /* 1c */ INVALID, INVALID, INVALID, INVALID, + /* 20 */ INVALID, INVALID, INVALID, INVALID, + /* 24 */ INVALID, INVALID, INVALID, INVALID, + /* 28 */ 859, 861, INVALID, 906, + /* 2c */ INVALID, INVALID, 1441, 128, + /* 30 */ INVALID, INVALID, INVALID, INVALID, + /* 34 */ INVALID, INVALID, INVALID, INVALID, + /* 38 */ INVALID, INVALID, INVALID, INVALID, + /* 3c */ INVALID, INVALID, INVALID, INVALID, + /* 40 */ INVALID, INVALID, INVALID, INVALID, + /* 44 */ INVALID, INVALID, INVALID, INVALID, + /* 48 */ INVALID, INVALID, INVALID, INVALID, + /* 4c */ INVALID, INVALID, INVALID, INVALID, + /* 50 */ 899, 1390, INVALID, INVALID, + /* 54 */ 60, 64, 975, 1498, + /* 58 */ 26, 945, 140, 144, + /* 5c */ 1418, 817, 188, 801, + /* 60 */ 1208, 1211, 1214, 986, + /* 64 */ 1037, 1040, 1043, 992, + /* 68 */ 1199, 1202, 1205, 989, + /* 6c */ 1548, 1546, GROUP(344), 1518, + /* 70 */ 1540, GROUP(345), GROUP(347), GROUP(349), + /* 74 */ 1029, 1032, 1035, INVALID, + /* 78 */ INVALID, INVALID, INVALID, INVALID, + /* 7c */ 1550, 1554, GROUP(351), 1516, + /* 80 */ INVALID, INVALID, INVALID, INVALID, + /* 84 */ INVALID, INVALID, INVALID, INVALID, + /* 88 */ INVALID, INVALID, INVALID, INVALID, + /* 8c */ INVALID, INVALID, INVALID, INVALID, + /* 90 */ INVALID, INVALID, INVALID, INVALID, + /* 94 */ INVALID, INVALID, INVALID, INVALID, + /* 98 */ INVALID, INVALID, INVALID, INVALID, + /* 9c */ INVALID, INVALID, INVALID, INVALID, + /* a0 */ INVALID, INVALID, INVALID, INVALID, + /* a4 */ INVALID, INVALID, INVALID, INVALID, + /* a8 */ INVALID, INVALID, INVALID, INVALID, + /* ac */ INVALID, INVALID, INVALID, INVALID, + /* b0 */ INVALID, INVALID, INVALID, INVALID, + /* b4 */ INVALID, INVALID, INVALID, INVALID, + /* b8 */ INVALID, INVALID, INVALID, INVALID, + /* bc */ INVALID, INVALID, INVALID, INVALID, + /* c0 */ INVALID, INVALID, 111, INVALID, + /* c4 */ 1061, 1054, 1380, INVALID, + /* c8 */ INVALID, INVALID, INVALID, INVALID, + /* cc */ INVALID, INVALID, INVALID, INVALID, + /* d0 */ 34, 1162, 1168, 1174, + /* d4 */ 1529, 1095, 919, GROUP(352), + /* d8 */ 1194, 1197, 1082, 1016, + /* dc */ 1011, 1014, 1077, 1019, + /* e0 */ 1022, 1149, 1155, 1025, + /* e4 */ 1089, 1091, 161, 903, + /* e8 */ 1188, 1191, 1079, 1117, + /* ec */ 1005, 1008, 1073, 1265, + /* f0 */ INVALID, GROUP(353), GROUP(354), GROUP(355), + /* f4 */ INVALID, 1071, 1132, GROUP(356), + /* f8 */ 1178, 1181, 1185, 1531, + /* fc */ 995, 999, 1002, INVALID, +}; + +static const uint16_t ud_itab__342[] = { + /* 0 */ 889, INVALID, +}; + +static const uint16_t ud_itab__343[] = { + /* 0 */ 879, INVALID, +}; + +static const uint16_t ud_itab__344[] = { + /* 0 */ 869, 871, 912, +}; + +static const uint16_t ud_itab__345[] = { + /* 0 */ INVALID, INVALID, 1164, INVALID, + /* 4 */ 1151, INVALID, GROUP(346), INVALID, +}; + +static const uint16_t ud_itab__346[] = { + /* 0 */ 1756, INVALID, +}; + +static const uint16_t ud_itab__347[] = { + /* 0 */ INVALID, INVALID, 1170, INVALID, + /* 4 */ 1158, INVALID, GROUP(348), INVALID, +}; + +static const uint16_t ud_itab__348[] = { + /* 0 */ 1758, INVALID, +}; + +static const uint16_t ud_itab__349[] = { + /* 0 */ INVALID, INVALID, 1176, 1544, + /* 4 */ INVALID, INVALID, GROUP(350), 1542, +}; + +static const uint16_t ud_itab__350[] = { + /* 0 */ 1760, INVALID, +}; + +static const uint16_t ud_itab__351[] = { + /* 0 */ 875, 877, 915, +}; + +static const uint16_t ud_itab__352[] = { + /* 0 */ 1085, INVALID, +}; + +static const uint16_t ud_itab__353[] = { + /* 0 */ 1755, INVALID, +}; + +static const uint16_t ud_itab__354[] = { + /* 0 */ 1757, INVALID, +}; + +static const uint16_t ud_itab__355[] = { + /* 0 */ 1759, INVALID, +}; + +static const uint16_t ud_itab__356[] = { + /* 0 */ INVALID, 1520, +}; + +static const uint16_t ud_itab__357[] = { + /* 0 */ 1584, 1587, 1590, 1593, + /* 4 */ 1596, 1599, 1602, 1605, + /* 8 */ 1608, 1614, 1611, 1617, + /* c */ GROUP(358), GROUP(359), GROUP(360), GROUP(361), + /* 10 */ INVALID, INVALID, INVALID, INVALID, + /* 14 */ INVALID, INVALID, INVALID, 1712, + /* 18 */ GROUP(362), GROUP(363), INVALID, INVALID, + /* 1c */ 1575, 1578, 1581, INVALID, + /* 20 */ 1686, 1688, 1690, 1692, + /* 24 */ 1694, INVALID, INVALID, INVALID, + /* 28 */ 1623, 1709, 1682, 1684, + /* 2c */ GROUP(365), GROUP(366), GROUP(367), GROUP(368), + /* 30 */ 1697, 1699, 1701, 1703, + /* 34 */ 1705, 1707, INVALID, 1718, + /* 38 */ 1625, 1627, 1629, 1631, + /* 3c */ 1633, 1635, 1639, 1637, + /* 40 */ 1641, 1643, INVALID, INVALID, + /* 44 */ INVALID, INVALID, INVALID, INVALID, + /* 48 */ INVALID, INVALID, INVALID, INVALID, + /* 4c */ INVALID, INVALID, INVALID, INVALID, + /* 50 */ INVALID, INVALID, INVALID, INVALID, + /* 54 */ INVALID, INVALID, INVALID, INVALID, + /* 58 */ INVALID, INVALID, INVALID, INVALID, + /* 5c */ INVALID, INVALID, INVALID, INVALID, + /* 60 */ INVALID, INVALID, INVALID, INVALID, + /* 64 */ INVALID, INVALID, INVALID, INVALID, + /* 68 */ INVALID, INVALID, INVALID, INVALID, + /* 6c */ INVALID, INVALID, INVALID, INVALID, + /* 70 */ INVALID, INVALID, INVALID, INVALID, + /* 74 */ INVALID, INVALID, INVALID, INVALID, + /* 78 */ INVALID, INVALID, INVALID, INVALID, + /* 7c */ INVALID, INVALID, INVALID, INVALID, + /* 80 */ INVALID, INVALID, INVALID, INVALID, + /* 84 */ INVALID, INVALID, INVALID, INVALID, + /* 88 */ INVALID, INVALID, INVALID, INVALID, + /* 8c */ INVALID, INVALID, INVALID, INVALID, + /* 90 */ INVALID, INVALID, INVALID, INVALID, + /* 94 */ INVALID, INVALID, INVALID, INVALID, + /* 98 */ INVALID, INVALID, INVALID, INVALID, + /* 9c */ INVALID, INVALID, INVALID, INVALID, + /* a0 */ INVALID, INVALID, INVALID, INVALID, + /* a4 */ INVALID, INVALID, INVALID, INVALID, + /* a8 */ INVALID, INVALID, INVALID, INVALID, + /* ac */ INVALID, INVALID, INVALID, INVALID, + /* b0 */ INVALID, INVALID, INVALID, INVALID, + /* b4 */ INVALID, INVALID, INVALID, INVALID, + /* b8 */ INVALID, INVALID, INVALID, INVALID, + /* bc */ INVALID, INVALID, INVALID, INVALID, + /* c0 */ INVALID, INVALID, INVALID, INVALID, + /* c4 */ INVALID, INVALID, INVALID, INVALID, + /* c8 */ INVALID, INVALID, INVALID, INVALID, + /* cc */ INVALID, INVALID, INVALID, INVALID, + /* d0 */ INVALID, INVALID, INVALID, INVALID, + /* d4 */ INVALID, INVALID, INVALID, INVALID, + /* d8 */ INVALID, INVALID, INVALID, 46, + /* dc */ 42, 44, 38, 40, + /* e0 */ INVALID, INVALID, INVALID, INVALID, + /* e4 */ INVALID, INVALID, INVALID, INVALID, + /* e8 */ INVALID, INVALID, INVALID, INVALID, + /* ec */ INVALID, INVALID, INVALID, INVALID, + /* f0 */ INVALID, INVALID, INVALID, INVALID, + /* f4 */ INVALID, INVALID, INVALID, INVALID, + /* f8 */ INVALID, INVALID, INVALID, INVALID, + /* fc */ INVALID, INVALID, INVALID, INVALID, +}; + +static const uint16_t ud_itab__358[] = { + /* 0 */ 1737, INVALID, +}; + +static const uint16_t ud_itab__359[] = { + /* 0 */ 1735, INVALID, +}; + +static const uint16_t ud_itab__360[] = { + /* 0 */ 1740, INVALID, +}; + +static const uint16_t ud_itab__361[] = { + /* 0 */ 1741, INVALID, +}; + +static const uint16_t ud_itab__362[] = { + /* 0 */ 1727, INVALID, +}; + +static const uint16_t ud_itab__363[] = { + /* 0 */ GROUP(364), INVALID, +}; + +static const uint16_t ud_itab__364[] = { + /* 0 */ INVALID, 1728, +}; + +static const uint16_t ud_itab__365[] = { + /* 0 */ 1731, INVALID, +}; + +static const uint16_t ud_itab__366[] = { + /* 0 */ 1733, INVALID, +}; + +static const uint16_t ud_itab__367[] = { + /* 0 */ 1732, INVALID, +}; + +static const uint16_t ud_itab__368[] = { + /* 0 */ 1734, INVALID, +}; + +static const uint16_t ud_itab__369[] = { + /* 0 */ INVALID, INVALID, INVALID, INVALID, + /* 4 */ GROUP(370), GROUP(371), GROUP(372), INVALID, + /* 8 */ 1645, 1647, 1649, 1651, + /* c */ 1655, 1653, 1678, 1620, + /* 10 */ INVALID, INVALID, INVALID, INVALID, + /* 14 */ GROUP(374), 1057, GROUP(375), 202, + /* 18 */ GROUP(379), GROUP(381), INVALID, INVALID, + /* 1c */ INVALID, INVALID, INVALID, INVALID, + /* 20 */ GROUP(383), 1558, GROUP(385), INVALID, + /* 24 */ INVALID, INVALID, INVALID, INVALID, + /* 28 */ INVALID, INVALID, INVALID, INVALID, + /* 2c */ INVALID, INVALID, INVALID, INVALID, + /* 30 */ INVALID, INVALID, INVALID, INVALID, + /* 34 */ INVALID, INVALID, INVALID, INVALID, + /* 38 */ INVALID, INVALID, INVALID, INVALID, + /* 3c */ INVALID, INVALID, INVALID, INVALID, + /* 40 */ 198, 196, 1680, INVALID, + /* 44 */ 1513, INVALID, INVALID, INVALID, + /* 48 */ INVALID, INVALID, GROUP(391), GROUP(392), + /* 4c */ GROUP(393), INVALID, INVALID, INVALID, + /* 50 */ INVALID, INVALID, INVALID, INVALID, + /* 54 */ INVALID, INVALID, INVALID, INVALID, + /* 58 */ INVALID, INVALID, INVALID, INVALID, + /* 5c */ INVALID, INVALID, INVALID, INVALID, + /* 60 */ 1716, 1714, 1722, 1720, + /* 64 */ INVALID, INVALID, INVALID, INVALID, + /* 68 */ INVALID, INVALID, INVALID, INVALID, + /* 6c */ INVALID, INVALID, INVALID, INVALID, + /* 70 */ INVALID, INVALID, INVALID, INVALID, + /* 74 */ INVALID, INVALID, INVALID, INVALID, + /* 78 */ INVALID, INVALID, INVALID, INVALID, + /* 7c */ INVALID, INVALID, INVALID, INVALID, + /* 80 */ INVALID, INVALID, INVALID, INVALID, + /* 84 */ INVALID, INVALID, INVALID, INVALID, + /* 88 */ INVALID, INVALID, INVALID, INVALID, + /* 8c */ INVALID, INVALID, INVALID, INVALID, + /* 90 */ INVALID, INVALID, INVALID, INVALID, + /* 94 */ INVALID, INVALID, INVALID, INVALID, + /* 98 */ INVALID, INVALID, INVALID, INVALID, + /* 9c */ INVALID, INVALID, INVALID, INVALID, + /* a0 */ INVALID, INVALID, INVALID, INVALID, + /* a4 */ INVALID, INVALID, INVALID, INVALID, + /* a8 */ INVALID, INVALID, INVALID, INVALID, + /* ac */ INVALID, INVALID, INVALID, INVALID, + /* b0 */ INVALID, INVALID, INVALID, INVALID, + /* b4 */ INVALID, INVALID, INVALID, INVALID, + /* b8 */ INVALID, INVALID, INVALID, INVALID, + /* bc */ INVALID, INVALID, INVALID, INVALID, + /* c0 */ INVALID, INVALID, INVALID, INVALID, + /* c4 */ INVALID, INVALID, INVALID, INVALID, + /* c8 */ INVALID, INVALID, INVALID, INVALID, + /* cc */ INVALID, INVALID, INVALID, INVALID, + /* d0 */ INVALID, INVALID, INVALID, INVALID, + /* d4 */ INVALID, INVALID, INVALID, INVALID, + /* d8 */ INVALID, INVALID, INVALID, INVALID, + /* dc */ INVALID, INVALID, INVALID, 48, + /* e0 */ INVALID, INVALID, INVALID, INVALID, + /* e4 */ INVALID, INVALID, INVALID, INVALID, + /* e8 */ INVALID, INVALID, INVALID, INVALID, + /* ec */ INVALID, INVALID, INVALID, INVALID, + /* f0 */ INVALID, INVALID, INVALID, INVALID, + /* f4 */ INVALID, INVALID, INVALID, INVALID, + /* f8 */ INVALID, INVALID, INVALID, INVALID, + /* fc */ INVALID, INVALID, INVALID, INVALID, +}; + +static const uint16_t ud_itab__370[] = { + /* 0 */ 1738, INVALID, +}; + +static const uint16_t ud_itab__371[] = { + /* 0 */ 1736, INVALID, +}; + +static const uint16_t ud_itab__372[] = { + /* 0 */ GROUP(373), INVALID, +}; + +static const uint16_t ud_itab__373[] = { + /* 0 */ INVALID, 1739, +}; + +static const uint16_t ud_itab__374[] = { + /* 0 */ 1046, INVALID, +}; + +static const uint16_t ud_itab__375[] = { + /* 0 */ GROUP(376), GROUP(377), GROUP(378), +}; + +static const uint16_t ud_itab__376[] = { + /* 0 */ 1048, INVALID, +}; + +static const uint16_t ud_itab__377[] = { + /* 0 */ 1050, INVALID, +}; + +static const uint16_t ud_itab__378[] = { + /* 0 */ INVALID, 1052, +}; + +static const uint16_t ud_itab__379[] = { + /* 0 */ GROUP(380), INVALID, +}; + +static const uint16_t ud_itab__380[] = { + /* 0 */ INVALID, 1730, +}; + +static const uint16_t ud_itab__381[] = { + /* 0 */ GROUP(382), INVALID, +}; + +static const uint16_t ud_itab__382[] = { + /* 0 */ INVALID, 1729, +}; + +static const uint16_t ud_itab__383[] = { + /* 0 */ GROUP(384), INVALID, +}; + +static const uint16_t ud_itab__384[] = { + /* 0 */ 1065, INVALID, +}; + +static const uint16_t ud_itab__385[] = { + /* 0 */ GROUP(386), GROUP(388), +}; + +static const uint16_t ud_itab__386[] = { + /* 0 */ GROUP(387), INVALID, +}; + +static const uint16_t ud_itab__387[] = { + /* 0 */ 1066, INVALID, +}; + +static const uint16_t ud_itab__388[] = { + /* 0 */ GROUP(389), GROUP(390), +}; + +static const uint16_t ud_itab__389[] = { + /* 0 */ 1067, INVALID, +}; + +static const uint16_t ud_itab__390[] = { + /* 0 */ 1068, INVALID, +}; + +static const uint16_t ud_itab__391[] = { + /* 0 */ 1745, INVALID, +}; + +static const uint16_t ud_itab__392[] = { + /* 0 */ 1744, INVALID, +}; + +static const uint16_t ud_itab__393[] = { + /* 0 */ 1754, INVALID, +}; + +static const uint16_t ud_itab__394[] = { + /* 0 */ INVALID, INVALID, INVALID, INVALID, + /* 4 */ INVALID, INVALID, INVALID, INVALID, + /* 8 */ INVALID, INVALID, INVALID, INVALID, + /* c */ INVALID, INVALID, INVALID, INVALID, + /* 10 */ GROUP(395), GROUP(396), GROUP(397), INVALID, + /* 14 */ INVALID, INVALID, GROUP(398), INVALID, + /* 18 */ INVALID, INVALID, INVALID, INVALID, + /* 1c */ INVALID, INVALID, INVALID, INVALID, + /* 20 */ INVALID, INVALID, INVALID, INVALID, + /* 24 */ INVALID, INVALID, INVALID, INVALID, + /* 28 */ INVALID, INVALID, 155, INVALID, + /* 2c */ 169, 159, INVALID, INVALID, + /* 30 */ INVALID, INVALID, INVALID, INVALID, + /* 34 */ INVALID, INVALID, INVALID, INVALID, + /* 38 */ INVALID, INVALID, INVALID, INVALID, + /* 3c */ INVALID, INVALID, INVALID, INVALID, + /* 40 */ INVALID, INVALID, INVALID, INVALID, + /* 44 */ INVALID, INVALID, INVALID, INVALID, + /* 48 */ INVALID, INVALID, INVALID, INVALID, + /* 4c */ INVALID, INVALID, INVALID, INVALID, + /* 50 */ INVALID, 1394, 1309, 1294, + /* 54 */ INVALID, INVALID, INVALID, INVALID, + /* 58 */ 32, 951, 157, 164, + /* 5c */ 1424, 823, 194, 807, + /* 60 */ INVALID, INVALID, INVALID, INVALID, + /* 64 */ INVALID, INVALID, INVALID, INVALID, + /* 68 */ INVALID, INVALID, INVALID, INVALID, + /* 6c */ INVALID, INVALID, INVALID, 1523, + /* 70 */ 1536, INVALID, INVALID, INVALID, + /* 74 */ INVALID, INVALID, INVALID, INVALID, + /* 78 */ INVALID, INVALID, INVALID, INVALID, + /* 7c */ INVALID, INVALID, 917, 1525, + /* 80 */ INVALID, INVALID, INVALID, INVALID, + /* 84 */ INVALID, INVALID, INVALID, INVALID, + /* 88 */ INVALID, INVALID, INVALID, INVALID, + /* 8c */ INVALID, INVALID, INVALID, INVALID, + /* 90 */ INVALID, INVALID, INVALID, INVALID, + /* 94 */ INVALID, INVALID, INVALID, INVALID, + /* 98 */ INVALID, INVALID, INVALID, INVALID, + /* 9c */ INVALID, INVALID, INVALID, INVALID, + /* a0 */ INVALID, INVALID, INVALID, INVALID, + /* a4 */ INVALID, INVALID, INVALID, INVALID, + /* a8 */ INVALID, INVALID, INVALID, INVALID, + /* ac */ INVALID, INVALID, INVALID, INVALID, + /* b0 */ INVALID, INVALID, INVALID, INVALID, + /* b4 */ INVALID, INVALID, INVALID, INVALID, + /* b8 */ INVALID, INVALID, INVALID, INVALID, + /* bc */ INVALID, INVALID, INVALID, INVALID, + /* c0 */ INVALID, INVALID, 121, INVALID, + /* c4 */ INVALID, INVALID, INVALID, INVALID, + /* c8 */ INVALID, INVALID, INVALID, INVALID, + /* cc */ INVALID, INVALID, INVALID, INVALID, + /* d0 */ INVALID, INVALID, INVALID, INVALID, + /* d4 */ INVALID, INVALID, INVALID, INVALID, + /* d8 */ INVALID, INVALID, INVALID, INVALID, + /* dc */ INVALID, INVALID, INVALID, INVALID, + /* e0 */ INVALID, INVALID, INVALID, INVALID, + /* e4 */ INVALID, INVALID, 133, INVALID, + /* e8 */ INVALID, INVALID, INVALID, INVALID, + /* ec */ INVALID, INVALID, INVALID, INVALID, + /* f0 */ INVALID, INVALID, INVALID, INVALID, + /* f4 */ INVALID, INVALID, INVALID, INVALID, + /* f8 */ INVALID, INVALID, INVALID, INVALID, + /* fc */ INVALID, INVALID, INVALID, INVALID, +}; + +static const uint16_t ud_itab__395[] = { + /* 0 */ 1751, 1750, +}; + +static const uint16_t ud_itab__396[] = { + /* 0 */ 1753, 1752, +}; + +static const uint16_t ud_itab__397[] = { + /* 0 */ 1572, 1570, +}; + +static const uint16_t ud_itab__398[] = { + /* 0 */ 1568, 1566, +}; + +static const uint16_t ud_itab__399[] = { + /* 0 */ INVALID, INVALID, INVALID, INVALID, + /* 4 */ INVALID, INVALID, INVALID, INVALID, + /* 8 */ INVALID, INVALID, INVALID, INVALID, + /* c */ INVALID, INVALID, INVALID, INVALID, + /* 10 */ GROUP(402), GROUP(400), GROUP(401), INVALID, + /* 14 */ INVALID, INVALID, INVALID, INVALID, + /* 18 */ INVALID, INVALID, INVALID, INVALID, + /* 1c */ INVALID, INVALID, INVALID, INVALID, + /* 20 */ INVALID, INVALID, INVALID, INVALID, + /* 24 */ INVALID, INVALID, INVALID, INVALID, + /* 28 */ INVALID, INVALID, 153, INVALID, + /* 2c */ 167, 149, INVALID, INVALID, + /* 30 */ INVALID, INVALID, INVALID, INVALID, + /* 34 */ INVALID, INVALID, INVALID, INVALID, + /* 38 */ INVALID, INVALID, INVALID, INVALID, + /* 3c */ INVALID, INVALID, INVALID, INVALID, + /* 40 */ INVALID, INVALID, INVALID, INVALID, + /* 44 */ INVALID, INVALID, INVALID, INVALID, + /* 48 */ INVALID, INVALID, INVALID, INVALID, + /* 4c */ INVALID, INVALID, INVALID, INVALID, + /* 50 */ INVALID, 1392, INVALID, INVALID, + /* 54 */ INVALID, INVALID, INVALID, INVALID, + /* 58 */ 30, 949, 151, INVALID, + /* 5c */ 1422, 821, 192, 805, + /* 60 */ INVALID, INVALID, INVALID, INVALID, + /* 64 */ INVALID, INVALID, INVALID, INVALID, + /* 68 */ INVALID, INVALID, INVALID, INVALID, + /* 6c */ INVALID, INVALID, INVALID, INVALID, + /* 70 */ 1538, INVALID, INVALID, INVALID, + /* 74 */ INVALID, INVALID, INVALID, INVALID, + /* 78 */ INVALID, INVALID, INVALID, INVALID, + /* 7c */ 1552, 1556, INVALID, INVALID, + /* 80 */ INVALID, INVALID, INVALID, INVALID, + /* 84 */ INVALID, INVALID, INVALID, INVALID, + /* 88 */ INVALID, INVALID, INVALID, INVALID, + /* 8c */ INVALID, INVALID, INVALID, INVALID, + /* 90 */ INVALID, INVALID, INVALID, INVALID, + /* 94 */ INVALID, INVALID, INVALID, INVALID, + /* 98 */ INVALID, INVALID, INVALID, INVALID, + /* 9c */ INVALID, INVALID, INVALID, INVALID, + /* a0 */ INVALID, INVALID, INVALID, INVALID, + /* a4 */ INVALID, INVALID, INVALID, INVALID, + /* a8 */ INVALID, INVALID, INVALID, INVALID, + /* ac */ INVALID, INVALID, INVALID, INVALID, + /* b0 */ INVALID, INVALID, INVALID, INVALID, + /* b4 */ INVALID, INVALID, INVALID, INVALID, + /* b8 */ INVALID, INVALID, INVALID, INVALID, + /* bc */ INVALID, INVALID, INVALID, INVALID, + /* c0 */ INVALID, INVALID, 118, INVALID, + /* c4 */ INVALID, INVALID, INVALID, INVALID, + /* c8 */ INVALID, INVALID, INVALID, INVALID, + /* cc */ INVALID, INVALID, INVALID, INVALID, + /* d0 */ 36, INVALID, INVALID, INVALID, + /* d4 */ INVALID, INVALID, INVALID, INVALID, + /* d8 */ INVALID, INVALID, INVALID, INVALID, + /* dc */ INVALID, INVALID, INVALID, INVALID, + /* e0 */ INVALID, INVALID, INVALID, INVALID, + /* e4 */ INVALID, INVALID, 137, INVALID, + /* e8 */ INVALID, INVALID, INVALID, INVALID, + /* ec */ INVALID, INVALID, INVALID, INVALID, + /* f0 */ 1560, INVALID, INVALID, INVALID, + /* f4 */ INVALID, INVALID, INVALID, INVALID, + /* f8 */ INVALID, INVALID, INVALID, INVALID, + /* fc */ INVALID, INVALID, INVALID, INVALID, +}; + +static const uint16_t ud_itab__400[] = { + /* 0 */ 1749, 1748, +}; + +static const uint16_t ud_itab__401[] = { + /* 0 */ 1564, 1562, +}; + +static const uint16_t ud_itab__402[] = { + /* 0 */ 1747, 1746, +}; + +static const uint16_t ud_itab__403[] = { + /* 0 */ GROUP(404), GROUP(335), INVALID, INVALID, + /* 4 */ INVALID, GROUP(341), GROUP(357), GROUP(369), + /* 8 */ INVALID, GROUP(394), INVALID, INVALID, + /* c */ INVALID, GROUP(399), INVALID, INVALID, +}; + +static const uint16_t ud_itab__404[] = { + /* 0 */ 769, INVALID, +}; + +static const uint16_t ud_itab__405[] = { + /* 0 */ 826, INVALID, INVALID, INVALID, + /* 4 */ INVALID, INVALID, INVALID, INVALID, +}; + +static const uint16_t ud_itab__406[] = { + /* 0 */ 827, INVALID, INVALID, INVALID, + /* 4 */ INVALID, INVALID, INVALID, INVALID, +}; + +static const uint16_t ud_itab__407[] = { + /* 0 */ 715, INVALID, +}; + +static const uint16_t ud_itab__408[] = { + /* 0 */ 723, 724, 725, +}; + +static const uint16_t ud_itab__409[] = { + /* 0 */ 1280, 1285, 1269, 1273, + /* 4 */ 1326, 1333, 1320, 1314, +}; + +static const uint16_t ud_itab__410[] = { + /* 0 */ 1281, 1288, 1272, 1276, + /* 4 */ 1325, 1332, 1329, 1312, +}; + +static const uint16_t ud_itab__411[] = { + /* 0 */ 1282, 1289, 1270, 1277, + /* 4 */ 1324, 1331, 1321, 1316, +}; + +static const uint16_t ud_itab__412[] = { + /* 0 */ 1283, 1290, 1271, 1278, + /* 4 */ 1328, 1335, 1322, 1317, +}; + +static const uint16_t ud_itab__413[] = { + /* 0 */ 3, INVALID, +}; + +static const uint16_t ud_itab__414[] = { + /* 0 */ 2, INVALID, +}; + +static const uint16_t ud_itab__415[] = { + /* 0 */ 1311, INVALID, +}; + +static const uint16_t ud_itab__416[] = { + /* 0 */ GROUP(417), GROUP(418), +}; + +static const uint16_t ud_itab__417[] = { + /* 0 */ 206, 503, 307, 357, + /* 4 */ 587, 630, 387, 413, +}; + +static const uint16_t ud_itab__418[] = { + /* 0 */ 215, 216, 217, 218, + /* 4 */ 219, 220, 221, 222, + /* 8 */ 504, 505, 506, 507, + /* c */ 508, 509, 510, 511, + /* 10 */ 309, 310, 311, 312, + /* 14 */ 313, 314, 315, 316, + /* 18 */ 359, 360, 361, 362, + /* 1c */ 363, 364, 365, 366, + /* 20 */ 589, 590, 591, 592, + /* 24 */ 593, 594, 595, 596, + /* 28 */ 614, 615, 616, 617, + /* 2c */ 618, 619, 620, 621, + /* 30 */ 388, 389, 390, 391, + /* 34 */ 392, 393, 394, 395, + /* 38 */ 414, 415, 416, 417, + /* 3c */ 418, 419, 420, 421, +}; + +static const uint16_t ud_itab__419[] = { + /* 0 */ GROUP(420), GROUP(421), +}; + +static const uint16_t ud_itab__420[] = { + /* 0 */ 476, INVALID, 573, 540, + /* 4 */ 493, 492, 584, 583, +}; + +static const uint16_t ud_itab__421[] = { + /* 0 */ 477, 478, 479, 480, + /* 4 */ 481, 482, 483, 484, + /* 8 */ 658, 659, 660, 661, + /* c */ 662, 663, 664, 665, + /* 10 */ 522, INVALID, INVALID, INVALID, + /* 14 */ INVALID, INVALID, INVALID, INVALID, + /* 18 */ 549, 550, 551, 552, + /* 1c */ 553, 554, 555, 556, + /* 20 */ 233, 204, INVALID, INVALID, + /* 24 */ 639, 657, INVALID, INVALID, + /* 28 */ 485, 486, 487, 488, + /* 2c */ 489, 490, 491, INVALID, + /* 30 */ 203, 685, 529, 526, + /* 34 */ 684, 528, 377, 454, + /* 38 */ 527, 686, 537, 536, + /* 3c */ 530, 534, 535, 376, +}; + +static const uint16_t ud_itab__422[] = { + /* 0 */ GROUP(423), GROUP(424), +}; + +static const uint16_t ud_itab__423[] = { + /* 0 */ 456, 520, 448, 450, + /* 4 */ 462, 464, 460, 458, +}; + +static const uint16_t ud_itab__424[] = { + /* 0 */ 235, 236, 237, 238, + /* 4 */ 239, 240, 241, 242, + /* 8 */ 243, 244, 245, 246, + /* c */ 247, 248, 249, 250, + /* 10 */ 251, 252, 253, 254, + /* 14 */ 255, 256, 257, 258, + /* 18 */ 259, 260, 261, 262, + /* 1c */ 263, 264, 265, 266, + /* 20 */ INVALID, INVALID, INVALID, INVALID, + /* 24 */ INVALID, INVALID, INVALID, INVALID, + /* 28 */ INVALID, 656, INVALID, INVALID, + /* 2c */ INVALID, INVALID, INVALID, INVALID, + /* 30 */ INVALID, INVALID, INVALID, INVALID, + /* 34 */ INVALID, INVALID, INVALID, INVALID, + /* 38 */ INVALID, INVALID, INVALID, INVALID, + /* 3c */ INVALID, INVALID, INVALID, INVALID, +}; + +static const uint16_t ud_itab__425[] = { + /* 0 */ GROUP(426), GROUP(427), +}; + +static const uint16_t ud_itab__426[] = { + /* 0 */ 453, 471, 467, 470, + /* 4 */ INVALID, 474, INVALID, 538, +}; + +static const uint16_t ud_itab__427[] = { + /* 0 */ 267, 268, 269, 270, + /* 4 */ 271, 272, 273, 274, + /* 8 */ 275, 276, 277, 278, + /* c */ 279, 280, 281, 282, + /* 10 */ 283, 284, 285, 286, + /* 14 */ 287, 288, 289, 290, + /* 18 */ 291, 292, 293, 294, + /* 1c */ 295, 296, 297, 298, + /* 20 */ 524, 523, 234, 455, + /* 24 */ 525, 532, INVALID, INVALID, + /* 28 */ 299, 300, 301, 302, + /* 2c */ 303, 304, 305, 306, + /* 30 */ 333, 334, 335, 336, + /* 34 */ 337, 338, 339, 340, + /* 38 */ INVALID, INVALID, INVALID, INVALID, + /* 3c */ INVALID, INVALID, INVALID, INVALID, +}; + +static const uint16_t ud_itab__428[] = { + /* 0 */ GROUP(429), GROUP(430), +}; + +static const uint16_t ud_itab__429[] = { + /* 0 */ 205, 494, 308, 358, + /* 4 */ 588, 613, 378, 404, +}; + +static const uint16_t ud_itab__430[] = { + /* 0 */ 207, 208, 209, 210, + /* 4 */ 211, 212, 213, 214, + /* 8 */ 495, 496, 497, 498, + /* c */ 499, 500, 501, 502, + /* 10 */ 317, 318, 319, 320, + /* 14 */ 321, 322, 323, 324, + /* 18 */ 325, 326, 327, 328, + /* 1c */ 329, 330, 331, 332, + /* 20 */ 622, 623, 624, 625, + /* 24 */ 626, 627, 628, 629, + /* 28 */ 597, 598, 599, 600, + /* 2c */ 601, 602, 603, 604, + /* 30 */ 405, 406, 407, 408, + /* 34 */ 409, 410, 411, 412, + /* 38 */ 379, 380, 381, 382, + /* 3c */ 383, 384, 385, 386, +}; + +static const uint16_t ud_itab__431[] = { + /* 0 */ GROUP(432), GROUP(433), +}; + +static const uint16_t ud_itab__432[] = { + /* 0 */ 475, 472, 574, 539, + /* 4 */ 531, INVALID, 533, 585, +}; + +static const uint16_t ud_itab__433[] = { + /* 0 */ 431, 432, 433, 434, + /* 4 */ 435, 436, 437, 438, + /* 8 */ 666, 667, 668, 669, + /* c */ 670, 671, 672, 673, + /* 10 */ 575, 576, 577, 578, + /* 14 */ 579, 580, 581, 582, + /* 18 */ 541, 542, 543, 544, + /* 1c */ 545, 546, 547, 548, + /* 20 */ 640, 641, 642, 643, + /* 24 */ 644, 645, 646, 647, + /* 28 */ 648, 649, 650, 651, + /* 2c */ 652, 653, 654, 655, + /* 30 */ INVALID, INVALID, INVALID, INVALID, + /* 34 */ INVALID, INVALID, INVALID, INVALID, + /* 38 */ INVALID, INVALID, INVALID, INVALID, + /* 3c */ INVALID, INVALID, INVALID, INVALID, +}; + +static const uint16_t ud_itab__434[] = { + /* 0 */ GROUP(435), GROUP(436), +}; + +static const uint16_t ud_itab__435[] = { + /* 0 */ 457, 521, 447, 449, + /* 4 */ 463, 465, 461, 459, +}; + +static const uint16_t ud_itab__436[] = { + /* 0 */ 223, 224, 225, 226, + /* 4 */ 227, 228, 229, 230, + /* 8 */ 512, 513, 514, 515, + /* c */ 516, 517, 518, 519, + /* 10 */ 367, 368, 369, 370, + /* 14 */ 371, 372, 373, 374, + /* 18 */ INVALID, 375, INVALID, INVALID, + /* 1c */ INVALID, INVALID, INVALID, INVALID, + /* 20 */ 631, 632, 633, 634, + /* 24 */ 635, 636, 637, 638, + /* 28 */ 605, 606, 607, 608, + /* 2c */ 609, 610, 611, 612, + /* 30 */ 422, 423, 424, 425, + /* 34 */ 426, 427, 428, 429, + /* 38 */ 396, 397, 398, 399, + /* 3c */ 400, 401, 402, 403, +}; + +static const uint16_t ud_itab__437[] = { + /* 0 */ GROUP(438), GROUP(439), +}; + +static const uint16_t ud_itab__438[] = { + /* 0 */ 451, 473, 466, 468, + /* 4 */ 231, 452, 232, 469, +}; + +static const uint16_t ud_itab__439[] = { + /* 0 */ 439, 440, 441, 442, + /* 4 */ 443, 444, 445, 446, + /* 8 */ 674, 675, 676, 677, + /* c */ 678, 679, 680, 681, + /* 10 */ 557, 558, 559, 560, + /* 14 */ 561, 562, 563, 564, + /* 18 */ 565, 566, 567, 568, + /* 1c */ 569, 570, 571, 572, + /* 20 */ 586, INVALID, INVALID, INVALID, + /* 24 */ INVALID, INVALID, INVALID, INVALID, + /* 28 */ 341, 342, 343, 344, + /* 2c */ 345, 346, 347, 348, + /* 30 */ 349, 350, 351, 352, + /* 34 */ 353, 354, 355, 356, + /* 38 */ INVALID, INVALID, INVALID, INVALID, + /* 3c */ INVALID, INVALID, INVALID, INVALID, +}; + +static const uint16_t ud_itab__440[] = { + /* 0 */ 758, 759, 760, +}; + +static const uint16_t ud_itab__441[] = { + /* 0 */ 764, INVALID, +}; + +static const uint16_t ud_itab__442[] = { + /* 0 */ 1432, 1437, 962, 953, + /* 4 */ 942, 695, 186, 689, +}; + +static const uint16_t ud_itab__443[] = { + /* 0 */ 1438, 1439, 963, 954, + /* 4 */ 943, 696, 185, 688, +}; + +static const uint16_t ud_itab__444[] = { + /* 0 */ 708, 183, INVALID, INVALID, + /* 4 */ INVALID, INVALID, INVALID, INVALID, +}; + +static const uint16_t ud_itab__445[] = { + /* 0 */ 707, 184, GROUP(446), 71, + /* 4 */ 761, 762, 1255, INVALID, +}; + +static const uint16_t ud_itab__446[] = { + /* 0 */ 69, 70, +}; + + +struct ud_lookup_table_list_entry ud_lookup_table_list[] = { + /* 000 */ { ud_itab__0, UD_TAB__OPC_TABLE, "opctbl" }, + /* 001 */ { ud_itab__1, UD_TAB__OPC_MODE, "/m" }, + /* 002 */ { ud_itab__2, UD_TAB__OPC_MODE, "/m" }, + /* 003 */ { ud_itab__3, UD_TAB__OPC_MODE, "/m" }, + /* 004 */ { ud_itab__4, UD_TAB__OPC_TABLE, "opctbl" }, + /* 005 */ { ud_itab__5, UD_TAB__OPC_REG, "/reg" }, + /* 006 */ { ud_itab__6, UD_TAB__OPC_MOD, "/mod" }, + /* 007 */ { ud_itab__7, UD_TAB__OPC_REG, "/reg" }, + /* 008 */ { ud_itab__8, UD_TAB__OPC_REG, "/reg" }, + /* 009 */ { ud_itab__9, UD_TAB__OPC_RM, "/rm" }, + /* 010 */ { ud_itab__10, UD_TAB__OPC_VENDOR, "/vendor" }, + /* 011 */ { ud_itab__11, UD_TAB__OPC_VENDOR, "/vendor" }, + /* 012 */ { ud_itab__12, UD_TAB__OPC_VENDOR, "/vendor" }, + /* 013 */ { ud_itab__13, UD_TAB__OPC_VENDOR, "/vendor" }, + /* 014 */ { ud_itab__14, UD_TAB__OPC_RM, "/rm" }, + /* 015 */ { ud_itab__15, UD_TAB__OPC_RM, "/rm" }, + /* 016 */ { ud_itab__16, UD_TAB__OPC_RM, "/rm" }, + /* 017 */ { ud_itab__17, UD_TAB__OPC_VENDOR, "/vendor" }, + /* 018 */ { ud_itab__18, UD_TAB__OPC_VENDOR, "/vendor" }, + /* 019 */ { ud_itab__19, UD_TAB__OPC_VENDOR, "/vendor" }, + /* 020 */ { ud_itab__20, UD_TAB__OPC_VENDOR, "/vendor" }, + /* 021 */ { ud_itab__21, UD_TAB__OPC_VENDOR, "/vendor" }, + /* 022 */ { ud_itab__22, UD_TAB__OPC_VENDOR, "/vendor" }, + /* 023 */ { ud_itab__23, UD_TAB__OPC_VENDOR, "/vendor" }, + /* 024 */ { ud_itab__24, UD_TAB__OPC_VENDOR, "/vendor" }, + /* 025 */ { ud_itab__25, UD_TAB__OPC_RM, "/rm" }, + /* 026 */ { ud_itab__26, UD_TAB__OPC_VENDOR, "/vendor" }, + /* 027 */ { ud_itab__27, UD_TAB__OPC_REG, "/reg" }, + /* 028 */ { ud_itab__28, UD_TAB__OPC_3DNOW, "/3dnow" }, + /* 029 */ { ud_itab__29, UD_TAB__OPC_SSE, "/sse" }, + /* 030 */ { ud_itab__30, UD_TAB__OPC_SSE, "/sse" }, + /* 031 */ { ud_itab__31, UD_TAB__OPC_MOD, "/mod" }, + /* 032 */ { ud_itab__32, UD_TAB__OPC_SSE, "/sse" }, + /* 033 */ { ud_itab__33, UD_TAB__OPC_SSE, "/sse" }, + /* 034 */ { ud_itab__34, UD_TAB__OPC_SSE, "/sse" }, + /* 035 */ { ud_itab__35, UD_TAB__OPC_SSE, "/sse" }, + /* 036 */ { ud_itab__36, UD_TAB__OPC_SSE, "/sse" }, + /* 037 */ { ud_itab__37, UD_TAB__OPC_MOD, "/mod" }, + /* 038 */ { ud_itab__38, UD_TAB__OPC_SSE, "/sse" }, + /* 039 */ { ud_itab__39, UD_TAB__OPC_SSE, "/sse" }, + /* 040 */ { ud_itab__40, UD_TAB__OPC_SSE, "/sse" }, + /* 041 */ { ud_itab__41, UD_TAB__OPC_REG, "/reg" }, + /* 042 */ { ud_itab__42, UD_TAB__OPC_SSE, "/sse" }, + /* 043 */ { ud_itab__43, UD_TAB__OPC_SSE, "/sse" }, + /* 044 */ { ud_itab__44, UD_TAB__OPC_SSE, "/sse" }, + /* 045 */ { ud_itab__45, UD_TAB__OPC_SSE, "/sse" }, + /* 046 */ { ud_itab__46, UD_TAB__OPC_SSE, "/sse" }, + /* 047 */ { ud_itab__47, UD_TAB__OPC_SSE, "/sse" }, + /* 048 */ { ud_itab__48, UD_TAB__OPC_SSE, "/sse" }, + /* 049 */ { ud_itab__49, UD_TAB__OPC_SSE, "/sse" }, + /* 050 */ { ud_itab__50, UD_TAB__OPC_MODE, "/m" }, + /* 051 */ { ud_itab__51, UD_TAB__OPC_VENDOR, "/vendor" }, + /* 052 */ { ud_itab__52, UD_TAB__OPC_MODE, "/m" }, + /* 053 */ { ud_itab__53, UD_TAB__OPC_VENDOR, "/vendor" }, + /* 054 */ { ud_itab__54, UD_TAB__OPC_TABLE, "opctbl" }, + /* 055 */ { ud_itab__55, UD_TAB__OPC_SSE, "/sse" }, + /* 056 */ { ud_itab__56, UD_TAB__OPC_MODE, "/m" }, + /* 057 */ { ud_itab__57, UD_TAB__OPC_VENDOR, "/vendor" }, + /* 058 */ { ud_itab__58, UD_TAB__OPC_VENDOR, "/vendor" }, + /* 059 */ { ud_itab__59, UD_TAB__OPC_SSE, "/sse" }, + /* 060 */ { ud_itab__60, UD_TAB__OPC_MODE, "/m" }, + /* 061 */ { ud_itab__61, UD_TAB__OPC_VENDOR, "/vendor" }, + /* 062 */ { ud_itab__62, UD_TAB__OPC_VENDOR, "/vendor" }, + /* 063 */ { ud_itab__63, UD_TAB__OPC_SSE, "/sse" }, + /* 064 */ { ud_itab__64, UD_TAB__OPC_SSE, "/sse" }, + /* 065 */ { ud_itab__65, UD_TAB__OPC_SSE, "/sse" }, + /* 066 */ { ud_itab__66, UD_TAB__OPC_SSE, "/sse" }, + /* 067 */ { ud_itab__67, UD_TAB__OPC_SSE, "/sse" }, + /* 068 */ { ud_itab__68, UD_TAB__OPC_SSE, "/sse" }, + /* 069 */ { ud_itab__69, UD_TAB__OPC_SSE, "/sse" }, + /* 070 */ { ud_itab__70, UD_TAB__OPC_SSE, "/sse" }, + /* 071 */ { ud_itab__71, UD_TAB__OPC_SSE, "/sse" }, + /* 072 */ { ud_itab__72, UD_TAB__OPC_SSE, "/sse" }, + /* 073 */ { ud_itab__73, UD_TAB__OPC_SSE, "/sse" }, + /* 074 */ { ud_itab__74, UD_TAB__OPC_SSE, "/sse" }, + /* 075 */ { ud_itab__75, UD_TAB__OPC_SSE, "/sse" }, + /* 076 */ { ud_itab__76, UD_TAB__OPC_SSE, "/sse" }, + /* 077 */ { ud_itab__77, UD_TAB__OPC_SSE, "/sse" }, + /* 078 */ { ud_itab__78, UD_TAB__OPC_SSE, "/sse" }, + /* 079 */ { ud_itab__79, UD_TAB__OPC_SSE, "/sse" }, + /* 080 */ { ud_itab__80, UD_TAB__OPC_SSE, "/sse" }, + /* 081 */ { ud_itab__81, UD_TAB__OPC_SSE, "/sse" }, + /* 082 */ { ud_itab__82, UD_TAB__OPC_SSE, "/sse" }, + /* 083 */ { ud_itab__83, UD_TAB__OPC_SSE, "/sse" }, + /* 084 */ { ud_itab__84, UD_TAB__OPC_SSE, "/sse" }, + /* 085 */ { ud_itab__85, UD_TAB__OPC_SSE, "/sse" }, + /* 086 */ { ud_itab__86, UD_TAB__OPC_SSE, "/sse" }, + /* 087 */ { ud_itab__87, UD_TAB__OPC_SSE, "/sse" }, + /* 088 */ { ud_itab__88, UD_TAB__OPC_SSE, "/sse" }, + /* 089 */ { ud_itab__89, UD_TAB__OPC_SSE, "/sse" }, + /* 090 */ { ud_itab__90, UD_TAB__OPC_SSE, "/sse" }, + /* 091 */ { ud_itab__91, UD_TAB__OPC_SSE, "/sse" }, + /* 092 */ { ud_itab__92, UD_TAB__OPC_SSE, "/sse" }, + /* 093 */ { ud_itab__93, UD_TAB__OPC_SSE, "/sse" }, + /* 094 */ { ud_itab__94, UD_TAB__OPC_SSE, "/sse" }, + /* 095 */ { ud_itab__95, UD_TAB__OPC_SSE, "/sse" }, + /* 096 */ { ud_itab__96, UD_TAB__OPC_SSE, "/sse" }, + /* 097 */ { ud_itab__97, UD_TAB__OPC_SSE, "/sse" }, + /* 098 */ { ud_itab__98, UD_TAB__OPC_SSE, "/sse" }, + /* 099 */ { ud_itab__99, UD_TAB__OPC_SSE, "/sse" }, + /* 100 */ { ud_itab__100, UD_TAB__OPC_SSE, "/sse" }, + /* 101 */ { ud_itab__101, UD_TAB__OPC_SSE, "/sse" }, + /* 102 */ { ud_itab__102, UD_TAB__OPC_SSE, "/sse" }, + /* 103 */ { ud_itab__103, UD_TAB__OPC_SSE, "/sse" }, + /* 104 */ { ud_itab__104, UD_TAB__OPC_SSE, "/sse" }, + /* 105 */ { ud_itab__105, UD_TAB__OPC_SSE, "/sse" }, + /* 106 */ { ud_itab__106, UD_TAB__OPC_SSE, "/sse" }, + /* 107 */ { ud_itab__107, UD_TAB__OPC_SSE, "/sse" }, + /* 108 */ { ud_itab__108, UD_TAB__OPC_SSE, "/sse" }, + /* 109 */ { ud_itab__109, UD_TAB__OPC_SSE, "/sse" }, + /* 110 */ { ud_itab__110, UD_TAB__OPC_SSE, "/sse" }, + /* 111 */ { ud_itab__111, UD_TAB__OPC_SSE, "/sse" }, + /* 112 */ { ud_itab__112, UD_TAB__OPC_SSE, "/sse" }, + /* 113 */ { ud_itab__113, UD_TAB__OPC_SSE, "/sse" }, + /* 114 */ { ud_itab__114, UD_TAB__OPC_SSE, "/sse" }, + /* 115 */ { ud_itab__115, UD_TAB__OPC_SSE, "/sse" }, + /* 116 */ { ud_itab__116, UD_TAB__OPC_TABLE, "opctbl" }, + /* 117 */ { ud_itab__117, UD_TAB__OPC_SSE, "/sse" }, + /* 118 */ { ud_itab__118, UD_TAB__OPC_SSE, "/sse" }, + /* 119 */ { ud_itab__119, UD_TAB__OPC_SSE, "/sse" }, + /* 120 */ { ud_itab__120, UD_TAB__OPC_SSE, "/sse" }, + /* 121 */ { ud_itab__121, UD_TAB__OPC_SSE, "/sse" }, + /* 122 */ { ud_itab__122, UD_TAB__OPC_SSE, "/sse" }, + /* 123 */ { ud_itab__123, UD_TAB__OPC_SSE, "/sse" }, + /* 124 */ { ud_itab__124, UD_TAB__OPC_SSE, "/sse" }, + /* 125 */ { ud_itab__125, UD_TAB__OPC_SSE, "/sse" }, + /* 126 */ { ud_itab__126, UD_TAB__OPC_SSE, "/sse" }, + /* 127 */ { ud_itab__127, UD_TAB__OPC_SSE, "/sse" }, + /* 128 */ { ud_itab__128, UD_TAB__OPC_OSIZE, "/o" }, + /* 129 */ { ud_itab__129, UD_TAB__OPC_SSE, "/sse" }, + /* 130 */ { ud_itab__130, UD_TAB__OPC_SSE, "/sse" }, + /* 131 */ { ud_itab__131, UD_TAB__OPC_SSE, "/sse" }, + /* 132 */ { ud_itab__132, UD_TAB__OPC_SSE, "/sse" }, + /* 133 */ { ud_itab__133, UD_TAB__OPC_OSIZE, "/o" }, + /* 134 */ { ud_itab__134, UD_TAB__OPC_SSE, "/sse" }, + /* 135 */ { ud_itab__135, UD_TAB__OPC_SSE, "/sse" }, + /* 136 */ { ud_itab__136, UD_TAB__OPC_SSE, "/sse" }, + /* 137 */ { ud_itab__137, UD_TAB__OPC_SSE, "/sse" }, + /* 138 */ { ud_itab__138, UD_TAB__OPC_SSE, "/sse" }, + /* 139 */ { ud_itab__139, UD_TAB__OPC_SSE, "/sse" }, + /* 140 */ { ud_itab__140, UD_TAB__OPC_SSE, "/sse" }, + /* 141 */ { ud_itab__141, UD_TAB__OPC_SSE, "/sse" }, + /* 142 */ { ud_itab__142, UD_TAB__OPC_SSE, "/sse" }, + /* 143 */ { ud_itab__143, UD_TAB__OPC_SSE, "/sse" }, + /* 144 */ { ud_itab__144, UD_TAB__OPC_SSE, "/sse" }, + /* 145 */ { ud_itab__145, UD_TAB__OPC_SSE, "/sse" }, + /* 146 */ { ud_itab__146, UD_TAB__OPC_SSE, "/sse" }, + /* 147 */ { ud_itab__147, UD_TAB__OPC_SSE, "/sse" }, + /* 148 */ { ud_itab__148, UD_TAB__OPC_SSE, "/sse" }, + /* 149 */ { ud_itab__149, UD_TAB__OPC_SSE, "/sse" }, + /* 150 */ { ud_itab__150, UD_TAB__OPC_SSE, "/sse" }, + /* 151 */ { ud_itab__151, UD_TAB__OPC_SSE, "/sse" }, + /* 152 */ { ud_itab__152, UD_TAB__OPC_SSE, "/sse" }, + /* 153 */ { ud_itab__153, UD_TAB__OPC_SSE, "/sse" }, + /* 154 */ { ud_itab__154, UD_TAB__OPC_SSE, "/sse" }, + /* 155 */ { ud_itab__155, UD_TAB__OPC_SSE, "/sse" }, + /* 156 */ { ud_itab__156, UD_TAB__OPC_SSE, "/sse" }, + /* 157 */ { ud_itab__157, UD_TAB__OPC_SSE, "/sse" }, + /* 158 */ { ud_itab__158, UD_TAB__OPC_SSE, "/sse" }, + /* 159 */ { ud_itab__159, UD_TAB__OPC_SSE, "/sse" }, + /* 160 */ { ud_itab__160, UD_TAB__OPC_SSE, "/sse" }, + /* 161 */ { ud_itab__161, UD_TAB__OPC_SSE, "/sse" }, + /* 162 */ { ud_itab__162, UD_TAB__OPC_SSE, "/sse" }, + /* 163 */ { ud_itab__163, UD_TAB__OPC_SSE, "/sse" }, + /* 164 */ { ud_itab__164, UD_TAB__OPC_SSE, "/sse" }, + /* 165 */ { ud_itab__165, UD_TAB__OPC_SSE, "/sse" }, + /* 166 */ { ud_itab__166, UD_TAB__OPC_SSE, "/sse" }, + /* 167 */ { ud_itab__167, UD_TAB__OPC_SSE, "/sse" }, + /* 168 */ { ud_itab__168, UD_TAB__OPC_SSE, "/sse" }, + /* 169 */ { ud_itab__169, UD_TAB__OPC_SSE, "/sse" }, + /* 170 */ { ud_itab__170, UD_TAB__OPC_SSE, "/sse" }, + /* 171 */ { ud_itab__171, UD_TAB__OPC_SSE, "/sse" }, + /* 172 */ { ud_itab__172, UD_TAB__OPC_SSE, "/sse" }, + /* 173 */ { ud_itab__173, UD_TAB__OPC_SSE, "/sse" }, + /* 174 */ { ud_itab__174, UD_TAB__OPC_OSIZE, "/o" }, + /* 175 */ { ud_itab__175, UD_TAB__OPC_OSIZE, "/o" }, + /* 176 */ { ud_itab__176, UD_TAB__OPC_SSE, "/sse" }, + /* 177 */ { ud_itab__177, UD_TAB__OPC_SSE, "/sse" }, + /* 178 */ { ud_itab__178, UD_TAB__OPC_REG, "/reg" }, + /* 179 */ { ud_itab__179, UD_TAB__OPC_SSE, "/sse" }, + /* 180 */ { ud_itab__180, UD_TAB__OPC_SSE, "/sse" }, + /* 181 */ { ud_itab__181, UD_TAB__OPC_SSE, "/sse" }, + /* 182 */ { ud_itab__182, UD_TAB__OPC_REG, "/reg" }, + /* 183 */ { ud_itab__183, UD_TAB__OPC_SSE, "/sse" }, + /* 184 */ { ud_itab__184, UD_TAB__OPC_SSE, "/sse" }, + /* 185 */ { ud_itab__185, UD_TAB__OPC_SSE, "/sse" }, + /* 186 */ { ud_itab__186, UD_TAB__OPC_REG, "/reg" }, + /* 187 */ { ud_itab__187, UD_TAB__OPC_SSE, "/sse" }, + /* 188 */ { ud_itab__188, UD_TAB__OPC_SSE, "/sse" }, + /* 189 */ { ud_itab__189, UD_TAB__OPC_SSE, "/sse" }, + /* 190 */ { ud_itab__190, UD_TAB__OPC_SSE, "/sse" }, + /* 191 */ { ud_itab__191, UD_TAB__OPC_SSE, "/sse" }, + /* 192 */ { ud_itab__192, UD_TAB__OPC_SSE, "/sse" }, + /* 193 */ { ud_itab__193, UD_TAB__OPC_SSE, "/sse" }, + /* 194 */ { ud_itab__194, UD_TAB__OPC_VENDOR, "/vendor" }, + /* 195 */ { ud_itab__195, UD_TAB__OPC_VENDOR, "/vendor" }, + /* 196 */ { ud_itab__196, UD_TAB__OPC_SSE, "/sse" }, + /* 197 */ { ud_itab__197, UD_TAB__OPC_SSE, "/sse" }, + /* 198 */ { ud_itab__198, UD_TAB__OPC_SSE, "/sse" }, + /* 199 */ { ud_itab__199, UD_TAB__OPC_OSIZE, "/o" }, + /* 200 */ { ud_itab__200, UD_TAB__OPC_OSIZE, "/o" }, + /* 201 */ { ud_itab__201, UD_TAB__OPC_SSE, "/sse" }, + /* 202 */ { ud_itab__202, UD_TAB__OPC_MOD, "/mod" }, + /* 203 */ { ud_itab__203, UD_TAB__OPC_REG, "/reg" }, + /* 204 */ { ud_itab__204, UD_TAB__OPC_RM, "/rm" }, + /* 205 */ { ud_itab__205, UD_TAB__OPC_RM, "/rm" }, + /* 206 */ { ud_itab__206, UD_TAB__OPC_RM, "/rm" }, + /* 207 */ { ud_itab__207, UD_TAB__OPC_MOD, "/mod" }, + /* 208 */ { ud_itab__208, UD_TAB__OPC_REG, "/reg" }, + /* 209 */ { ud_itab__209, UD_TAB__OPC_RM, "/rm" }, + /* 210 */ { ud_itab__210, UD_TAB__OPC_RM, "/rm" }, + /* 211 */ { ud_itab__211, UD_TAB__OPC_RM, "/rm" }, + /* 212 */ { ud_itab__212, UD_TAB__OPC_RM, "/rm" }, + /* 213 */ { ud_itab__213, UD_TAB__OPC_RM, "/rm" }, + /* 214 */ { ud_itab__214, UD_TAB__OPC_RM, "/rm" }, + /* 215 */ { ud_itab__215, UD_TAB__OPC_MOD, "/mod" }, + /* 216 */ { ud_itab__216, UD_TAB__OPC_REG, "/reg" }, + /* 217 */ { ud_itab__217, UD_TAB__OPC_REG, "/reg" }, + /* 218 */ { ud_itab__218, UD_TAB__OPC_RM, "/rm" }, + /* 219 */ { ud_itab__219, UD_TAB__OPC_RM, "/rm" }, + /* 220 */ { ud_itab__220, UD_TAB__OPC_RM, "/rm" }, + /* 221 */ { ud_itab__221, UD_TAB__OPC_SSE, "/sse" }, + /* 222 */ { ud_itab__222, UD_TAB__OPC_REG, "/reg" }, + /* 223 */ { ud_itab__223, UD_TAB__OPC_SSE, "/sse" }, + /* 224 */ { ud_itab__224, UD_TAB__OPC_SSE, "/sse" }, + /* 225 */ { ud_itab__225, UD_TAB__OPC_SSE, "/sse" }, + /* 226 */ { ud_itab__226, UD_TAB__OPC_SSE, "/sse" }, + /* 227 */ { ud_itab__227, UD_TAB__OPC_MOD, "/mod" }, + /* 228 */ { ud_itab__228, UD_TAB__OPC_REG, "/reg" }, + /* 229 */ { ud_itab__229, UD_TAB__OPC_OSIZE, "/o" }, + /* 230 */ { ud_itab__230, UD_TAB__OPC_SSE, "/sse" }, + /* 231 */ { ud_itab__231, UD_TAB__OPC_VENDOR, "/vendor" }, + /* 232 */ { ud_itab__232, UD_TAB__OPC_VENDOR, "/vendor" }, + /* 233 */ { ud_itab__233, UD_TAB__OPC_VENDOR, "/vendor" }, + /* 234 */ { ud_itab__234, UD_TAB__OPC_VENDOR, "/vendor" }, + /* 235 */ { ud_itab__235, UD_TAB__OPC_REG, "/reg" }, + /* 236 */ { ud_itab__236, UD_TAB__OPC_SSE, "/sse" }, + /* 237 */ { ud_itab__237, UD_TAB__OPC_SSE, "/sse" }, + /* 238 */ { ud_itab__238, UD_TAB__OPC_SSE, "/sse" }, + /* 239 */ { ud_itab__239, UD_TAB__OPC_SSE, "/sse" }, + /* 240 */ { ud_itab__240, UD_TAB__OPC_SSE, "/sse" }, + /* 241 */ { ud_itab__241, UD_TAB__OPC_SSE, "/sse" }, + /* 242 */ { ud_itab__242, UD_TAB__OPC_SSE, "/sse" }, + /* 243 */ { ud_itab__243, UD_TAB__OPC_SSE, "/sse" }, + /* 244 */ { ud_itab__244, UD_TAB__OPC_SSE, "/sse" }, + /* 245 */ { ud_itab__245, UD_TAB__OPC_SSE, "/sse" }, + /* 246 */ { ud_itab__246, UD_TAB__OPC_SSE, "/sse" }, + /* 247 */ { ud_itab__247, UD_TAB__OPC_SSE, "/sse" }, + /* 248 */ { ud_itab__248, UD_TAB__OPC_SSE, "/sse" }, + /* 249 */ { ud_itab__249, UD_TAB__OPC_SSE, "/sse" }, + /* 250 */ { ud_itab__250, UD_TAB__OPC_SSE, "/sse" }, + /* 251 */ { ud_itab__251, UD_TAB__OPC_SSE, "/sse" }, + /* 252 */ { ud_itab__252, UD_TAB__OPC_SSE, "/sse" }, + /* 253 */ { ud_itab__253, UD_TAB__OPC_SSE, "/sse" }, + /* 254 */ { ud_itab__254, UD_TAB__OPC_SSE, "/sse" }, + /* 255 */ { ud_itab__255, UD_TAB__OPC_SSE, "/sse" }, + /* 256 */ { ud_itab__256, UD_TAB__OPC_SSE, "/sse" }, + /* 257 */ { ud_itab__257, UD_TAB__OPC_SSE, "/sse" }, + /* 258 */ { ud_itab__258, UD_TAB__OPC_SSE, "/sse" }, + /* 259 */ { ud_itab__259, UD_TAB__OPC_SSE, "/sse" }, + /* 260 */ { ud_itab__260, UD_TAB__OPC_SSE, "/sse" }, + /* 261 */ { ud_itab__261, UD_TAB__OPC_SSE, "/sse" }, + /* 262 */ { ud_itab__262, UD_TAB__OPC_SSE, "/sse" }, + /* 263 */ { ud_itab__263, UD_TAB__OPC_SSE, "/sse" }, + /* 264 */ { ud_itab__264, UD_TAB__OPC_SSE, "/sse" }, + /* 265 */ { ud_itab__265, UD_TAB__OPC_SSE, "/sse" }, + /* 266 */ { ud_itab__266, UD_TAB__OPC_SSE, "/sse" }, + /* 267 */ { ud_itab__267, UD_TAB__OPC_SSE, "/sse" }, + /* 268 */ { ud_itab__268, UD_TAB__OPC_SSE, "/sse" }, + /* 269 */ { ud_itab__269, UD_TAB__OPC_SSE, "/sse" }, + /* 270 */ { ud_itab__270, UD_TAB__OPC_SSE, "/sse" }, + /* 271 */ { ud_itab__271, UD_TAB__OPC_SSE, "/sse" }, + /* 272 */ { ud_itab__272, UD_TAB__OPC_SSE, "/sse" }, + /* 273 */ { ud_itab__273, UD_TAB__OPC_SSE, "/sse" }, + /* 274 */ { ud_itab__274, UD_TAB__OPC_SSE, "/sse" }, + /* 275 */ { ud_itab__275, UD_TAB__OPC_MOD, "/mod" }, + /* 276 */ { ud_itab__276, UD_TAB__OPC_SSE, "/sse" }, + /* 277 */ { ud_itab__277, UD_TAB__OPC_SSE, "/sse" }, + /* 278 */ { ud_itab__278, UD_TAB__OPC_SSE, "/sse" }, + /* 279 */ { ud_itab__279, UD_TAB__OPC_SSE, "/sse" }, + /* 280 */ { ud_itab__280, UD_TAB__OPC_SSE, "/sse" }, + /* 281 */ { ud_itab__281, UD_TAB__OPC_SSE, "/sse" }, + /* 282 */ { ud_itab__282, UD_TAB__OPC_SSE, "/sse" }, + /* 283 */ { ud_itab__283, UD_TAB__OPC_SSE, "/sse" }, + /* 284 */ { ud_itab__284, UD_TAB__OPC_MODE, "/m" }, + /* 285 */ { ud_itab__285, UD_TAB__OPC_MODE, "/m" }, + /* 286 */ { ud_itab__286, UD_TAB__OPC_MODE, "/m" }, + /* 287 */ { ud_itab__287, UD_TAB__OPC_MODE, "/m" }, + /* 288 */ { ud_itab__288, UD_TAB__OPC_MODE, "/m" }, + /* 289 */ { ud_itab__289, UD_TAB__OPC_MODE, "/m" }, + /* 290 */ { ud_itab__290, UD_TAB__OPC_MODE, "/m" }, + /* 291 */ { ud_itab__291, UD_TAB__OPC_MODE, "/m" }, + /* 292 */ { ud_itab__292, UD_TAB__OPC_OSIZE, "/o" }, + /* 293 */ { ud_itab__293, UD_TAB__OPC_MODE, "/m" }, + /* 294 */ { ud_itab__294, UD_TAB__OPC_MODE, "/m" }, + /* 295 */ { ud_itab__295, UD_TAB__OPC_OSIZE, "/o" }, + /* 296 */ { ud_itab__296, UD_TAB__OPC_MODE, "/m" }, + /* 297 */ { ud_itab__297, UD_TAB__OPC_MODE, "/m" }, + /* 298 */ { ud_itab__298, UD_TAB__OPC_MODE, "/m" }, + /* 299 */ { ud_itab__299, UD_TAB__OPC_MODE, "/m" }, + /* 300 */ { ud_itab__300, UD_TAB__OPC_OSIZE, "/o" }, + /* 301 */ { ud_itab__301, UD_TAB__OPC_OSIZE, "/o" }, + /* 302 */ { ud_itab__302, UD_TAB__OPC_REG, "/reg" }, + /* 303 */ { ud_itab__303, UD_TAB__OPC_REG, "/reg" }, + /* 304 */ { ud_itab__304, UD_TAB__OPC_REG, "/reg" }, + /* 305 */ { ud_itab__305, UD_TAB__OPC_MODE, "/m" }, + /* 306 */ { ud_itab__306, UD_TAB__OPC_MODE, "/m" }, + /* 307 */ { ud_itab__307, UD_TAB__OPC_MODE, "/m" }, + /* 308 */ { ud_itab__308, UD_TAB__OPC_MODE, "/m" }, + /* 309 */ { ud_itab__309, UD_TAB__OPC_MODE, "/m" }, + /* 310 */ { ud_itab__310, UD_TAB__OPC_MODE, "/m" }, + /* 311 */ { ud_itab__311, UD_TAB__OPC_MODE, "/m" }, + /* 312 */ { ud_itab__312, UD_TAB__OPC_MODE, "/m" }, + /* 313 */ { ud_itab__313, UD_TAB__OPC_REG, "/reg" }, + /* 314 */ { ud_itab__314, UD_TAB__OPC_REG, "/reg" }, + /* 315 */ { ud_itab__315, UD_TAB__OPC_OSIZE, "/o" }, + /* 316 */ { ud_itab__316, UD_TAB__OPC_OSIZE, "/o" }, + /* 317 */ { ud_itab__317, UD_TAB__OPC_MODE, "/m" }, + /* 318 */ { ud_itab__318, UD_TAB__OPC_OSIZE, "/o" }, + /* 319 */ { ud_itab__319, UD_TAB__OPC_MODE, "/m" }, + /* 320 */ { ud_itab__320, UD_TAB__OPC_MODE, "/m" }, + /* 321 */ { ud_itab__321, UD_TAB__OPC_MODE, "/m" }, + /* 322 */ { ud_itab__322, UD_TAB__OPC_OSIZE, "/o" }, + /* 323 */ { ud_itab__323, UD_TAB__OPC_MODE, "/m" }, + /* 324 */ { ud_itab__324, UD_TAB__OPC_MODE, "/m" }, + /* 325 */ { ud_itab__325, UD_TAB__OPC_MODE, "/m" }, + /* 326 */ { ud_itab__326, UD_TAB__OPC_OSIZE, "/o" }, + /* 327 */ { ud_itab__327, UD_TAB__OPC_OSIZE, "/o" }, + /* 328 */ { ud_itab__328, UD_TAB__OPC_OSIZE, "/o" }, + /* 329 */ { ud_itab__329, UD_TAB__OPC_OSIZE, "/o" }, + /* 330 */ { ud_itab__330, UD_TAB__OPC_OSIZE, "/o" }, + /* 331 */ { ud_itab__331, UD_TAB__OPC_REG, "/reg" }, + /* 332 */ { ud_itab__332, UD_TAB__OPC_REG, "/reg" }, + /* 333 */ { ud_itab__333, UD_TAB__OPC_VEX, "/vex" }, + /* 334 */ { ud_itab__334, UD_TAB__OPC_MODE, "/m" }, + /* 335 */ { ud_itab__335, UD_TAB__OPC_TABLE, "opctbl" }, + /* 336 */ { ud_itab__336, UD_TAB__OPC_MOD, "/mod" }, + /* 337 */ { ud_itab__337, UD_TAB__OPC_MOD, "/mod" }, + /* 338 */ { ud_itab__338, UD_TAB__OPC_MOD, "/mod" }, + /* 339 */ { ud_itab__339, UD_TAB__OPC_REG, "/reg" }, + /* 340 */ { ud_itab__340, UD_TAB__OPC_VEX_L, "/vexl" }, + /* 341 */ { ud_itab__341, UD_TAB__OPC_TABLE, "opctbl" }, + /* 342 */ { ud_itab__342, UD_TAB__OPC_MOD, "/mod" }, + /* 343 */ { ud_itab__343, UD_TAB__OPC_MOD, "/mod" }, + /* 344 */ { ud_itab__344, UD_TAB__OPC_OSIZE, "/o" }, + /* 345 */ { ud_itab__345, UD_TAB__OPC_REG, "/reg" }, + /* 346 */ { ud_itab__346, UD_TAB__OPC_VEX_L, "/vexl" }, + /* 347 */ { ud_itab__347, UD_TAB__OPC_REG, "/reg" }, + /* 348 */ { ud_itab__348, UD_TAB__OPC_VEX_L, "/vexl" }, + /* 349 */ { ud_itab__349, UD_TAB__OPC_REG, "/reg" }, + /* 350 */ { ud_itab__350, UD_TAB__OPC_VEX_L, "/vexl" }, + /* 351 */ { ud_itab__351, UD_TAB__OPC_OSIZE, "/o" }, + /* 352 */ { ud_itab__352, UD_TAB__OPC_VEX_L, "/vexl" }, + /* 353 */ { ud_itab__353, UD_TAB__OPC_VEX_L, "/vexl" }, + /* 354 */ { ud_itab__354, UD_TAB__OPC_VEX_L, "/vexl" }, + /* 355 */ { ud_itab__355, UD_TAB__OPC_VEX_L, "/vexl" }, + /* 356 */ { ud_itab__356, UD_TAB__OPC_MOD, "/mod" }, + /* 357 */ { ud_itab__357, UD_TAB__OPC_TABLE, "opctbl" }, + /* 358 */ { ud_itab__358, UD_TAB__OPC_VEX_W, "/vexw" }, + /* 359 */ { ud_itab__359, UD_TAB__OPC_VEX_W, "/vexw" }, + /* 360 */ { ud_itab__360, UD_TAB__OPC_VEX_W, "/vexw" }, + /* 361 */ { ud_itab__361, UD_TAB__OPC_VEX_W, "/vexw" }, + /* 362 */ { ud_itab__362, UD_TAB__OPC_VEX_W, "/vexw" }, + /* 363 */ { ud_itab__363, UD_TAB__OPC_VEX_W, "/vexw" }, + /* 364 */ { ud_itab__364, UD_TAB__OPC_VEX_L, "/vexl" }, + /* 365 */ { ud_itab__365, UD_TAB__OPC_VEX_W, "/vexw" }, + /* 366 */ { ud_itab__366, UD_TAB__OPC_VEX_W, "/vexw" }, + /* 367 */ { ud_itab__367, UD_TAB__OPC_VEX_W, "/vexw" }, + /* 368 */ { ud_itab__368, UD_TAB__OPC_VEX_W, "/vexw" }, + /* 369 */ { ud_itab__369, UD_TAB__OPC_TABLE, "opctbl" }, + /* 370 */ { ud_itab__370, UD_TAB__OPC_VEX_W, "/vexw" }, + /* 371 */ { ud_itab__371, UD_TAB__OPC_VEX_W, "/vexw" }, + /* 372 */ { ud_itab__372, UD_TAB__OPC_VEX_W, "/vexw" }, + /* 373 */ { ud_itab__373, UD_TAB__OPC_VEX_L, "/vexl" }, + /* 374 */ { ud_itab__374, UD_TAB__OPC_VEX_W, "/vexw" }, + /* 375 */ { ud_itab__375, UD_TAB__OPC_OSIZE, "/o" }, + /* 376 */ { ud_itab__376, UD_TAB__OPC_VEX_W, "/vexw" }, + /* 377 */ { ud_itab__377, UD_TAB__OPC_VEX_W, "/vexw" }, + /* 378 */ { ud_itab__378, UD_TAB__OPC_VEX_W, "/vexw" }, + /* 379 */ { ud_itab__379, UD_TAB__OPC_VEX_W, "/vexw" }, + /* 380 */ { ud_itab__380, UD_TAB__OPC_VEX_L, "/vexl" }, + /* 381 */ { ud_itab__381, UD_TAB__OPC_VEX_W, "/vexw" }, + /* 382 */ { ud_itab__382, UD_TAB__OPC_VEX_L, "/vexl" }, + /* 383 */ { ud_itab__383, UD_TAB__OPC_VEX_W, "/vexw" }, + /* 384 */ { ud_itab__384, UD_TAB__OPC_VEX_L, "/vexl" }, + /* 385 */ { ud_itab__385, UD_TAB__OPC_MODE, "/m" }, + /* 386 */ { ud_itab__386, UD_TAB__OPC_VEX_W, "/vexw" }, + /* 387 */ { ud_itab__387, UD_TAB__OPC_VEX_L, "/vexl" }, + /* 388 */ { ud_itab__388, UD_TAB__OPC_VEX_W, "/vexw" }, + /* 389 */ { ud_itab__389, UD_TAB__OPC_VEX_L, "/vexl" }, + /* 390 */ { ud_itab__390, UD_TAB__OPC_VEX_L, "/vexl" }, + /* 391 */ { ud_itab__391, UD_TAB__OPC_VEX_W, "/vexw" }, + /* 392 */ { ud_itab__392, UD_TAB__OPC_VEX_W, "/vexw" }, + /* 393 */ { ud_itab__393, UD_TAB__OPC_VEX_W, "/vexw" }, + /* 394 */ { ud_itab__394, UD_TAB__OPC_TABLE, "opctbl" }, + /* 395 */ { ud_itab__395, UD_TAB__OPC_MOD, "/mod" }, + /* 396 */ { ud_itab__396, UD_TAB__OPC_MOD, "/mod" }, + /* 397 */ { ud_itab__397, UD_TAB__OPC_MOD, "/mod" }, + /* 398 */ { ud_itab__398, UD_TAB__OPC_MOD, "/mod" }, + /* 399 */ { ud_itab__399, UD_TAB__OPC_TABLE, "opctbl" }, + /* 400 */ { ud_itab__400, UD_TAB__OPC_MOD, "/mod" }, + /* 401 */ { ud_itab__401, UD_TAB__OPC_MOD, "/mod" }, + /* 402 */ { ud_itab__402, UD_TAB__OPC_MOD, "/mod" }, + /* 403 */ { ud_itab__403, UD_TAB__OPC_VEX, "/vex" }, + /* 404 */ { ud_itab__404, UD_TAB__OPC_MODE, "/m" }, + /* 405 */ { ud_itab__405, UD_TAB__OPC_REG, "/reg" }, + /* 406 */ { ud_itab__406, UD_TAB__OPC_REG, "/reg" }, + /* 407 */ { ud_itab__407, UD_TAB__OPC_MODE, "/m" }, + /* 408 */ { ud_itab__408, UD_TAB__OPC_OSIZE, "/o" }, + /* 409 */ { ud_itab__409, UD_TAB__OPC_REG, "/reg" }, + /* 410 */ { ud_itab__410, UD_TAB__OPC_REG, "/reg" }, + /* 411 */ { ud_itab__411, UD_TAB__OPC_REG, "/reg" }, + /* 412 */ { ud_itab__412, UD_TAB__OPC_REG, "/reg" }, + /* 413 */ { ud_itab__413, UD_TAB__OPC_MODE, "/m" }, + /* 414 */ { ud_itab__414, UD_TAB__OPC_MODE, "/m" }, + /* 415 */ { ud_itab__415, UD_TAB__OPC_MODE, "/m" }, + /* 416 */ { ud_itab__416, UD_TAB__OPC_MOD, "/mod" }, + /* 417 */ { ud_itab__417, UD_TAB__OPC_REG, "/reg" }, + /* 418 */ { ud_itab__418, UD_TAB__OPC_X87, "/x87" }, + /* 419 */ { ud_itab__419, UD_TAB__OPC_MOD, "/mod" }, + /* 420 */ { ud_itab__420, UD_TAB__OPC_REG, "/reg" }, + /* 421 */ { ud_itab__421, UD_TAB__OPC_X87, "/x87" }, + /* 422 */ { ud_itab__422, UD_TAB__OPC_MOD, "/mod" }, + /* 423 */ { ud_itab__423, UD_TAB__OPC_REG, "/reg" }, + /* 424 */ { ud_itab__424, UD_TAB__OPC_X87, "/x87" }, + /* 425 */ { ud_itab__425, UD_TAB__OPC_MOD, "/mod" }, + /* 426 */ { ud_itab__426, UD_TAB__OPC_REG, "/reg" }, + /* 427 */ { ud_itab__427, UD_TAB__OPC_X87, "/x87" }, + /* 428 */ { ud_itab__428, UD_TAB__OPC_MOD, "/mod" }, + /* 429 */ { ud_itab__429, UD_TAB__OPC_REG, "/reg" }, + /* 430 */ { ud_itab__430, UD_TAB__OPC_X87, "/x87" }, + /* 431 */ { ud_itab__431, UD_TAB__OPC_MOD, "/mod" }, + /* 432 */ { ud_itab__432, UD_TAB__OPC_REG, "/reg" }, + /* 433 */ { ud_itab__433, UD_TAB__OPC_X87, "/x87" }, + /* 434 */ { ud_itab__434, UD_TAB__OPC_MOD, "/mod" }, + /* 435 */ { ud_itab__435, UD_TAB__OPC_REG, "/reg" }, + /* 436 */ { ud_itab__436, UD_TAB__OPC_X87, "/x87" }, + /* 437 */ { ud_itab__437, UD_TAB__OPC_MOD, "/mod" }, + /* 438 */ { ud_itab__438, UD_TAB__OPC_REG, "/reg" }, + /* 439 */ { ud_itab__439, UD_TAB__OPC_X87, "/x87" }, + /* 440 */ { ud_itab__440, UD_TAB__OPC_ASIZE, "/a" }, + /* 441 */ { ud_itab__441, UD_TAB__OPC_MODE, "/m" }, + /* 442 */ { ud_itab__442, UD_TAB__OPC_REG, "/reg" }, + /* 443 */ { ud_itab__443, UD_TAB__OPC_REG, "/reg" }, + /* 444 */ { ud_itab__444, UD_TAB__OPC_REG, "/reg" }, + /* 445 */ { ud_itab__445, UD_TAB__OPC_REG, "/reg" }, + /* 446 */ { ud_itab__446, UD_TAB__OPC_MODE, "/m" }, +}; + +/* itab entry operand definitions (for readability) */ +#define O_AL { OP_AL, SZ_B } +#define O_AX { OP_AX, SZ_W } +#define O_Av { OP_A, SZ_V } +#define O_C { OP_C, SZ_NA } +#define O_CL { OP_CL, SZ_B } +#define O_CS { OP_CS, SZ_NA } +#define O_CX { OP_CX, SZ_W } +#define O_D { OP_D, SZ_NA } +#define O_DL { OP_DL, SZ_B } +#define O_DS { OP_DS, SZ_NA } +#define O_DX { OP_DX, SZ_W } +#define O_E { OP_E, SZ_NA } +#define O_ES { OP_ES, SZ_NA } +#define O_Eb { OP_E, SZ_B } +#define O_Ed { OP_E, SZ_D } +#define O_Eq { OP_E, SZ_Q } +#define O_Ev { OP_E, SZ_V } +#define O_Ew { OP_E, SZ_W } +#define O_Ey { OP_E, SZ_Y } +#define O_Ez { OP_E, SZ_Z } +#define O_FS { OP_FS, SZ_NA } +#define O_Fv { OP_F, SZ_V } +#define O_G { OP_G, SZ_NA } +#define O_GS { OP_GS, SZ_NA } +#define O_Gb { OP_G, SZ_B } +#define O_Gd { OP_G, SZ_D } +#define O_Gq { OP_G, SZ_Q } +#define O_Gv { OP_G, SZ_V } +#define O_Gw { OP_G, SZ_W } +#define O_Gy { OP_G, SZ_Y } +#define O_Gz { OP_G, SZ_Z } +#define O_H { OP_H, SZ_X } +#define O_Hqq { OP_H, SZ_QQ } +#define O_Hx { OP_H, SZ_X } +#define O_I1 { OP_I1, SZ_NA } +#define O_I3 { OP_I3, SZ_NA } +#define O_Ib { OP_I, SZ_B } +#define O_Iv { OP_I, SZ_V } +#define O_Iw { OP_I, SZ_W } +#define O_Iz { OP_I, SZ_Z } +#define O_Jb { OP_J, SZ_B } +#define O_Jv { OP_J, SZ_V } +#define O_Jz { OP_J, SZ_Z } +#define O_L { OP_L, SZ_O } +#define O_Lx { OP_L, SZ_X } +#define O_M { OP_M, SZ_NA } +#define O_Mb { OP_M, SZ_B } +#define O_MbRd { OP_MR, SZ_BD } +#define O_MbRv { OP_MR, SZ_BV } +#define O_Md { OP_M, SZ_D } +#define O_MdRy { OP_MR, SZ_DY } +#define O_MdU { OP_MU, SZ_DO } +#define O_Mdq { OP_M, SZ_DQ } +#define O_Mo { OP_M, SZ_O } +#define O_Mq { OP_M, SZ_Q } +#define O_MqU { OP_MU, SZ_QO } +#define O_Ms { OP_M, SZ_W } +#define O_Mt { OP_M, SZ_T } +#define O_Mv { OP_M, SZ_V } +#define O_Mw { OP_M, SZ_W } +#define O_MwRd { OP_MR, SZ_WD } +#define O_MwRv { OP_MR, SZ_WV } +#define O_MwRy { OP_MR, SZ_WY } +#define O_MwU { OP_MU, SZ_WO } +#define O_N { OP_N, SZ_Q } +#define O_NONE { OP_NONE, SZ_NA } +#define O_Ob { OP_O, SZ_B } +#define O_Ov { OP_O, SZ_V } +#define O_Ow { OP_O, SZ_W } +#define O_P { OP_P, SZ_Q } +#define O_Q { OP_Q, SZ_Q } +#define O_R { OP_R, SZ_RDQ } +#define O_R0b { OP_R0, SZ_B } +#define O_R0v { OP_R0, SZ_V } +#define O_R0w { OP_R0, SZ_W } +#define O_R0y { OP_R0, SZ_Y } +#define O_R0z { OP_R0, SZ_Z } +#define O_R1b { OP_R1, SZ_B } +#define O_R1v { OP_R1, SZ_V } +#define O_R1w { OP_R1, SZ_W } +#define O_R1y { OP_R1, SZ_Y } +#define O_R1z { OP_R1, SZ_Z } +#define O_R2b { OP_R2, SZ_B } +#define O_R2v { OP_R2, SZ_V } +#define O_R2w { OP_R2, SZ_W } +#define O_R2y { OP_R2, SZ_Y } +#define O_R2z { OP_R2, SZ_Z } +#define O_R3b { OP_R3, SZ_B } +#define O_R3v { OP_R3, SZ_V } +#define O_R3w { OP_R3, SZ_W } +#define O_R3y { OP_R3, SZ_Y } +#define O_R3z { OP_R3, SZ_Z } +#define O_R4b { OP_R4, SZ_B } +#define O_R4v { OP_R4, SZ_V } +#define O_R4w { OP_R4, SZ_W } +#define O_R4y { OP_R4, SZ_Y } +#define O_R4z { OP_R4, SZ_Z } +#define O_R5b { OP_R5, SZ_B } +#define O_R5v { OP_R5, SZ_V } +#define O_R5w { OP_R5, SZ_W } +#define O_R5y { OP_R5, SZ_Y } +#define O_R5z { OP_R5, SZ_Z } +#define O_R6b { OP_R6, SZ_B } +#define O_R6v { OP_R6, SZ_V } +#define O_R6w { OP_R6, SZ_W } +#define O_R6y { OP_R6, SZ_Y } +#define O_R6z { OP_R6, SZ_Z } +#define O_R7b { OP_R7, SZ_B } +#define O_R7v { OP_R7, SZ_V } +#define O_R7w { OP_R7, SZ_W } +#define O_R7y { OP_R7, SZ_Y } +#define O_R7z { OP_R7, SZ_Z } +#define O_S { OP_S, SZ_W } +#define O_SS { OP_SS, SZ_NA } +#define O_ST0 { OP_ST0, SZ_NA } +#define O_ST1 { OP_ST1, SZ_NA } +#define O_ST2 { OP_ST2, SZ_NA } +#define O_ST3 { OP_ST3, SZ_NA } +#define O_ST4 { OP_ST4, SZ_NA } +#define O_ST5 { OP_ST5, SZ_NA } +#define O_ST6 { OP_ST6, SZ_NA } +#define O_ST7 { OP_ST7, SZ_NA } +#define O_U { OP_U, SZ_O } +#define O_Ux { OP_U, SZ_X } +#define O_V { OP_V, SZ_DQ } +#define O_Vdq { OP_V, SZ_DQ } +#define O_Vqq { OP_V, SZ_QQ } +#define O_Vsd { OP_V, SZ_Q } +#define O_Vx { OP_V, SZ_X } +#define O_W { OP_W, SZ_DQ } +#define O_Wdq { OP_W, SZ_DQ } +#define O_Wqq { OP_W, SZ_QQ } +#define O_Wsd { OP_W, SZ_Q } +#define O_Wx { OP_W, SZ_X } +#define O_eAX { OP_eAX, SZ_Z } +#define O_eCX { OP_eCX, SZ_Z } +#define O_eDX { OP_eDX, SZ_Z } +#define O_rAX { OP_rAX, SZ_V } +#define O_rCX { OP_rCX, SZ_V } +#define O_rDX { OP_rDX, SZ_V } +#define O_sIb { OP_sI, SZ_B } +#define O_sIv { OP_sI, SZ_V } +#define O_sIz { OP_sI, SZ_Z } + +struct ud_itab_entry ud_itab[] = { + /* 0000 */ { UD_Iinvalid, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 0001 */ { UD_Iaaa, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 0002 */ { UD_Iaad, O_Ib, O_NONE, O_NONE, O_NONE, P_none }, + /* 0003 */ { UD_Iaam, O_Ib, O_NONE, O_NONE, O_NONE, P_none }, + /* 0004 */ { UD_Iaas, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 0005 */ { UD_Iadc, O_Eb, O_Gb, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0006 */ { UD_Iadc, O_Ev, O_Gv, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0007 */ { UD_Iadc, O_Gb, O_Eb, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0008 */ { UD_Iadc, O_Gv, O_Ev, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0009 */ { UD_Iadc, O_AL, O_Ib, O_NONE, O_NONE, P_none }, + /* 0010 */ { UD_Iadc, O_rAX, O_sIz, O_NONE, O_NONE, P_oso|P_rexw }, + /* 0011 */ { UD_Iadc, O_Eb, O_Ib, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0012 */ { UD_Iadc, O_Eb, O_Ib, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_inv64 }, + /* 0013 */ { UD_Iadc, O_Ev, O_sIz, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0014 */ { UD_Iadc, O_Ev, O_sIb, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0015 */ { UD_Iadd, O_Eb, O_Gb, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0016 */ { UD_Iadd, O_Ev, O_Gv, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0017 */ { UD_Iadd, O_Gb, O_Eb, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0018 */ { UD_Iadd, O_Gv, O_Ev, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0019 */ { UD_Iadd, O_AL, O_Ib, O_NONE, O_NONE, P_none }, + /* 0020 */ { UD_Iadd, O_rAX, O_sIz, O_NONE, O_NONE, P_oso|P_rexw }, + /* 0021 */ { UD_Iadd, O_Eb, O_Ib, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0022 */ { UD_Iadd, O_Eb, O_Ib, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_inv64 }, + /* 0023 */ { UD_Iadd, O_Ev, O_sIz, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0024 */ { UD_Iadd, O_Ev, O_sIb, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0025 */ { UD_Iaddpd, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0026 */ { UD_Ivaddpd, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 0027 */ { UD_Iaddps, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0028 */ { UD_Ivaddps, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 0029 */ { UD_Iaddsd, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0030 */ { UD_Ivaddsd, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0031 */ { UD_Iaddss, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0032 */ { UD_Ivaddss, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0033 */ { UD_Iaddsubpd, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0034 */ { UD_Ivaddsubpd, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0035 */ { UD_Iaddsubps, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0036 */ { UD_Ivaddsubps, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0037 */ { UD_Iaesdec, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0038 */ { UD_Ivaesdec, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0039 */ { UD_Iaesdeclast, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0040 */ { UD_Ivaesdeclast, O_Vx, O_Wx, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0041 */ { UD_Iaesenc, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0042 */ { UD_Ivaesenc, O_Vx, O_Wx, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0043 */ { UD_Iaesenclast, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0044 */ { UD_Ivaesenclast, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0045 */ { UD_Iaesimc, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0046 */ { UD_Ivaesimc, O_Vx, O_Wx, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0047 */ { UD_Iaeskeygenassist, O_V, O_W, O_Ib, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0048 */ { UD_Ivaeskeygenassist, O_Vx, O_Wx, O_Ib, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0049 */ { UD_Iand, O_Eb, O_Gb, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0050 */ { UD_Iand, O_Ev, O_Gv, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0051 */ { UD_Iand, O_Gb, O_Eb, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0052 */ { UD_Iand, O_Gv, O_Ev, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0053 */ { UD_Iand, O_AL, O_Ib, O_NONE, O_NONE, P_none }, + /* 0054 */ { UD_Iand, O_rAX, O_sIz, O_NONE, O_NONE, P_oso|P_rexw }, + /* 0055 */ { UD_Iand, O_Eb, O_Ib, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0056 */ { UD_Iand, O_Eb, O_Ib, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_inv64 }, + /* 0057 */ { UD_Iand, O_Ev, O_sIz, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0058 */ { UD_Iand, O_Ev, O_sIb, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0059 */ { UD_Iandpd, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0060 */ { UD_Ivandpd, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 0061 */ { UD_Iandps, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0062 */ { UD_Ivandps, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0063 */ { UD_Iandnpd, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0064 */ { UD_Ivandnpd, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0065 */ { UD_Iandnps, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0066 */ { UD_Ivandnps, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0067 */ { UD_Iarpl, O_Ew, O_Gw, O_NONE, O_NONE, P_aso }, + /* 0068 */ { UD_Imovsxd, O_Gq, O_Ed, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexx|P_rexr|P_rexb }, + /* 0069 */ { UD_Icall, O_Ev, O_NONE, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0070 */ { UD_Icall, O_Eq, O_NONE, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb|P_def64 }, + /* 0071 */ { UD_Icall, O_Fv, O_NONE, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0072 */ { UD_Icall, O_Jz, O_NONE, O_NONE, O_NONE, P_oso|P_def64 }, + /* 0073 */ { UD_Icall, O_Av, O_NONE, O_NONE, O_NONE, P_oso }, + /* 0074 */ { UD_Icbw, O_NONE, O_NONE, O_NONE, O_NONE, P_oso|P_rexw }, + /* 0075 */ { UD_Icwde, O_NONE, O_NONE, O_NONE, O_NONE, P_oso|P_rexw }, + /* 0076 */ { UD_Icdqe, O_NONE, O_NONE, O_NONE, O_NONE, P_oso|P_rexw }, + /* 0077 */ { UD_Iclc, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 0078 */ { UD_Icld, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 0079 */ { UD_Iclflush, O_M, O_NONE, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0080 */ { UD_Iclgi, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 0081 */ { UD_Icli, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 0082 */ { UD_Iclts, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 0083 */ { UD_Icmc, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 0084 */ { UD_Icmovo, O_Gv, O_Ev, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0085 */ { UD_Icmovno, O_Gv, O_Ev, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0086 */ { UD_Icmovb, O_Gv, O_Ev, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0087 */ { UD_Icmovae, O_Gv, O_Ev, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0088 */ { UD_Icmovz, O_Gv, O_Ev, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0089 */ { UD_Icmovnz, O_Gv, O_Ev, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0090 */ { UD_Icmovbe, O_Gv, O_Ev, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0091 */ { UD_Icmova, O_Gv, O_Ev, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0092 */ { UD_Icmovs, O_Gv, O_Ev, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0093 */ { UD_Icmovns, O_Gv, O_Ev, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0094 */ { UD_Icmovp, O_Gv, O_Ev, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0095 */ { UD_Icmovnp, O_Gv, O_Ev, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0096 */ { UD_Icmovl, O_Gv, O_Ev, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0097 */ { UD_Icmovge, O_Gv, O_Ev, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0098 */ { UD_Icmovle, O_Gv, O_Ev, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0099 */ { UD_Icmovg, O_Gv, O_Ev, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0100 */ { UD_Icmp, O_Eb, O_Gb, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0101 */ { UD_Icmp, O_Ev, O_Gv, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0102 */ { UD_Icmp, O_Gb, O_Eb, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0103 */ { UD_Icmp, O_Gv, O_Ev, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0104 */ { UD_Icmp, O_AL, O_Ib, O_NONE, O_NONE, P_none }, + /* 0105 */ { UD_Icmp, O_rAX, O_sIz, O_NONE, O_NONE, P_oso|P_rexw }, + /* 0106 */ { UD_Icmp, O_Eb, O_Ib, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0107 */ { UD_Icmp, O_Eb, O_Ib, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_inv64 }, + /* 0108 */ { UD_Icmp, O_Ev, O_sIz, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0109 */ { UD_Icmp, O_Ev, O_sIb, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0110 */ { UD_Icmppd, O_V, O_W, O_Ib, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0111 */ { UD_Ivcmppd, O_Vx, O_Hx, O_Wx, O_Ib, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 0112 */ { UD_Icmpps, O_V, O_W, O_Ib, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0113 */ { UD_Ivcmpps, O_Vx, O_Hx, O_Wx, O_Ib, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 0114 */ { UD_Icmpsb, O_NONE, O_NONE, O_NONE, O_NONE, P_strz|P_seg }, + /* 0115 */ { UD_Icmpsw, O_NONE, O_NONE, O_NONE, O_NONE, P_strz|P_oso|P_rexw|P_seg }, + /* 0116 */ { UD_Icmpsd, O_NONE, O_NONE, O_NONE, O_NONE, P_strz|P_oso|P_rexw|P_seg }, + /* 0117 */ { UD_Icmpsd, O_V, O_W, O_Ib, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0118 */ { UD_Ivcmpsd, O_Vx, O_Hx, O_Wx, O_Ib, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0119 */ { UD_Icmpsq, O_NONE, O_NONE, O_NONE, O_NONE, P_strz|P_oso|P_rexw|P_seg }, + /* 0120 */ { UD_Icmpss, O_V, O_W, O_Ib, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0121 */ { UD_Ivcmpss, O_Vx, O_Hx, O_Wx, O_Ib, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0122 */ { UD_Icmpxchg, O_Eb, O_Gb, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0123 */ { UD_Icmpxchg, O_Ev, O_Gv, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0124 */ { UD_Icmpxchg8b, O_M, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0125 */ { UD_Icmpxchg8b, O_M, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0126 */ { UD_Icmpxchg16b, O_M, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0127 */ { UD_Icomisd, O_Vsd, O_Wsd, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0128 */ { UD_Ivcomisd, O_Vsd, O_Wsd, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0129 */ { UD_Icomiss, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0130 */ { UD_Ivcomiss, O_Vx, O_Wx, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0131 */ { UD_Icpuid, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 0132 */ { UD_Icvtdq2pd, O_V, O_Wdq, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0133 */ { UD_Ivcvtdq2pd, O_Vx, O_Wdq, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 0134 */ { UD_Icvtdq2ps, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0135 */ { UD_Ivcvtdq2ps, O_Vx, O_Wx, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0136 */ { UD_Icvtpd2dq, O_Vdq, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0137 */ { UD_Ivcvtpd2dq, O_Vdq, O_Wx, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 0138 */ { UD_Icvtpd2pi, O_P, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0139 */ { UD_Icvtpd2ps, O_Vdq, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0140 */ { UD_Ivcvtpd2ps, O_Vdq, O_Wx, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 0141 */ { UD_Icvtpi2ps, O_V, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0142 */ { UD_Icvtpi2pd, O_V, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0143 */ { UD_Icvtps2dq, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0144 */ { UD_Ivcvtps2dq, O_Vx, O_Wx, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 0145 */ { UD_Icvtps2pd, O_V, O_Wdq, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0146 */ { UD_Ivcvtps2pd, O_Vx, O_Wdq, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 0147 */ { UD_Icvtps2pi, O_P, O_MqU, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0148 */ { UD_Icvtsd2si, O_Gy, O_MqU, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0149 */ { UD_Ivcvtsd2si, O_Gy, O_MqU, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0150 */ { UD_Icvtsd2ss, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0151 */ { UD_Ivcvtsd2ss, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0152 */ { UD_Icvtsi2sd, O_V, O_Ey, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0153 */ { UD_Ivcvtsi2sd, O_Vx, O_Hx, O_Ey, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0154 */ { UD_Icvtsi2ss, O_V, O_Ey, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0155 */ { UD_Ivcvtsi2ss, O_Vx, O_Hx, O_Ey, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0156 */ { UD_Icvtss2sd, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0157 */ { UD_Ivcvtss2sd, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0158 */ { UD_Icvtss2si, O_Gy, O_MdU, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0159 */ { UD_Ivcvtss2si, O_Gy, O_MdU, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0160 */ { UD_Icvttpd2dq, O_Vdq, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0161 */ { UD_Ivcvttpd2dq, O_Vdq, O_Wx, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 0162 */ { UD_Icvttpd2pi, O_P, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0163 */ { UD_Icvttps2dq, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0164 */ { UD_Ivcvttps2dq, O_Vx, O_Wx, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 0165 */ { UD_Icvttps2pi, O_P, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0166 */ { UD_Icvttsd2si, O_Gy, O_MqU, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0167 */ { UD_Ivcvttsd2si, O_Gy, O_MqU, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0168 */ { UD_Icvttss2si, O_Gy, O_MdU, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0169 */ { UD_Ivcvttss2si, O_Gy, O_MdU, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0170 */ { UD_Icwd, O_NONE, O_NONE, O_NONE, O_NONE, P_oso|P_rexw }, + /* 0171 */ { UD_Icdq, O_NONE, O_NONE, O_NONE, O_NONE, P_oso|P_rexw }, + /* 0172 */ { UD_Icqo, O_NONE, O_NONE, O_NONE, O_NONE, P_oso|P_rexw }, + /* 0173 */ { UD_Idaa, O_NONE, O_NONE, O_NONE, O_NONE, P_inv64 }, + /* 0174 */ { UD_Idas, O_NONE, O_NONE, O_NONE, O_NONE, P_inv64 }, + /* 0175 */ { UD_Idec, O_R0z, O_NONE, O_NONE, O_NONE, P_oso }, + /* 0176 */ { UD_Idec, O_R1z, O_NONE, O_NONE, O_NONE, P_oso }, + /* 0177 */ { UD_Idec, O_R2z, O_NONE, O_NONE, O_NONE, P_oso }, + /* 0178 */ { UD_Idec, O_R3z, O_NONE, O_NONE, O_NONE, P_oso }, + /* 0179 */ { UD_Idec, O_R4z, O_NONE, O_NONE, O_NONE, P_oso }, + /* 0180 */ { UD_Idec, O_R5z, O_NONE, O_NONE, O_NONE, P_oso }, + /* 0181 */ { UD_Idec, O_R6z, O_NONE, O_NONE, O_NONE, P_oso }, + /* 0182 */ { UD_Idec, O_R7z, O_NONE, O_NONE, O_NONE, P_oso }, + /* 0183 */ { UD_Idec, O_Eb, O_NONE, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0184 */ { UD_Idec, O_Ev, O_NONE, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0185 */ { UD_Idiv, O_Ev, O_NONE, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0186 */ { UD_Idiv, O_Eb, O_NONE, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0187 */ { UD_Idivpd, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0188 */ { UD_Ivdivpd, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 0189 */ { UD_Idivps, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0190 */ { UD_Ivdivps, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0191 */ { UD_Idivsd, O_V, O_MqU, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0192 */ { UD_Ivdivsd, O_Vx, O_Hx, O_MqU, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0193 */ { UD_Idivss, O_V, O_MdU, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0194 */ { UD_Ivdivss, O_Vx, O_Hx, O_MdU, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0195 */ { UD_Idppd, O_V, O_W, O_Ib, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0196 */ { UD_Ivdppd, O_Vx, O_Hx, O_Wx, O_Ib, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0197 */ { UD_Idpps, O_V, O_W, O_Ib, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0198 */ { UD_Ivdpps, O_Vx, O_Hx, O_Wx, O_Ib, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 0199 */ { UD_Iemms, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 0200 */ { UD_Ienter, O_Iw, O_Ib, O_NONE, O_NONE, P_def64 }, + /* 0201 */ { UD_Iextractps, O_MdRy, O_V, O_Ib, O_NONE, P_aso|P_rexr|P_rexw|P_rexx|P_rexb }, + /* 0202 */ { UD_Ivextractps, O_MdRy, O_Vx, O_Ib, O_NONE, P_aso|P_rexr|P_rexw|P_rexx|P_rexb }, + /* 0203 */ { UD_If2xm1, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 0204 */ { UD_Ifabs, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 0205 */ { UD_Ifadd, O_Mq, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0206 */ { UD_Ifadd, O_Md, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0207 */ { UD_Ifadd, O_ST0, O_ST0, O_NONE, O_NONE, P_none }, + /* 0208 */ { UD_Ifadd, O_ST1, O_ST0, O_NONE, O_NONE, P_none }, + /* 0209 */ { UD_Ifadd, O_ST2, O_ST0, O_NONE, O_NONE, P_none }, + /* 0210 */ { UD_Ifadd, O_ST3, O_ST0, O_NONE, O_NONE, P_none }, + /* 0211 */ { UD_Ifadd, O_ST4, O_ST0, O_NONE, O_NONE, P_none }, + /* 0212 */ { UD_Ifadd, O_ST5, O_ST0, O_NONE, O_NONE, P_none }, + /* 0213 */ { UD_Ifadd, O_ST6, O_ST0, O_NONE, O_NONE, P_none }, + /* 0214 */ { UD_Ifadd, O_ST7, O_ST0, O_NONE, O_NONE, P_none }, + /* 0215 */ { UD_Ifadd, O_ST0, O_ST0, O_NONE, O_NONE, P_none }, + /* 0216 */ { UD_Ifadd, O_ST0, O_ST1, O_NONE, O_NONE, P_none }, + /* 0217 */ { UD_Ifadd, O_ST0, O_ST2, O_NONE, O_NONE, P_none }, + /* 0218 */ { UD_Ifadd, O_ST0, O_ST3, O_NONE, O_NONE, P_none }, + /* 0219 */ { UD_Ifadd, O_ST0, O_ST4, O_NONE, O_NONE, P_none }, + /* 0220 */ { UD_Ifadd, O_ST0, O_ST5, O_NONE, O_NONE, P_none }, + /* 0221 */ { UD_Ifadd, O_ST0, O_ST6, O_NONE, O_NONE, P_none }, + /* 0222 */ { UD_Ifadd, O_ST0, O_ST7, O_NONE, O_NONE, P_none }, + /* 0223 */ { UD_Ifaddp, O_ST0, O_ST0, O_NONE, O_NONE, P_none }, + /* 0224 */ { UD_Ifaddp, O_ST1, O_ST0, O_NONE, O_NONE, P_none }, + /* 0225 */ { UD_Ifaddp, O_ST2, O_ST0, O_NONE, O_NONE, P_none }, + /* 0226 */ { UD_Ifaddp, O_ST3, O_ST0, O_NONE, O_NONE, P_none }, + /* 0227 */ { UD_Ifaddp, O_ST4, O_ST0, O_NONE, O_NONE, P_none }, + /* 0228 */ { UD_Ifaddp, O_ST5, O_ST0, O_NONE, O_NONE, P_none }, + /* 0229 */ { UD_Ifaddp, O_ST6, O_ST0, O_NONE, O_NONE, P_none }, + /* 0230 */ { UD_Ifaddp, O_ST7, O_ST0, O_NONE, O_NONE, P_none }, + /* 0231 */ { UD_Ifbld, O_Mt, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0232 */ { UD_Ifbstp, O_Mt, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0233 */ { UD_Ifchs, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 0234 */ { UD_Ifclex, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 0235 */ { UD_Ifcmovb, O_ST0, O_ST0, O_NONE, O_NONE, P_none }, + /* 0236 */ { UD_Ifcmovb, O_ST0, O_ST1, O_NONE, O_NONE, P_none }, + /* 0237 */ { UD_Ifcmovb, O_ST0, O_ST2, O_NONE, O_NONE, P_none }, + /* 0238 */ { UD_Ifcmovb, O_ST0, O_ST3, O_NONE, O_NONE, P_none }, + /* 0239 */ { UD_Ifcmovb, O_ST0, O_ST4, O_NONE, O_NONE, P_none }, + /* 0240 */ { UD_Ifcmovb, O_ST0, O_ST5, O_NONE, O_NONE, P_none }, + /* 0241 */ { UD_Ifcmovb, O_ST0, O_ST6, O_NONE, O_NONE, P_none }, + /* 0242 */ { UD_Ifcmovb, O_ST0, O_ST7, O_NONE, O_NONE, P_none }, + /* 0243 */ { UD_Ifcmove, O_ST0, O_ST0, O_NONE, O_NONE, P_none }, + /* 0244 */ { UD_Ifcmove, O_ST0, O_ST1, O_NONE, O_NONE, P_none }, + /* 0245 */ { UD_Ifcmove, O_ST0, O_ST2, O_NONE, O_NONE, P_none }, + /* 0246 */ { UD_Ifcmove, O_ST0, O_ST3, O_NONE, O_NONE, P_none }, + /* 0247 */ { UD_Ifcmove, O_ST0, O_ST4, O_NONE, O_NONE, P_none }, + /* 0248 */ { UD_Ifcmove, O_ST0, O_ST5, O_NONE, O_NONE, P_none }, + /* 0249 */ { UD_Ifcmove, O_ST0, O_ST6, O_NONE, O_NONE, P_none }, + /* 0250 */ { UD_Ifcmove, O_ST0, O_ST7, O_NONE, O_NONE, P_none }, + /* 0251 */ { UD_Ifcmovbe, O_ST0, O_ST0, O_NONE, O_NONE, P_none }, + /* 0252 */ { UD_Ifcmovbe, O_ST0, O_ST1, O_NONE, O_NONE, P_none }, + /* 0253 */ { UD_Ifcmovbe, O_ST0, O_ST2, O_NONE, O_NONE, P_none }, + /* 0254 */ { UD_Ifcmovbe, O_ST0, O_ST3, O_NONE, O_NONE, P_none }, + /* 0255 */ { UD_Ifcmovbe, O_ST0, O_ST4, O_NONE, O_NONE, P_none }, + /* 0256 */ { UD_Ifcmovbe, O_ST0, O_ST5, O_NONE, O_NONE, P_none }, + /* 0257 */ { UD_Ifcmovbe, O_ST0, O_ST6, O_NONE, O_NONE, P_none }, + /* 0258 */ { UD_Ifcmovbe, O_ST0, O_ST7, O_NONE, O_NONE, P_none }, + /* 0259 */ { UD_Ifcmovu, O_ST0, O_ST0, O_NONE, O_NONE, P_none }, + /* 0260 */ { UD_Ifcmovu, O_ST0, O_ST1, O_NONE, O_NONE, P_none }, + /* 0261 */ { UD_Ifcmovu, O_ST0, O_ST2, O_NONE, O_NONE, P_none }, + /* 0262 */ { UD_Ifcmovu, O_ST0, O_ST3, O_NONE, O_NONE, P_none }, + /* 0263 */ { UD_Ifcmovu, O_ST0, O_ST4, O_NONE, O_NONE, P_none }, + /* 0264 */ { UD_Ifcmovu, O_ST0, O_ST5, O_NONE, O_NONE, P_none }, + /* 0265 */ { UD_Ifcmovu, O_ST0, O_ST6, O_NONE, O_NONE, P_none }, + /* 0266 */ { UD_Ifcmovu, O_ST0, O_ST7, O_NONE, O_NONE, P_none }, + /* 0267 */ { UD_Ifcmovnb, O_ST0, O_ST0, O_NONE, O_NONE, P_none }, + /* 0268 */ { UD_Ifcmovnb, O_ST0, O_ST1, O_NONE, O_NONE, P_none }, + /* 0269 */ { UD_Ifcmovnb, O_ST0, O_ST2, O_NONE, O_NONE, P_none }, + /* 0270 */ { UD_Ifcmovnb, O_ST0, O_ST3, O_NONE, O_NONE, P_none }, + /* 0271 */ { UD_Ifcmovnb, O_ST0, O_ST4, O_NONE, O_NONE, P_none }, + /* 0272 */ { UD_Ifcmovnb, O_ST0, O_ST5, O_NONE, O_NONE, P_none }, + /* 0273 */ { UD_Ifcmovnb, O_ST0, O_ST6, O_NONE, O_NONE, P_none }, + /* 0274 */ { UD_Ifcmovnb, O_ST0, O_ST7, O_NONE, O_NONE, P_none }, + /* 0275 */ { UD_Ifcmovne, O_ST0, O_ST0, O_NONE, O_NONE, P_none }, + /* 0276 */ { UD_Ifcmovne, O_ST0, O_ST1, O_NONE, O_NONE, P_none }, + /* 0277 */ { UD_Ifcmovne, O_ST0, O_ST2, O_NONE, O_NONE, P_none }, + /* 0278 */ { UD_Ifcmovne, O_ST0, O_ST3, O_NONE, O_NONE, P_none }, + /* 0279 */ { UD_Ifcmovne, O_ST0, O_ST4, O_NONE, O_NONE, P_none }, + /* 0280 */ { UD_Ifcmovne, O_ST0, O_ST5, O_NONE, O_NONE, P_none }, + /* 0281 */ { UD_Ifcmovne, O_ST0, O_ST6, O_NONE, O_NONE, P_none }, + /* 0282 */ { UD_Ifcmovne, O_ST0, O_ST7, O_NONE, O_NONE, P_none }, + /* 0283 */ { UD_Ifcmovnbe, O_ST0, O_ST0, O_NONE, O_NONE, P_none }, + /* 0284 */ { UD_Ifcmovnbe, O_ST0, O_ST1, O_NONE, O_NONE, P_none }, + /* 0285 */ { UD_Ifcmovnbe, O_ST0, O_ST2, O_NONE, O_NONE, P_none }, + /* 0286 */ { UD_Ifcmovnbe, O_ST0, O_ST3, O_NONE, O_NONE, P_none }, + /* 0287 */ { UD_Ifcmovnbe, O_ST0, O_ST4, O_NONE, O_NONE, P_none }, + /* 0288 */ { UD_Ifcmovnbe, O_ST0, O_ST5, O_NONE, O_NONE, P_none }, + /* 0289 */ { UD_Ifcmovnbe, O_ST0, O_ST6, O_NONE, O_NONE, P_none }, + /* 0290 */ { UD_Ifcmovnbe, O_ST0, O_ST7, O_NONE, O_NONE, P_none }, + /* 0291 */ { UD_Ifcmovnu, O_ST0, O_ST0, O_NONE, O_NONE, P_none }, + /* 0292 */ { UD_Ifcmovnu, O_ST0, O_ST1, O_NONE, O_NONE, P_none }, + /* 0293 */ { UD_Ifcmovnu, O_ST0, O_ST2, O_NONE, O_NONE, P_none }, + /* 0294 */ { UD_Ifcmovnu, O_ST0, O_ST3, O_NONE, O_NONE, P_none }, + /* 0295 */ { UD_Ifcmovnu, O_ST0, O_ST4, O_NONE, O_NONE, P_none }, + /* 0296 */ { UD_Ifcmovnu, O_ST0, O_ST5, O_NONE, O_NONE, P_none }, + /* 0297 */ { UD_Ifcmovnu, O_ST0, O_ST6, O_NONE, O_NONE, P_none }, + /* 0298 */ { UD_Ifcmovnu, O_ST0, O_ST7, O_NONE, O_NONE, P_none }, + /* 0299 */ { UD_Ifucomi, O_ST0, O_ST0, O_NONE, O_NONE, P_none }, + /* 0300 */ { UD_Ifucomi, O_ST0, O_ST1, O_NONE, O_NONE, P_none }, + /* 0301 */ { UD_Ifucomi, O_ST0, O_ST2, O_NONE, O_NONE, P_none }, + /* 0302 */ { UD_Ifucomi, O_ST0, O_ST3, O_NONE, O_NONE, P_none }, + /* 0303 */ { UD_Ifucomi, O_ST0, O_ST4, O_NONE, O_NONE, P_none }, + /* 0304 */ { UD_Ifucomi, O_ST0, O_ST5, O_NONE, O_NONE, P_none }, + /* 0305 */ { UD_Ifucomi, O_ST0, O_ST6, O_NONE, O_NONE, P_none }, + /* 0306 */ { UD_Ifucomi, O_ST0, O_ST7, O_NONE, O_NONE, P_none }, + /* 0307 */ { UD_Ifcom, O_Md, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0308 */ { UD_Ifcom, O_Mq, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0309 */ { UD_Ifcom, O_ST0, O_ST0, O_NONE, O_NONE, P_none }, + /* 0310 */ { UD_Ifcom, O_ST0, O_ST1, O_NONE, O_NONE, P_none }, + /* 0311 */ { UD_Ifcom, O_ST0, O_ST2, O_NONE, O_NONE, P_none }, + /* 0312 */ { UD_Ifcom, O_ST0, O_ST3, O_NONE, O_NONE, P_none }, + /* 0313 */ { UD_Ifcom, O_ST0, O_ST4, O_NONE, O_NONE, P_none }, + /* 0314 */ { UD_Ifcom, O_ST0, O_ST5, O_NONE, O_NONE, P_none }, + /* 0315 */ { UD_Ifcom, O_ST0, O_ST6, O_NONE, O_NONE, P_none }, + /* 0316 */ { UD_Ifcom, O_ST0, O_ST7, O_NONE, O_NONE, P_none }, + /* 0317 */ { UD_Ifcom2, O_ST0, O_NONE, O_NONE, O_NONE, P_none }, + /* 0318 */ { UD_Ifcom2, O_ST1, O_NONE, O_NONE, O_NONE, P_none }, + /* 0319 */ { UD_Ifcom2, O_ST2, O_NONE, O_NONE, O_NONE, P_none }, + /* 0320 */ { UD_Ifcom2, O_ST3, O_NONE, O_NONE, O_NONE, P_none }, + /* 0321 */ { UD_Ifcom2, O_ST4, O_NONE, O_NONE, O_NONE, P_none }, + /* 0322 */ { UD_Ifcom2, O_ST5, O_NONE, O_NONE, O_NONE, P_none }, + /* 0323 */ { UD_Ifcom2, O_ST6, O_NONE, O_NONE, O_NONE, P_none }, + /* 0324 */ { UD_Ifcom2, O_ST7, O_NONE, O_NONE, O_NONE, P_none }, + /* 0325 */ { UD_Ifcomp3, O_ST0, O_NONE, O_NONE, O_NONE, P_none }, + /* 0326 */ { UD_Ifcomp3, O_ST1, O_NONE, O_NONE, O_NONE, P_none }, + /* 0327 */ { UD_Ifcomp3, O_ST2, O_NONE, O_NONE, O_NONE, P_none }, + /* 0328 */ { UD_Ifcomp3, O_ST3, O_NONE, O_NONE, O_NONE, P_none }, + /* 0329 */ { UD_Ifcomp3, O_ST4, O_NONE, O_NONE, O_NONE, P_none }, + /* 0330 */ { UD_Ifcomp3, O_ST5, O_NONE, O_NONE, O_NONE, P_none }, + /* 0331 */ { UD_Ifcomp3, O_ST6, O_NONE, O_NONE, O_NONE, P_none }, + /* 0332 */ { UD_Ifcomp3, O_ST7, O_NONE, O_NONE, O_NONE, P_none }, + /* 0333 */ { UD_Ifcomi, O_ST0, O_ST0, O_NONE, O_NONE, P_none }, + /* 0334 */ { UD_Ifcomi, O_ST0, O_ST1, O_NONE, O_NONE, P_none }, + /* 0335 */ { UD_Ifcomi, O_ST0, O_ST2, O_NONE, O_NONE, P_none }, + /* 0336 */ { UD_Ifcomi, O_ST0, O_ST3, O_NONE, O_NONE, P_none }, + /* 0337 */ { UD_Ifcomi, O_ST0, O_ST4, O_NONE, O_NONE, P_none }, + /* 0338 */ { UD_Ifcomi, O_ST0, O_ST5, O_NONE, O_NONE, P_none }, + /* 0339 */ { UD_Ifcomi, O_ST0, O_ST6, O_NONE, O_NONE, P_none }, + /* 0340 */ { UD_Ifcomi, O_ST0, O_ST7, O_NONE, O_NONE, P_none }, + /* 0341 */ { UD_Ifucomip, O_ST0, O_ST0, O_NONE, O_NONE, P_none }, + /* 0342 */ { UD_Ifucomip, O_ST0, O_ST1, O_NONE, O_NONE, P_none }, + /* 0343 */ { UD_Ifucomip, O_ST0, O_ST2, O_NONE, O_NONE, P_none }, + /* 0344 */ { UD_Ifucomip, O_ST0, O_ST3, O_NONE, O_NONE, P_none }, + /* 0345 */ { UD_Ifucomip, O_ST0, O_ST4, O_NONE, O_NONE, P_none }, + /* 0346 */ { UD_Ifucomip, O_ST0, O_ST5, O_NONE, O_NONE, P_none }, + /* 0347 */ { UD_Ifucomip, O_ST0, O_ST6, O_NONE, O_NONE, P_none }, + /* 0348 */ { UD_Ifucomip, O_ST0, O_ST7, O_NONE, O_NONE, P_none }, + /* 0349 */ { UD_Ifcomip, O_ST0, O_ST0, O_NONE, O_NONE, P_none }, + /* 0350 */ { UD_Ifcomip, O_ST0, O_ST1, O_NONE, O_NONE, P_none }, + /* 0351 */ { UD_Ifcomip, O_ST0, O_ST2, O_NONE, O_NONE, P_none }, + /* 0352 */ { UD_Ifcomip, O_ST0, O_ST3, O_NONE, O_NONE, P_none }, + /* 0353 */ { UD_Ifcomip, O_ST0, O_ST4, O_NONE, O_NONE, P_none }, + /* 0354 */ { UD_Ifcomip, O_ST0, O_ST5, O_NONE, O_NONE, P_none }, + /* 0355 */ { UD_Ifcomip, O_ST0, O_ST6, O_NONE, O_NONE, P_none }, + /* 0356 */ { UD_Ifcomip, O_ST0, O_ST7, O_NONE, O_NONE, P_none }, + /* 0357 */ { UD_Ifcomp, O_Md, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0358 */ { UD_Ifcomp, O_Mq, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0359 */ { UD_Ifcomp, O_ST0, O_ST0, O_NONE, O_NONE, P_none }, + /* 0360 */ { UD_Ifcomp, O_ST0, O_ST1, O_NONE, O_NONE, P_none }, + /* 0361 */ { UD_Ifcomp, O_ST0, O_ST2, O_NONE, O_NONE, P_none }, + /* 0362 */ { UD_Ifcomp, O_ST0, O_ST3, O_NONE, O_NONE, P_none }, + /* 0363 */ { UD_Ifcomp, O_ST0, O_ST4, O_NONE, O_NONE, P_none }, + /* 0364 */ { UD_Ifcomp, O_ST0, O_ST5, O_NONE, O_NONE, P_none }, + /* 0365 */ { UD_Ifcomp, O_ST0, O_ST6, O_NONE, O_NONE, P_none }, + /* 0366 */ { UD_Ifcomp, O_ST0, O_ST7, O_NONE, O_NONE, P_none }, + /* 0367 */ { UD_Ifcomp5, O_ST0, O_NONE, O_NONE, O_NONE, P_none }, + /* 0368 */ { UD_Ifcomp5, O_ST1, O_NONE, O_NONE, O_NONE, P_none }, + /* 0369 */ { UD_Ifcomp5, O_ST2, O_NONE, O_NONE, O_NONE, P_none }, + /* 0370 */ { UD_Ifcomp5, O_ST3, O_NONE, O_NONE, O_NONE, P_none }, + /* 0371 */ { UD_Ifcomp5, O_ST4, O_NONE, O_NONE, O_NONE, P_none }, + /* 0372 */ { UD_Ifcomp5, O_ST5, O_NONE, O_NONE, O_NONE, P_none }, + /* 0373 */ { UD_Ifcomp5, O_ST6, O_NONE, O_NONE, O_NONE, P_none }, + /* 0374 */ { UD_Ifcomp5, O_ST7, O_NONE, O_NONE, O_NONE, P_none }, + /* 0375 */ { UD_Ifcompp, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 0376 */ { UD_Ifcos, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 0377 */ { UD_Ifdecstp, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 0378 */ { UD_Ifdiv, O_Mq, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0379 */ { UD_Ifdiv, O_ST0, O_ST0, O_NONE, O_NONE, P_none }, + /* 0380 */ { UD_Ifdiv, O_ST1, O_ST0, O_NONE, O_NONE, P_none }, + /* 0381 */ { UD_Ifdiv, O_ST2, O_ST0, O_NONE, O_NONE, P_none }, + /* 0382 */ { UD_Ifdiv, O_ST3, O_ST0, O_NONE, O_NONE, P_none }, + /* 0383 */ { UD_Ifdiv, O_ST4, O_ST0, O_NONE, O_NONE, P_none }, + /* 0384 */ { UD_Ifdiv, O_ST5, O_ST0, O_NONE, O_NONE, P_none }, + /* 0385 */ { UD_Ifdiv, O_ST6, O_ST0, O_NONE, O_NONE, P_none }, + /* 0386 */ { UD_Ifdiv, O_ST7, O_ST0, O_NONE, O_NONE, P_none }, + /* 0387 */ { UD_Ifdiv, O_Md, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0388 */ { UD_Ifdiv, O_ST0, O_ST0, O_NONE, O_NONE, P_none }, + /* 0389 */ { UD_Ifdiv, O_ST0, O_ST1, O_NONE, O_NONE, P_none }, + /* 0390 */ { UD_Ifdiv, O_ST0, O_ST2, O_NONE, O_NONE, P_none }, + /* 0391 */ { UD_Ifdiv, O_ST0, O_ST3, O_NONE, O_NONE, P_none }, + /* 0392 */ { UD_Ifdiv, O_ST0, O_ST4, O_NONE, O_NONE, P_none }, + /* 0393 */ { UD_Ifdiv, O_ST0, O_ST5, O_NONE, O_NONE, P_none }, + /* 0394 */ { UD_Ifdiv, O_ST0, O_ST6, O_NONE, O_NONE, P_none }, + /* 0395 */ { UD_Ifdiv, O_ST0, O_ST7, O_NONE, O_NONE, P_none }, + /* 0396 */ { UD_Ifdivp, O_ST0, O_ST0, O_NONE, O_NONE, P_none }, + /* 0397 */ { UD_Ifdivp, O_ST1, O_ST0, O_NONE, O_NONE, P_none }, + /* 0398 */ { UD_Ifdivp, O_ST2, O_ST0, O_NONE, O_NONE, P_none }, + /* 0399 */ { UD_Ifdivp, O_ST3, O_ST0, O_NONE, O_NONE, P_none }, + /* 0400 */ { UD_Ifdivp, O_ST4, O_ST0, O_NONE, O_NONE, P_none }, + /* 0401 */ { UD_Ifdivp, O_ST5, O_ST0, O_NONE, O_NONE, P_none }, + /* 0402 */ { UD_Ifdivp, O_ST6, O_ST0, O_NONE, O_NONE, P_none }, + /* 0403 */ { UD_Ifdivp, O_ST7, O_ST0, O_NONE, O_NONE, P_none }, + /* 0404 */ { UD_Ifdivr, O_Mq, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0405 */ { UD_Ifdivr, O_ST0, O_ST0, O_NONE, O_NONE, P_none }, + /* 0406 */ { UD_Ifdivr, O_ST1, O_ST0, O_NONE, O_NONE, P_none }, + /* 0407 */ { UD_Ifdivr, O_ST2, O_ST0, O_NONE, O_NONE, P_none }, + /* 0408 */ { UD_Ifdivr, O_ST3, O_ST0, O_NONE, O_NONE, P_none }, + /* 0409 */ { UD_Ifdivr, O_ST4, O_ST0, O_NONE, O_NONE, P_none }, + /* 0410 */ { UD_Ifdivr, O_ST5, O_ST0, O_NONE, O_NONE, P_none }, + /* 0411 */ { UD_Ifdivr, O_ST6, O_ST0, O_NONE, O_NONE, P_none }, + /* 0412 */ { UD_Ifdivr, O_ST7, O_ST0, O_NONE, O_NONE, P_none }, + /* 0413 */ { UD_Ifdivr, O_Md, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0414 */ { UD_Ifdivr, O_ST0, O_ST0, O_NONE, O_NONE, P_none }, + /* 0415 */ { UD_Ifdivr, O_ST0, O_ST1, O_NONE, O_NONE, P_none }, + /* 0416 */ { UD_Ifdivr, O_ST0, O_ST2, O_NONE, O_NONE, P_none }, + /* 0417 */ { UD_Ifdivr, O_ST0, O_ST3, O_NONE, O_NONE, P_none }, + /* 0418 */ { UD_Ifdivr, O_ST0, O_ST4, O_NONE, O_NONE, P_none }, + /* 0419 */ { UD_Ifdivr, O_ST0, O_ST5, O_NONE, O_NONE, P_none }, + /* 0420 */ { UD_Ifdivr, O_ST0, O_ST6, O_NONE, O_NONE, P_none }, + /* 0421 */ { UD_Ifdivr, O_ST0, O_ST7, O_NONE, O_NONE, P_none }, + /* 0422 */ { UD_Ifdivrp, O_ST0, O_ST0, O_NONE, O_NONE, P_none }, + /* 0423 */ { UD_Ifdivrp, O_ST1, O_ST0, O_NONE, O_NONE, P_none }, + /* 0424 */ { UD_Ifdivrp, O_ST2, O_ST0, O_NONE, O_NONE, P_none }, + /* 0425 */ { UD_Ifdivrp, O_ST3, O_ST0, O_NONE, O_NONE, P_none }, + /* 0426 */ { UD_Ifdivrp, O_ST4, O_ST0, O_NONE, O_NONE, P_none }, + /* 0427 */ { UD_Ifdivrp, O_ST5, O_ST0, O_NONE, O_NONE, P_none }, + /* 0428 */ { UD_Ifdivrp, O_ST6, O_ST0, O_NONE, O_NONE, P_none }, + /* 0429 */ { UD_Ifdivrp, O_ST7, O_ST0, O_NONE, O_NONE, P_none }, + /* 0430 */ { UD_Ifemms, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 0431 */ { UD_Iffree, O_ST0, O_NONE, O_NONE, O_NONE, P_none }, + /* 0432 */ { UD_Iffree, O_ST1, O_NONE, O_NONE, O_NONE, P_none }, + /* 0433 */ { UD_Iffree, O_ST2, O_NONE, O_NONE, O_NONE, P_none }, + /* 0434 */ { UD_Iffree, O_ST3, O_NONE, O_NONE, O_NONE, P_none }, + /* 0435 */ { UD_Iffree, O_ST4, O_NONE, O_NONE, O_NONE, P_none }, + /* 0436 */ { UD_Iffree, O_ST5, O_NONE, O_NONE, O_NONE, P_none }, + /* 0437 */ { UD_Iffree, O_ST6, O_NONE, O_NONE, O_NONE, P_none }, + /* 0438 */ { UD_Iffree, O_ST7, O_NONE, O_NONE, O_NONE, P_none }, + /* 0439 */ { UD_Iffreep, O_ST0, O_NONE, O_NONE, O_NONE, P_none }, + /* 0440 */ { UD_Iffreep, O_ST1, O_NONE, O_NONE, O_NONE, P_none }, + /* 0441 */ { UD_Iffreep, O_ST2, O_NONE, O_NONE, O_NONE, P_none }, + /* 0442 */ { UD_Iffreep, O_ST3, O_NONE, O_NONE, O_NONE, P_none }, + /* 0443 */ { UD_Iffreep, O_ST4, O_NONE, O_NONE, O_NONE, P_none }, + /* 0444 */ { UD_Iffreep, O_ST5, O_NONE, O_NONE, O_NONE, P_none }, + /* 0445 */ { UD_Iffreep, O_ST6, O_NONE, O_NONE, O_NONE, P_none }, + /* 0446 */ { UD_Iffreep, O_ST7, O_NONE, O_NONE, O_NONE, P_none }, + /* 0447 */ { UD_Ificom, O_Mw, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0448 */ { UD_Ificom, O_Md, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0449 */ { UD_Ificomp, O_Mw, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0450 */ { UD_Ificomp, O_Md, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0451 */ { UD_Ifild, O_Mw, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0452 */ { UD_Ifild, O_Mq, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0453 */ { UD_Ifild, O_Md, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0454 */ { UD_Ifincstp, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 0455 */ { UD_Ifninit, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 0456 */ { UD_Ifiadd, O_Md, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0457 */ { UD_Ifiadd, O_Mw, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0458 */ { UD_Ifidivr, O_Md, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0459 */ { UD_Ifidivr, O_Mw, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0460 */ { UD_Ifidiv, O_Md, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0461 */ { UD_Ifidiv, O_Mw, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0462 */ { UD_Ifisub, O_Md, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0463 */ { UD_Ifisub, O_Mw, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0464 */ { UD_Ifisubr, O_Md, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0465 */ { UD_Ifisubr, O_Mw, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0466 */ { UD_Ifist, O_Mw, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0467 */ { UD_Ifist, O_Md, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0468 */ { UD_Ifistp, O_Mw, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0469 */ { UD_Ifistp, O_Mq, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0470 */ { UD_Ifistp, O_Md, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0471 */ { UD_Ifisttp, O_Md, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0472 */ { UD_Ifisttp, O_Mq, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0473 */ { UD_Ifisttp, O_Mw, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0474 */ { UD_Ifld, O_Mt, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0475 */ { UD_Ifld, O_Mq, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0476 */ { UD_Ifld, O_Md, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0477 */ { UD_Ifld, O_ST0, O_NONE, O_NONE, O_NONE, P_none }, + /* 0478 */ { UD_Ifld, O_ST1, O_NONE, O_NONE, O_NONE, P_none }, + /* 0479 */ { UD_Ifld, O_ST2, O_NONE, O_NONE, O_NONE, P_none }, + /* 0480 */ { UD_Ifld, O_ST3, O_NONE, O_NONE, O_NONE, P_none }, + /* 0481 */ { UD_Ifld, O_ST4, O_NONE, O_NONE, O_NONE, P_none }, + /* 0482 */ { UD_Ifld, O_ST5, O_NONE, O_NONE, O_NONE, P_none }, + /* 0483 */ { UD_Ifld, O_ST6, O_NONE, O_NONE, O_NONE, P_none }, + /* 0484 */ { UD_Ifld, O_ST7, O_NONE, O_NONE, O_NONE, P_none }, + /* 0485 */ { UD_Ifld1, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 0486 */ { UD_Ifldl2t, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 0487 */ { UD_Ifldl2e, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 0488 */ { UD_Ifldpi, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 0489 */ { UD_Ifldlg2, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 0490 */ { UD_Ifldln2, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 0491 */ { UD_Ifldz, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 0492 */ { UD_Ifldcw, O_Mw, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0493 */ { UD_Ifldenv, O_M, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0494 */ { UD_Ifmul, O_Mq, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0495 */ { UD_Ifmul, O_ST0, O_ST0, O_NONE, O_NONE, P_none }, + /* 0496 */ { UD_Ifmul, O_ST1, O_ST0, O_NONE, O_NONE, P_none }, + /* 0497 */ { UD_Ifmul, O_ST2, O_ST0, O_NONE, O_NONE, P_none }, + /* 0498 */ { UD_Ifmul, O_ST3, O_ST0, O_NONE, O_NONE, P_none }, + /* 0499 */ { UD_Ifmul, O_ST4, O_ST0, O_NONE, O_NONE, P_none }, + /* 0500 */ { UD_Ifmul, O_ST5, O_ST0, O_NONE, O_NONE, P_none }, + /* 0501 */ { UD_Ifmul, O_ST6, O_ST0, O_NONE, O_NONE, P_none }, + /* 0502 */ { UD_Ifmul, O_ST7, O_ST0, O_NONE, O_NONE, P_none }, + /* 0503 */ { UD_Ifmul, O_Md, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0504 */ { UD_Ifmul, O_ST0, O_ST0, O_NONE, O_NONE, P_none }, + /* 0505 */ { UD_Ifmul, O_ST0, O_ST1, O_NONE, O_NONE, P_none }, + /* 0506 */ { UD_Ifmul, O_ST0, O_ST2, O_NONE, O_NONE, P_none }, + /* 0507 */ { UD_Ifmul, O_ST0, O_ST3, O_NONE, O_NONE, P_none }, + /* 0508 */ { UD_Ifmul, O_ST0, O_ST4, O_NONE, O_NONE, P_none }, + /* 0509 */ { UD_Ifmul, O_ST0, O_ST5, O_NONE, O_NONE, P_none }, + /* 0510 */ { UD_Ifmul, O_ST0, O_ST6, O_NONE, O_NONE, P_none }, + /* 0511 */ { UD_Ifmul, O_ST0, O_ST7, O_NONE, O_NONE, P_none }, + /* 0512 */ { UD_Ifmulp, O_ST0, O_ST0, O_NONE, O_NONE, P_none }, + /* 0513 */ { UD_Ifmulp, O_ST1, O_ST0, O_NONE, O_NONE, P_none }, + /* 0514 */ { UD_Ifmulp, O_ST2, O_ST0, O_NONE, O_NONE, P_none }, + /* 0515 */ { UD_Ifmulp, O_ST3, O_ST0, O_NONE, O_NONE, P_none }, + /* 0516 */ { UD_Ifmulp, O_ST4, O_ST0, O_NONE, O_NONE, P_none }, + /* 0517 */ { UD_Ifmulp, O_ST5, O_ST0, O_NONE, O_NONE, P_none }, + /* 0518 */ { UD_Ifmulp, O_ST6, O_ST0, O_NONE, O_NONE, P_none }, + /* 0519 */ { UD_Ifmulp, O_ST7, O_ST0, O_NONE, O_NONE, P_none }, + /* 0520 */ { UD_Ifimul, O_Md, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0521 */ { UD_Ifimul, O_Mw, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0522 */ { UD_Ifnop, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 0523 */ { UD_Ifndisi, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 0524 */ { UD_Ifneni, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 0525 */ { UD_Ifnsetpm, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 0526 */ { UD_Ifpatan, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 0527 */ { UD_Ifprem, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 0528 */ { UD_Ifprem1, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 0529 */ { UD_Ifptan, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 0530 */ { UD_Ifrndint, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 0531 */ { UD_Ifrstor, O_M, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0532 */ { UD_Ifrstpm, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 0533 */ { UD_Ifnsave, O_M, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0534 */ { UD_Ifscale, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 0535 */ { UD_Ifsin, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 0536 */ { UD_Ifsincos, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 0537 */ { UD_Ifsqrt, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 0538 */ { UD_Ifstp, O_Mt, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0539 */ { UD_Ifstp, O_Mq, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0540 */ { UD_Ifstp, O_Md, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0541 */ { UD_Ifstp, O_ST0, O_NONE, O_NONE, O_NONE, P_none }, + /* 0542 */ { UD_Ifstp, O_ST1, O_NONE, O_NONE, O_NONE, P_none }, + /* 0543 */ { UD_Ifstp, O_ST2, O_NONE, O_NONE, O_NONE, P_none }, + /* 0544 */ { UD_Ifstp, O_ST3, O_NONE, O_NONE, O_NONE, P_none }, + /* 0545 */ { UD_Ifstp, O_ST4, O_NONE, O_NONE, O_NONE, P_none }, + /* 0546 */ { UD_Ifstp, O_ST5, O_NONE, O_NONE, O_NONE, P_none }, + /* 0547 */ { UD_Ifstp, O_ST6, O_NONE, O_NONE, O_NONE, P_none }, + /* 0548 */ { UD_Ifstp, O_ST7, O_NONE, O_NONE, O_NONE, P_none }, + /* 0549 */ { UD_Ifstp1, O_ST0, O_NONE, O_NONE, O_NONE, P_none }, + /* 0550 */ { UD_Ifstp1, O_ST1, O_NONE, O_NONE, O_NONE, P_none }, + /* 0551 */ { UD_Ifstp1, O_ST2, O_NONE, O_NONE, O_NONE, P_none }, + /* 0552 */ { UD_Ifstp1, O_ST3, O_NONE, O_NONE, O_NONE, P_none }, + /* 0553 */ { UD_Ifstp1, O_ST4, O_NONE, O_NONE, O_NONE, P_none }, + /* 0554 */ { UD_Ifstp1, O_ST5, O_NONE, O_NONE, O_NONE, P_none }, + /* 0555 */ { UD_Ifstp1, O_ST6, O_NONE, O_NONE, O_NONE, P_none }, + /* 0556 */ { UD_Ifstp1, O_ST7, O_NONE, O_NONE, O_NONE, P_none }, + /* 0557 */ { UD_Ifstp8, O_ST0, O_NONE, O_NONE, O_NONE, P_none }, + /* 0558 */ { UD_Ifstp8, O_ST1, O_NONE, O_NONE, O_NONE, P_none }, + /* 0559 */ { UD_Ifstp8, O_ST2, O_NONE, O_NONE, O_NONE, P_none }, + /* 0560 */ { UD_Ifstp8, O_ST3, O_NONE, O_NONE, O_NONE, P_none }, + /* 0561 */ { UD_Ifstp8, O_ST4, O_NONE, O_NONE, O_NONE, P_none }, + /* 0562 */ { UD_Ifstp8, O_ST5, O_NONE, O_NONE, O_NONE, P_none }, + /* 0563 */ { UD_Ifstp8, O_ST6, O_NONE, O_NONE, O_NONE, P_none }, + /* 0564 */ { UD_Ifstp8, O_ST7, O_NONE, O_NONE, O_NONE, P_none }, + /* 0565 */ { UD_Ifstp9, O_ST0, O_NONE, O_NONE, O_NONE, P_none }, + /* 0566 */ { UD_Ifstp9, O_ST1, O_NONE, O_NONE, O_NONE, P_none }, + /* 0567 */ { UD_Ifstp9, O_ST2, O_NONE, O_NONE, O_NONE, P_none }, + /* 0568 */ { UD_Ifstp9, O_ST3, O_NONE, O_NONE, O_NONE, P_none }, + /* 0569 */ { UD_Ifstp9, O_ST4, O_NONE, O_NONE, O_NONE, P_none }, + /* 0570 */ { UD_Ifstp9, O_ST5, O_NONE, O_NONE, O_NONE, P_none }, + /* 0571 */ { UD_Ifstp9, O_ST6, O_NONE, O_NONE, O_NONE, P_none }, + /* 0572 */ { UD_Ifstp9, O_ST7, O_NONE, O_NONE, O_NONE, P_none }, + /* 0573 */ { UD_Ifst, O_Md, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0574 */ { UD_Ifst, O_Mq, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0575 */ { UD_Ifst, O_ST0, O_NONE, O_NONE, O_NONE, P_none }, + /* 0576 */ { UD_Ifst, O_ST1, O_NONE, O_NONE, O_NONE, P_none }, + /* 0577 */ { UD_Ifst, O_ST2, O_NONE, O_NONE, O_NONE, P_none }, + /* 0578 */ { UD_Ifst, O_ST3, O_NONE, O_NONE, O_NONE, P_none }, + /* 0579 */ { UD_Ifst, O_ST4, O_NONE, O_NONE, O_NONE, P_none }, + /* 0580 */ { UD_Ifst, O_ST5, O_NONE, O_NONE, O_NONE, P_none }, + /* 0581 */ { UD_Ifst, O_ST6, O_NONE, O_NONE, O_NONE, P_none }, + /* 0582 */ { UD_Ifst, O_ST7, O_NONE, O_NONE, O_NONE, P_none }, + /* 0583 */ { UD_Ifnstcw, O_Mw, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0584 */ { UD_Ifnstenv, O_M, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0585 */ { UD_Ifnstsw, O_Mw, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0586 */ { UD_Ifnstsw, O_AX, O_NONE, O_NONE, O_NONE, P_none }, + /* 0587 */ { UD_Ifsub, O_Md, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0588 */ { UD_Ifsub, O_Mq, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0589 */ { UD_Ifsub, O_ST0, O_ST0, O_NONE, O_NONE, P_none }, + /* 0590 */ { UD_Ifsub, O_ST0, O_ST1, O_NONE, O_NONE, P_none }, + /* 0591 */ { UD_Ifsub, O_ST0, O_ST2, O_NONE, O_NONE, P_none }, + /* 0592 */ { UD_Ifsub, O_ST0, O_ST3, O_NONE, O_NONE, P_none }, + /* 0593 */ { UD_Ifsub, O_ST0, O_ST4, O_NONE, O_NONE, P_none }, + /* 0594 */ { UD_Ifsub, O_ST0, O_ST5, O_NONE, O_NONE, P_none }, + /* 0595 */ { UD_Ifsub, O_ST0, O_ST6, O_NONE, O_NONE, P_none }, + /* 0596 */ { UD_Ifsub, O_ST0, O_ST7, O_NONE, O_NONE, P_none }, + /* 0597 */ { UD_Ifsub, O_ST0, O_ST0, O_NONE, O_NONE, P_none }, + /* 0598 */ { UD_Ifsub, O_ST1, O_ST0, O_NONE, O_NONE, P_none }, + /* 0599 */ { UD_Ifsub, O_ST2, O_ST0, O_NONE, O_NONE, P_none }, + /* 0600 */ { UD_Ifsub, O_ST3, O_ST0, O_NONE, O_NONE, P_none }, + /* 0601 */ { UD_Ifsub, O_ST4, O_ST0, O_NONE, O_NONE, P_none }, + /* 0602 */ { UD_Ifsub, O_ST5, O_ST0, O_NONE, O_NONE, P_none }, + /* 0603 */ { UD_Ifsub, O_ST6, O_ST0, O_NONE, O_NONE, P_none }, + /* 0604 */ { UD_Ifsub, O_ST7, O_ST0, O_NONE, O_NONE, P_none }, + /* 0605 */ { UD_Ifsubp, O_ST0, O_ST0, O_NONE, O_NONE, P_none }, + /* 0606 */ { UD_Ifsubp, O_ST1, O_ST0, O_NONE, O_NONE, P_none }, + /* 0607 */ { UD_Ifsubp, O_ST2, O_ST0, O_NONE, O_NONE, P_none }, + /* 0608 */ { UD_Ifsubp, O_ST3, O_ST0, O_NONE, O_NONE, P_none }, + /* 0609 */ { UD_Ifsubp, O_ST4, O_ST0, O_NONE, O_NONE, P_none }, + /* 0610 */ { UD_Ifsubp, O_ST5, O_ST0, O_NONE, O_NONE, P_none }, + /* 0611 */ { UD_Ifsubp, O_ST6, O_ST0, O_NONE, O_NONE, P_none }, + /* 0612 */ { UD_Ifsubp, O_ST7, O_ST0, O_NONE, O_NONE, P_none }, + /* 0613 */ { UD_Ifsubr, O_Mq, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0614 */ { UD_Ifsubr, O_ST0, O_ST0, O_NONE, O_NONE, P_none }, + /* 0615 */ { UD_Ifsubr, O_ST0, O_ST1, O_NONE, O_NONE, P_none }, + /* 0616 */ { UD_Ifsubr, O_ST0, O_ST2, O_NONE, O_NONE, P_none }, + /* 0617 */ { UD_Ifsubr, O_ST0, O_ST3, O_NONE, O_NONE, P_none }, + /* 0618 */ { UD_Ifsubr, O_ST0, O_ST4, O_NONE, O_NONE, P_none }, + /* 0619 */ { UD_Ifsubr, O_ST0, O_ST5, O_NONE, O_NONE, P_none }, + /* 0620 */ { UD_Ifsubr, O_ST0, O_ST6, O_NONE, O_NONE, P_none }, + /* 0621 */ { UD_Ifsubr, O_ST0, O_ST7, O_NONE, O_NONE, P_none }, + /* 0622 */ { UD_Ifsubr, O_ST0, O_ST0, O_NONE, O_NONE, P_none }, + /* 0623 */ { UD_Ifsubr, O_ST1, O_ST0, O_NONE, O_NONE, P_none }, + /* 0624 */ { UD_Ifsubr, O_ST2, O_ST0, O_NONE, O_NONE, P_none }, + /* 0625 */ { UD_Ifsubr, O_ST3, O_ST0, O_NONE, O_NONE, P_none }, + /* 0626 */ { UD_Ifsubr, O_ST4, O_ST0, O_NONE, O_NONE, P_none }, + /* 0627 */ { UD_Ifsubr, O_ST5, O_ST0, O_NONE, O_NONE, P_none }, + /* 0628 */ { UD_Ifsubr, O_ST6, O_ST0, O_NONE, O_NONE, P_none }, + /* 0629 */ { UD_Ifsubr, O_ST7, O_ST0, O_NONE, O_NONE, P_none }, + /* 0630 */ { UD_Ifsubr, O_Md, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0631 */ { UD_Ifsubrp, O_ST0, O_ST0, O_NONE, O_NONE, P_none }, + /* 0632 */ { UD_Ifsubrp, O_ST1, O_ST0, O_NONE, O_NONE, P_none }, + /* 0633 */ { UD_Ifsubrp, O_ST2, O_ST0, O_NONE, O_NONE, P_none }, + /* 0634 */ { UD_Ifsubrp, O_ST3, O_ST0, O_NONE, O_NONE, P_none }, + /* 0635 */ { UD_Ifsubrp, O_ST4, O_ST0, O_NONE, O_NONE, P_none }, + /* 0636 */ { UD_Ifsubrp, O_ST5, O_ST0, O_NONE, O_NONE, P_none }, + /* 0637 */ { UD_Ifsubrp, O_ST6, O_ST0, O_NONE, O_NONE, P_none }, + /* 0638 */ { UD_Ifsubrp, O_ST7, O_ST0, O_NONE, O_NONE, P_none }, + /* 0639 */ { UD_Iftst, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 0640 */ { UD_Ifucom, O_ST0, O_NONE, O_NONE, O_NONE, P_none }, + /* 0641 */ { UD_Ifucom, O_ST1, O_NONE, O_NONE, O_NONE, P_none }, + /* 0642 */ { UD_Ifucom, O_ST2, O_NONE, O_NONE, O_NONE, P_none }, + /* 0643 */ { UD_Ifucom, O_ST3, O_NONE, O_NONE, O_NONE, P_none }, + /* 0644 */ { UD_Ifucom, O_ST4, O_NONE, O_NONE, O_NONE, P_none }, + /* 0645 */ { UD_Ifucom, O_ST5, O_NONE, O_NONE, O_NONE, P_none }, + /* 0646 */ { UD_Ifucom, O_ST6, O_NONE, O_NONE, O_NONE, P_none }, + /* 0647 */ { UD_Ifucom, O_ST7, O_NONE, O_NONE, O_NONE, P_none }, + /* 0648 */ { UD_Ifucomp, O_ST0, O_NONE, O_NONE, O_NONE, P_none }, + /* 0649 */ { UD_Ifucomp, O_ST1, O_NONE, O_NONE, O_NONE, P_none }, + /* 0650 */ { UD_Ifucomp, O_ST2, O_NONE, O_NONE, O_NONE, P_none }, + /* 0651 */ { UD_Ifucomp, O_ST3, O_NONE, O_NONE, O_NONE, P_none }, + /* 0652 */ { UD_Ifucomp, O_ST4, O_NONE, O_NONE, O_NONE, P_none }, + /* 0653 */ { UD_Ifucomp, O_ST5, O_NONE, O_NONE, O_NONE, P_none }, + /* 0654 */ { UD_Ifucomp, O_ST6, O_NONE, O_NONE, O_NONE, P_none }, + /* 0655 */ { UD_Ifucomp, O_ST7, O_NONE, O_NONE, O_NONE, P_none }, + /* 0656 */ { UD_Ifucompp, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 0657 */ { UD_Ifxam, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 0658 */ { UD_Ifxch, O_ST0, O_ST0, O_NONE, O_NONE, P_none }, + /* 0659 */ { UD_Ifxch, O_ST0, O_ST1, O_NONE, O_NONE, P_none }, + /* 0660 */ { UD_Ifxch, O_ST0, O_ST2, O_NONE, O_NONE, P_none }, + /* 0661 */ { UD_Ifxch, O_ST0, O_ST3, O_NONE, O_NONE, P_none }, + /* 0662 */ { UD_Ifxch, O_ST0, O_ST4, O_NONE, O_NONE, P_none }, + /* 0663 */ { UD_Ifxch, O_ST0, O_ST5, O_NONE, O_NONE, P_none }, + /* 0664 */ { UD_Ifxch, O_ST0, O_ST6, O_NONE, O_NONE, P_none }, + /* 0665 */ { UD_Ifxch, O_ST0, O_ST7, O_NONE, O_NONE, P_none }, + /* 0666 */ { UD_Ifxch4, O_ST0, O_NONE, O_NONE, O_NONE, P_none }, + /* 0667 */ { UD_Ifxch4, O_ST1, O_NONE, O_NONE, O_NONE, P_none }, + /* 0668 */ { UD_Ifxch4, O_ST2, O_NONE, O_NONE, O_NONE, P_none }, + /* 0669 */ { UD_Ifxch4, O_ST3, O_NONE, O_NONE, O_NONE, P_none }, + /* 0670 */ { UD_Ifxch4, O_ST4, O_NONE, O_NONE, O_NONE, P_none }, + /* 0671 */ { UD_Ifxch4, O_ST5, O_NONE, O_NONE, O_NONE, P_none }, + /* 0672 */ { UD_Ifxch4, O_ST6, O_NONE, O_NONE, O_NONE, P_none }, + /* 0673 */ { UD_Ifxch4, O_ST7, O_NONE, O_NONE, O_NONE, P_none }, + /* 0674 */ { UD_Ifxch7, O_ST0, O_NONE, O_NONE, O_NONE, P_none }, + /* 0675 */ { UD_Ifxch7, O_ST1, O_NONE, O_NONE, O_NONE, P_none }, + /* 0676 */ { UD_Ifxch7, O_ST2, O_NONE, O_NONE, O_NONE, P_none }, + /* 0677 */ { UD_Ifxch7, O_ST3, O_NONE, O_NONE, O_NONE, P_none }, + /* 0678 */ { UD_Ifxch7, O_ST4, O_NONE, O_NONE, O_NONE, P_none }, + /* 0679 */ { UD_Ifxch7, O_ST5, O_NONE, O_NONE, O_NONE, P_none }, + /* 0680 */ { UD_Ifxch7, O_ST6, O_NONE, O_NONE, O_NONE, P_none }, + /* 0681 */ { UD_Ifxch7, O_ST7, O_NONE, O_NONE, O_NONE, P_none }, + /* 0682 */ { UD_Ifxrstor, O_M, O_NONE, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0683 */ { UD_Ifxsave, O_M, O_NONE, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0684 */ { UD_Ifxtract, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 0685 */ { UD_Ifyl2x, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 0686 */ { UD_Ifyl2xp1, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 0687 */ { UD_Ihlt, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 0688 */ { UD_Iidiv, O_Ev, O_NONE, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0689 */ { UD_Iidiv, O_Eb, O_NONE, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0690 */ { UD_Iin, O_AL, O_Ib, O_NONE, O_NONE, P_none }, + /* 0691 */ { UD_Iin, O_eAX, O_Ib, O_NONE, O_NONE, P_oso }, + /* 0692 */ { UD_Iin, O_AL, O_DX, O_NONE, O_NONE, P_none }, + /* 0693 */ { UD_Iin, O_eAX, O_DX, O_NONE, O_NONE, P_oso }, + /* 0694 */ { UD_Iimul, O_Gv, O_Ev, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0695 */ { UD_Iimul, O_Eb, O_NONE, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0696 */ { UD_Iimul, O_Ev, O_NONE, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0697 */ { UD_Iimul, O_Gv, O_Ev, O_Iz, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0698 */ { UD_Iimul, O_Gv, O_Ev, O_sIb, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0699 */ { UD_Iinc, O_R0z, O_NONE, O_NONE, O_NONE, P_oso }, + /* 0700 */ { UD_Iinc, O_R1z, O_NONE, O_NONE, O_NONE, P_oso }, + /* 0701 */ { UD_Iinc, O_R2z, O_NONE, O_NONE, O_NONE, P_oso }, + /* 0702 */ { UD_Iinc, O_R3z, O_NONE, O_NONE, O_NONE, P_oso }, + /* 0703 */ { UD_Iinc, O_R4z, O_NONE, O_NONE, O_NONE, P_oso }, + /* 0704 */ { UD_Iinc, O_R5z, O_NONE, O_NONE, O_NONE, P_oso }, + /* 0705 */ { UD_Iinc, O_R6z, O_NONE, O_NONE, O_NONE, P_oso }, + /* 0706 */ { UD_Iinc, O_R7z, O_NONE, O_NONE, O_NONE, P_oso }, + /* 0707 */ { UD_Iinc, O_Ev, O_NONE, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0708 */ { UD_Iinc, O_Eb, O_NONE, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0709 */ { UD_Iinsb, O_NONE, O_NONE, O_NONE, O_NONE, P_str|P_seg }, + /* 0710 */ { UD_Iinsw, O_NONE, O_NONE, O_NONE, O_NONE, P_str|P_oso|P_seg }, + /* 0711 */ { UD_Iinsd, O_NONE, O_NONE, O_NONE, O_NONE, P_str|P_oso|P_seg }, + /* 0712 */ { UD_Iint1, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 0713 */ { UD_Iint3, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 0714 */ { UD_Iint, O_Ib, O_NONE, O_NONE, O_NONE, P_none }, + /* 0715 */ { UD_Iinto, O_NONE, O_NONE, O_NONE, O_NONE, P_inv64 }, + /* 0716 */ { UD_Iinvd, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 0717 */ { UD_Iinvept, O_Gd, O_Mo, O_NONE, O_NONE, P_none }, + /* 0718 */ { UD_Iinvept, O_Gq, O_Mo, O_NONE, O_NONE, P_none }, + /* 0719 */ { UD_Iinvlpg, O_M, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0720 */ { UD_Iinvlpga, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 0721 */ { UD_Iinvvpid, O_Gd, O_Mo, O_NONE, O_NONE, P_none }, + /* 0722 */ { UD_Iinvvpid, O_Gq, O_Mo, O_NONE, O_NONE, P_none }, + /* 0723 */ { UD_Iiretw, O_NONE, O_NONE, O_NONE, O_NONE, P_oso|P_rexw }, + /* 0724 */ { UD_Iiretd, O_NONE, O_NONE, O_NONE, O_NONE, P_oso|P_rexw }, + /* 0725 */ { UD_Iiretq, O_NONE, O_NONE, O_NONE, O_NONE, P_oso|P_rexw }, + /* 0726 */ { UD_Ijo, O_Jb, O_NONE, O_NONE, O_NONE, P_none }, + /* 0727 */ { UD_Ijo, O_Jz, O_NONE, O_NONE, O_NONE, P_oso|P_def64 }, + /* 0728 */ { UD_Ijno, O_Jb, O_NONE, O_NONE, O_NONE, P_none }, + /* 0729 */ { UD_Ijno, O_Jz, O_NONE, O_NONE, O_NONE, P_oso|P_def64 }, + /* 0730 */ { UD_Ijb, O_Jb, O_NONE, O_NONE, O_NONE, P_none }, + /* 0731 */ { UD_Ijb, O_Jz, O_NONE, O_NONE, O_NONE, P_oso|P_def64 }, + /* 0732 */ { UD_Ijae, O_Jb, O_NONE, O_NONE, O_NONE, P_none }, + /* 0733 */ { UD_Ijae, O_Jz, O_NONE, O_NONE, O_NONE, P_oso|P_def64 }, + /* 0734 */ { UD_Ijz, O_Jb, O_NONE, O_NONE, O_NONE, P_none }, + /* 0735 */ { UD_Ijz, O_Jz, O_NONE, O_NONE, O_NONE, P_oso|P_def64 }, + /* 0736 */ { UD_Ijnz, O_Jb, O_NONE, O_NONE, O_NONE, P_none }, + /* 0737 */ { UD_Ijnz, O_Jz, O_NONE, O_NONE, O_NONE, P_oso|P_def64 }, + /* 0738 */ { UD_Ijbe, O_Jb, O_NONE, O_NONE, O_NONE, P_none }, + /* 0739 */ { UD_Ijbe, O_Jz, O_NONE, O_NONE, O_NONE, P_oso|P_def64 }, + /* 0740 */ { UD_Ija, O_Jb, O_NONE, O_NONE, O_NONE, P_none }, + /* 0741 */ { UD_Ija, O_Jz, O_NONE, O_NONE, O_NONE, P_oso|P_def64 }, + /* 0742 */ { UD_Ijs, O_Jb, O_NONE, O_NONE, O_NONE, P_none }, + /* 0743 */ { UD_Ijs, O_Jz, O_NONE, O_NONE, O_NONE, P_oso|P_def64 }, + /* 0744 */ { UD_Ijns, O_Jb, O_NONE, O_NONE, O_NONE, P_none }, + /* 0745 */ { UD_Ijns, O_Jz, O_NONE, O_NONE, O_NONE, P_oso|P_def64 }, + /* 0746 */ { UD_Ijp, O_Jb, O_NONE, O_NONE, O_NONE, P_none }, + /* 0747 */ { UD_Ijp, O_Jz, O_NONE, O_NONE, O_NONE, P_oso|P_def64 }, + /* 0748 */ { UD_Ijnp, O_Jb, O_NONE, O_NONE, O_NONE, P_none }, + /* 0749 */ { UD_Ijnp, O_Jz, O_NONE, O_NONE, O_NONE, P_oso|P_def64 }, + /* 0750 */ { UD_Ijl, O_Jb, O_NONE, O_NONE, O_NONE, P_none }, + /* 0751 */ { UD_Ijl, O_Jz, O_NONE, O_NONE, O_NONE, P_oso|P_def64 }, + /* 0752 */ { UD_Ijge, O_Jb, O_NONE, O_NONE, O_NONE, P_none }, + /* 0753 */ { UD_Ijge, O_Jz, O_NONE, O_NONE, O_NONE, P_oso|P_def64 }, + /* 0754 */ { UD_Ijle, O_Jb, O_NONE, O_NONE, O_NONE, P_none }, + /* 0755 */ { UD_Ijle, O_Jz, O_NONE, O_NONE, O_NONE, P_oso|P_def64 }, + /* 0756 */ { UD_Ijg, O_Jb, O_NONE, O_NONE, O_NONE, P_none }, + /* 0757 */ { UD_Ijg, O_Jz, O_NONE, O_NONE, O_NONE, P_oso|P_def64 }, + /* 0758 */ { UD_Ijcxz, O_Jb, O_NONE, O_NONE, O_NONE, P_aso }, + /* 0759 */ { UD_Ijecxz, O_Jb, O_NONE, O_NONE, O_NONE, P_aso }, + /* 0760 */ { UD_Ijrcxz, O_Jb, O_NONE, O_NONE, O_NONE, P_aso }, + /* 0761 */ { UD_Ijmp, O_Ev, O_NONE, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb|P_def64 }, + /* 0762 */ { UD_Ijmp, O_Fv, O_NONE, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0763 */ { UD_Ijmp, O_Jz, O_NONE, O_NONE, O_NONE, P_oso|P_def64 }, + /* 0764 */ { UD_Ijmp, O_Av, O_NONE, O_NONE, O_NONE, P_oso }, + /* 0765 */ { UD_Ijmp, O_Jb, O_NONE, O_NONE, O_NONE, P_def64 }, + /* 0766 */ { UD_Ilahf, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 0767 */ { UD_Ilar, O_Gv, O_Ew, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0768 */ { UD_Ildmxcsr, O_Md, O_NONE, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0769 */ { UD_Ilds, O_Gv, O_M, O_NONE, O_NONE, P_aso|P_oso }, + /* 0770 */ { UD_Ilea, O_Gv, O_M, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0771 */ { UD_Iles, O_Gv, O_M, O_NONE, O_NONE, P_aso|P_oso }, + /* 0772 */ { UD_Ilfs, O_Gz, O_M, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0773 */ { UD_Ilgs, O_Gz, O_M, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0774 */ { UD_Ilidt, O_M, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0775 */ { UD_Ilss, O_Gv, O_M, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0776 */ { UD_Ileave, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 0777 */ { UD_Ilfence, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 0778 */ { UD_Ilfence, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 0779 */ { UD_Ilfence, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 0780 */ { UD_Ilfence, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 0781 */ { UD_Ilfence, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 0782 */ { UD_Ilfence, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 0783 */ { UD_Ilfence, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 0784 */ { UD_Ilfence, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 0785 */ { UD_Ilgdt, O_M, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0786 */ { UD_Illdt, O_Ew, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0787 */ { UD_Ilmsw, O_Ew, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0788 */ { UD_Ilmsw, O_Ew, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0789 */ { UD_Ilock, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 0790 */ { UD_Ilodsb, O_NONE, O_NONE, O_NONE, O_NONE, P_str|P_seg }, + /* 0791 */ { UD_Ilodsw, O_NONE, O_NONE, O_NONE, O_NONE, P_str|P_seg|P_oso|P_rexw }, + /* 0792 */ { UD_Ilodsd, O_NONE, O_NONE, O_NONE, O_NONE, P_str|P_seg|P_oso|P_rexw }, + /* 0793 */ { UD_Ilodsq, O_NONE, O_NONE, O_NONE, O_NONE, P_str|P_seg|P_oso|P_rexw }, + /* 0794 */ { UD_Iloopne, O_Jb, O_NONE, O_NONE, O_NONE, P_none }, + /* 0795 */ { UD_Iloope, O_Jb, O_NONE, O_NONE, O_NONE, P_none }, + /* 0796 */ { UD_Iloop, O_Jb, O_NONE, O_NONE, O_NONE, P_none }, + /* 0797 */ { UD_Ilsl, O_Gv, O_Ew, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0798 */ { UD_Iltr, O_Ew, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0799 */ { UD_Imaskmovq, O_P, O_N, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0800 */ { UD_Imaxpd, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0801 */ { UD_Ivmaxpd, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 0802 */ { UD_Imaxps, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0803 */ { UD_Ivmaxps, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 0804 */ { UD_Imaxsd, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0805 */ { UD_Ivmaxsd, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0806 */ { UD_Imaxss, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0807 */ { UD_Ivmaxss, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0808 */ { UD_Imfence, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 0809 */ { UD_Imfence, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 0810 */ { UD_Imfence, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 0811 */ { UD_Imfence, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 0812 */ { UD_Imfence, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 0813 */ { UD_Imfence, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 0814 */ { UD_Imfence, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 0815 */ { UD_Imfence, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 0816 */ { UD_Iminpd, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0817 */ { UD_Ivminpd, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 0818 */ { UD_Iminps, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0819 */ { UD_Ivminps, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0820 */ { UD_Iminsd, O_V, O_MqU, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0821 */ { UD_Ivminsd, O_Vx, O_Hx, O_MqU, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0822 */ { UD_Iminss, O_V, O_MdU, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0823 */ { UD_Ivminss, O_Vx, O_Hx, O_MdU, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0824 */ { UD_Imonitor, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 0825 */ { UD_Imontmul, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 0826 */ { UD_Imov, O_Eb, O_Ib, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0827 */ { UD_Imov, O_Ev, O_sIz, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0828 */ { UD_Imov, O_Eb, O_Gb, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0829 */ { UD_Imov, O_Ev, O_Gv, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0830 */ { UD_Imov, O_Gb, O_Eb, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0831 */ { UD_Imov, O_Gv, O_Ev, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0832 */ { UD_Imov, O_MwRv, O_S, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0833 */ { UD_Imov, O_S, O_MwRv, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0834 */ { UD_Imov, O_AL, O_Ob, O_NONE, O_NONE, P_none }, + /* 0835 */ { UD_Imov, O_rAX, O_Ov, O_NONE, O_NONE, P_aso|P_oso|P_rexw }, + /* 0836 */ { UD_Imov, O_Ob, O_AL, O_NONE, O_NONE, P_none }, + /* 0837 */ { UD_Imov, O_Ov, O_rAX, O_NONE, O_NONE, P_aso|P_oso|P_rexw }, + /* 0838 */ { UD_Imov, O_R0b, O_Ib, O_NONE, O_NONE, P_rexb }, + /* 0839 */ { UD_Imov, O_R1b, O_Ib, O_NONE, O_NONE, P_rexb }, + /* 0840 */ { UD_Imov, O_R2b, O_Ib, O_NONE, O_NONE, P_rexb }, + /* 0841 */ { UD_Imov, O_R3b, O_Ib, O_NONE, O_NONE, P_rexb }, + /* 0842 */ { UD_Imov, O_R4b, O_Ib, O_NONE, O_NONE, P_rexb }, + /* 0843 */ { UD_Imov, O_R5b, O_Ib, O_NONE, O_NONE, P_rexb }, + /* 0844 */ { UD_Imov, O_R6b, O_Ib, O_NONE, O_NONE, P_rexb }, + /* 0845 */ { UD_Imov, O_R7b, O_Ib, O_NONE, O_NONE, P_rexb }, + /* 0846 */ { UD_Imov, O_R0v, O_Iv, O_NONE, O_NONE, P_oso|P_rexw|P_rexb }, + /* 0847 */ { UD_Imov, O_R1v, O_Iv, O_NONE, O_NONE, P_oso|P_rexw|P_rexb }, + /* 0848 */ { UD_Imov, O_R2v, O_Iv, O_NONE, O_NONE, P_oso|P_rexw|P_rexb }, + /* 0849 */ { UD_Imov, O_R3v, O_Iv, O_NONE, O_NONE, P_oso|P_rexw|P_rexb }, + /* 0850 */ { UD_Imov, O_R4v, O_Iv, O_NONE, O_NONE, P_oso|P_rexw|P_rexb }, + /* 0851 */ { UD_Imov, O_R5v, O_Iv, O_NONE, O_NONE, P_oso|P_rexw|P_rexb }, + /* 0852 */ { UD_Imov, O_R6v, O_Iv, O_NONE, O_NONE, P_oso|P_rexw|P_rexb }, + /* 0853 */ { UD_Imov, O_R7v, O_Iv, O_NONE, O_NONE, P_oso|P_rexw|P_rexb }, + /* 0854 */ { UD_Imov, O_R, O_C, O_NONE, O_NONE, P_rexr|P_rexw|P_rexb }, + /* 0855 */ { UD_Imov, O_R, O_D, O_NONE, O_NONE, P_rexr|P_rexw|P_rexb }, + /* 0856 */ { UD_Imov, O_C, O_R, O_NONE, O_NONE, P_rexr|P_rexw|P_rexb }, + /* 0857 */ { UD_Imov, O_D, O_R, O_NONE, O_NONE, P_rexr|P_rexw|P_rexb }, + /* 0858 */ { UD_Imovapd, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0859 */ { UD_Ivmovapd, O_Vx, O_Wx, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 0860 */ { UD_Imovapd, O_W, O_V, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0861 */ { UD_Ivmovapd, O_Wx, O_Vx, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 0862 */ { UD_Imovaps, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0863 */ { UD_Ivmovaps, O_Vx, O_Wx, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 0864 */ { UD_Imovaps, O_W, O_V, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0865 */ { UD_Ivmovaps, O_Wx, O_Vx, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 0866 */ { UD_Imovd, O_P, O_Ey, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0867 */ { UD_Imovd, O_P, O_Ey, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0868 */ { UD_Imovd, O_V, O_Ey, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0869 */ { UD_Ivmovd, O_Vx, O_Ey, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0870 */ { UD_Imovd, O_V, O_Ey, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0871 */ { UD_Ivmovd, O_Vx, O_Ey, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0872 */ { UD_Imovd, O_Ey, O_P, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0873 */ { UD_Imovd, O_Ey, O_P, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0874 */ { UD_Imovd, O_Ey, O_V, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0875 */ { UD_Ivmovd, O_Ey, O_Vx, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0876 */ { UD_Imovd, O_Ey, O_V, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0877 */ { UD_Ivmovd, O_Ey, O_Vx, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0878 */ { UD_Imovhpd, O_V, O_M, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0879 */ { UD_Ivmovhpd, O_Vx, O_Hx, O_M, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0880 */ { UD_Imovhpd, O_M, O_V, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0881 */ { UD_Ivmovhpd, O_M, O_Vx, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0882 */ { UD_Imovhps, O_V, O_M, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0883 */ { UD_Ivmovhps, O_Vx, O_Hx, O_M, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0884 */ { UD_Imovhps, O_M, O_V, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0885 */ { UD_Ivmovhps, O_M, O_Vx, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0886 */ { UD_Imovlhps, O_V, O_U, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0887 */ { UD_Ivmovlhps, O_Vx, O_Hx, O_Ux, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0888 */ { UD_Imovlpd, O_V, O_M, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0889 */ { UD_Ivmovlpd, O_Vx, O_M, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0890 */ { UD_Imovlpd, O_M, O_V, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0891 */ { UD_Ivmovlpd, O_M, O_Vx, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0892 */ { UD_Imovlps, O_V, O_M, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0893 */ { UD_Ivmovlps, O_Vx, O_M, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0894 */ { UD_Imovlps, O_M, O_V, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0895 */ { UD_Ivmovlps, O_M, O_Vx, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0896 */ { UD_Imovhlps, O_V, O_U, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0897 */ { UD_Ivmovhlps, O_Vx, O_Ux, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0898 */ { UD_Imovmskpd, O_Gd, O_U, O_NONE, O_NONE, P_oso|P_rexr|P_rexb }, + /* 0899 */ { UD_Ivmovmskpd, O_Gd, O_Ux, O_NONE, O_NONE, P_oso|P_rexr|P_rexb|P_vexl }, + /* 0900 */ { UD_Imovmskps, O_Gd, O_U, O_NONE, O_NONE, P_oso|P_rexr|P_rexb }, + /* 0901 */ { UD_Ivmovmskps, O_Gd, O_Ux, O_NONE, O_NONE, P_oso|P_rexr|P_rexb }, + /* 0902 */ { UD_Imovntdq, O_M, O_V, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0903 */ { UD_Ivmovntdq, O_M, O_Vx, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 0904 */ { UD_Imovnti, O_M, O_Gy, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0905 */ { UD_Imovntpd, O_M, O_V, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0906 */ { UD_Ivmovntpd, O_M, O_Vx, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 0907 */ { UD_Imovntps, O_M, O_V, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0908 */ { UD_Ivmovntps, O_M, O_Vx, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 0909 */ { UD_Imovntq, O_M, O_P, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0910 */ { UD_Imovq, O_P, O_Eq, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0911 */ { UD_Imovq, O_V, O_Eq, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0912 */ { UD_Ivmovq, O_Vx, O_Eq, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0913 */ { UD_Imovq, O_Eq, O_P, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0914 */ { UD_Imovq, O_Eq, O_V, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0915 */ { UD_Ivmovq, O_Eq, O_Vx, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0916 */ { UD_Imovq, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0917 */ { UD_Ivmovq, O_Vx, O_Wx, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0918 */ { UD_Imovq, O_W, O_V, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0919 */ { UD_Ivmovq, O_Wx, O_Vx, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0920 */ { UD_Imovq, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0921 */ { UD_Imovq, O_Q, O_P, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0922 */ { UD_Imovsb, O_NONE, O_NONE, O_NONE, O_NONE, P_str|P_seg }, + /* 0923 */ { UD_Imovsw, O_NONE, O_NONE, O_NONE, O_NONE, P_str|P_seg|P_oso|P_rexw }, + /* 0924 */ { UD_Imovsd, O_NONE, O_NONE, O_NONE, O_NONE, P_str|P_seg|P_oso|P_rexw }, + /* 0925 */ { UD_Imovsd, O_V, O_MqU, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0926 */ { UD_Imovsd, O_W, O_V, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0927 */ { UD_Imovsq, O_NONE, O_NONE, O_NONE, O_NONE, P_str|P_seg|P_oso|P_rexw }, + /* 0928 */ { UD_Imovss, O_V, O_MdU, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0929 */ { UD_Imovss, O_W, O_V, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0930 */ { UD_Imovsx, O_Gv, O_Eb, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0931 */ { UD_Imovsx, O_Gy, O_Ew, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0932 */ { UD_Imovupd, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0933 */ { UD_Ivmovupd, O_Vx, O_Wx, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 0934 */ { UD_Imovupd, O_W, O_V, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0935 */ { UD_Ivmovupd, O_Wx, O_Vx, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 0936 */ { UD_Imovups, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0937 */ { UD_Ivmovups, O_Vx, O_Wx, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 0938 */ { UD_Imovups, O_W, O_V, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0939 */ { UD_Ivmovups, O_Wx, O_Vx, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 0940 */ { UD_Imovzx, O_Gv, O_Eb, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0941 */ { UD_Imovzx, O_Gy, O_Ew, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0942 */ { UD_Imul, O_Eb, O_NONE, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0943 */ { UD_Imul, O_Ev, O_NONE, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0944 */ { UD_Imulpd, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0945 */ { UD_Ivmulpd, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 0946 */ { UD_Imulps, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0947 */ { UD_Ivmulps, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 0948 */ { UD_Imulsd, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0949 */ { UD_Ivmulsd, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0950 */ { UD_Imulss, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0951 */ { UD_Ivmulss, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0952 */ { UD_Imwait, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 0953 */ { UD_Ineg, O_Eb, O_NONE, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0954 */ { UD_Ineg, O_Ev, O_NONE, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0955 */ { UD_Inop, O_M, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0956 */ { UD_Inop, O_M, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0957 */ { UD_Inop, O_M, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0958 */ { UD_Inop, O_M, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0959 */ { UD_Inop, O_M, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0960 */ { UD_Inop, O_M, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0961 */ { UD_Inop, O_M, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0962 */ { UD_Inot, O_Eb, O_NONE, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0963 */ { UD_Inot, O_Ev, O_NONE, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0964 */ { UD_Ior, O_Eb, O_Gb, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0965 */ { UD_Ior, O_Ev, O_Gv, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0966 */ { UD_Ior, O_Gb, O_Eb, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0967 */ { UD_Ior, O_Gv, O_Ev, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0968 */ { UD_Ior, O_AL, O_Ib, O_NONE, O_NONE, P_none }, + /* 0969 */ { UD_Ior, O_rAX, O_sIz, O_NONE, O_NONE, P_oso|P_rexw }, + /* 0970 */ { UD_Ior, O_Eb, O_Ib, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0971 */ { UD_Ior, O_Ev, O_sIz, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0972 */ { UD_Ior, O_Eb, O_Ib, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0973 */ { UD_Ior, O_Ev, O_sIb, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 0974 */ { UD_Iorpd, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0975 */ { UD_Ivorpd, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 0976 */ { UD_Iorps, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0977 */ { UD_Ivorps, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 0978 */ { UD_Iout, O_Ib, O_AL, O_NONE, O_NONE, P_none }, + /* 0979 */ { UD_Iout, O_Ib, O_eAX, O_NONE, O_NONE, P_oso }, + /* 0980 */ { UD_Iout, O_DX, O_AL, O_NONE, O_NONE, P_none }, + /* 0981 */ { UD_Iout, O_DX, O_eAX, O_NONE, O_NONE, P_oso }, + /* 0982 */ { UD_Ioutsb, O_NONE, O_NONE, O_NONE, O_NONE, P_str|P_seg }, + /* 0983 */ { UD_Ioutsw, O_NONE, O_NONE, O_NONE, O_NONE, P_str|P_oso|P_seg }, + /* 0984 */ { UD_Ioutsd, O_NONE, O_NONE, O_NONE, O_NONE, P_str|P_oso|P_seg }, + /* 0985 */ { UD_Ipacksswb, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0986 */ { UD_Ivpacksswb, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 0987 */ { UD_Ipacksswb, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0988 */ { UD_Ipackssdw, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0989 */ { UD_Ivpackssdw, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 0990 */ { UD_Ipackssdw, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0991 */ { UD_Ipackuswb, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0992 */ { UD_Ivpackuswb, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 0993 */ { UD_Ipackuswb, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0994 */ { UD_Ipaddb, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0995 */ { UD_Ivpaddb, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 0996 */ { UD_Ipaddb, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0997 */ { UD_Ipaddw, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0998 */ { UD_Ipaddw, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 0999 */ { UD_Ivpaddw, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 1000 */ { UD_Ipaddd, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1001 */ { UD_Ipaddd, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1002 */ { UD_Ivpaddd, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 1003 */ { UD_Ipaddsb, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1004 */ { UD_Ipaddsb, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1005 */ { UD_Ivpaddsb, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1006 */ { UD_Ipaddsw, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1007 */ { UD_Ipaddsw, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1008 */ { UD_Ivpaddsw, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1009 */ { UD_Ipaddusb, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1010 */ { UD_Ipaddusb, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1011 */ { UD_Ivpaddusb, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1012 */ { UD_Ipaddusw, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1013 */ { UD_Ipaddusw, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1014 */ { UD_Ivpaddusw, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1015 */ { UD_Ipand, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1016 */ { UD_Ivpand, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1017 */ { UD_Ipand, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1018 */ { UD_Ipandn, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1019 */ { UD_Ivpandn, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1020 */ { UD_Ipandn, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1021 */ { UD_Ipavgb, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1022 */ { UD_Ivpavgb, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1023 */ { UD_Ipavgb, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1024 */ { UD_Ipavgw, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1025 */ { UD_Ivpavgw, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1026 */ { UD_Ipavgw, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1027 */ { UD_Ipcmpeqb, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1028 */ { UD_Ipcmpeqb, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1029 */ { UD_Ivpcmpeqb, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1030 */ { UD_Ipcmpeqw, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1031 */ { UD_Ipcmpeqw, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1032 */ { UD_Ivpcmpeqw, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1033 */ { UD_Ipcmpeqd, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1034 */ { UD_Ipcmpeqd, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1035 */ { UD_Ivpcmpeqd, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1036 */ { UD_Ipcmpgtb, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1037 */ { UD_Ivpcmpgtb, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1038 */ { UD_Ipcmpgtb, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1039 */ { UD_Ipcmpgtw, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1040 */ { UD_Ivpcmpgtw, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1041 */ { UD_Ipcmpgtw, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1042 */ { UD_Ipcmpgtd, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1043 */ { UD_Ivpcmpgtd, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1044 */ { UD_Ipcmpgtd, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1045 */ { UD_Ipextrb, O_MbRv, O_V, O_Ib, O_NONE, P_aso|P_rexx|P_rexr|P_rexb|P_def64 }, + /* 1046 */ { UD_Ivpextrb, O_MbRv, O_Vx, O_Ib, O_NONE, P_aso|P_rexx|P_rexr|P_rexb|P_def64 }, + /* 1047 */ { UD_Ipextrd, O_Ed, O_V, O_Ib, O_NONE, P_aso|P_rexr|P_rexx|P_rexw|P_rexb }, + /* 1048 */ { UD_Ivpextrd, O_Ed, O_Vx, O_Ib, O_NONE, P_aso|P_rexr|P_rexx|P_rexw|P_rexb }, + /* 1049 */ { UD_Ipextrd, O_Ed, O_V, O_Ib, O_NONE, P_aso|P_rexr|P_rexx|P_rexw|P_rexb }, + /* 1050 */ { UD_Ivpextrd, O_Ed, O_Vx, O_Ib, O_NONE, P_aso|P_rexr|P_rexx|P_rexw|P_rexb }, + /* 1051 */ { UD_Ipextrq, O_Eq, O_V, O_Ib, O_NONE, P_aso|P_rexr|P_rexw|P_rexb|P_def64 }, + /* 1052 */ { UD_Ivpextrq, O_Eq, O_Vx, O_Ib, O_NONE, P_aso|P_rexr|P_rexw|P_rexb|P_def64 }, + /* 1053 */ { UD_Ipextrw, O_Gd, O_U, O_Ib, O_NONE, P_aso|P_rexw|P_rexr|P_rexb }, + /* 1054 */ { UD_Ivpextrw, O_Gd, O_Ux, O_Ib, O_NONE, P_aso|P_rexw|P_rexr|P_rexb }, + /* 1055 */ { UD_Ipextrw, O_Gd, O_N, O_Ib, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1056 */ { UD_Ipextrw, O_MwRd, O_V, O_Ib, O_NONE, P_aso|P_rexw|P_rexx|P_rexr|P_rexb }, + /* 1057 */ { UD_Ivpextrw, O_MwRd, O_Vx, O_Ib, O_NONE, P_aso|P_rexw|P_rexx|P_rexr|P_rexb }, + /* 1058 */ { UD_Ipinsrb, O_V, O_MbRd, O_Ib, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1059 */ { UD_Ipinsrw, O_P, O_MwRy, O_Ib, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb|P_def64 }, + /* 1060 */ { UD_Ipinsrw, O_V, O_MwRy, O_Ib, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb|P_def64 }, + /* 1061 */ { UD_Ivpinsrw, O_Vx, O_MwRy, O_Ib, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb|P_def64 }, + /* 1062 */ { UD_Ipinsrd, O_V, O_Ed, O_Ib, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1063 */ { UD_Ipinsrd, O_V, O_Ed, O_Ib, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1064 */ { UD_Ipinsrq, O_V, O_Eq, O_Ib, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1065 */ { UD_Ivpinsrb, O_V, O_H, O_MbRd, O_Ib, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1066 */ { UD_Ivpinsrd, O_V, O_H, O_Ed, O_Ib, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1067 */ { UD_Ivpinsrd, O_V, O_H, O_Ed, O_Ib, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1068 */ { UD_Ivpinsrq, O_V, O_H, O_Eq, O_Ib, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1069 */ { UD_Ipmaddwd, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1070 */ { UD_Ipmaddwd, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1071 */ { UD_Ivpmaddwd, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1072 */ { UD_Ipmaxsw, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1073 */ { UD_Ivpmaxsw, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1074 */ { UD_Ipmaxsw, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1075 */ { UD_Ipmaxub, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1076 */ { UD_Ipmaxub, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1077 */ { UD_Ivpmaxub, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1078 */ { UD_Ipminsw, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1079 */ { UD_Ivpminsw, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1080 */ { UD_Ipminsw, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1081 */ { UD_Ipminub, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1082 */ { UD_Ivpminub, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1083 */ { UD_Ipminub, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1084 */ { UD_Ipmovmskb, O_Gd, O_U, O_NONE, O_NONE, P_oso|P_rexr|P_rexw|P_rexb }, + /* 1085 */ { UD_Ivpmovmskb, O_Gd, O_Ux, O_NONE, O_NONE, P_oso|P_rexr|P_rexw|P_rexb }, + /* 1086 */ { UD_Ipmovmskb, O_Gd, O_N, O_NONE, O_NONE, P_oso|P_rexr|P_rexw|P_rexb }, + /* 1087 */ { UD_Ipmulhuw, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1088 */ { UD_Ipmulhuw, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1089 */ { UD_Ivpmulhuw, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1090 */ { UD_Ipmulhw, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1091 */ { UD_Ivpmulhw, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1092 */ { UD_Ipmulhw, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1093 */ { UD_Ipmullw, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1094 */ { UD_Ipmullw, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1095 */ { UD_Ivpmullw, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1096 */ { UD_Ipop, O_ES, O_NONE, O_NONE, O_NONE, P_inv64 }, + /* 1097 */ { UD_Ipop, O_SS, O_NONE, O_NONE, O_NONE, P_inv64 }, + /* 1098 */ { UD_Ipop, O_DS, O_NONE, O_NONE, O_NONE, P_inv64 }, + /* 1099 */ { UD_Ipop, O_GS, O_NONE, O_NONE, O_NONE, P_none }, + /* 1100 */ { UD_Ipop, O_FS, O_NONE, O_NONE, O_NONE, P_none }, + /* 1101 */ { UD_Ipop, O_R0v, O_NONE, O_NONE, O_NONE, P_oso|P_rexb|P_def64 }, + /* 1102 */ { UD_Ipop, O_R1v, O_NONE, O_NONE, O_NONE, P_oso|P_rexb|P_def64 }, + /* 1103 */ { UD_Ipop, O_R2v, O_NONE, O_NONE, O_NONE, P_oso|P_rexb|P_def64 }, + /* 1104 */ { UD_Ipop, O_R3v, O_NONE, O_NONE, O_NONE, P_oso|P_rexb|P_def64 }, + /* 1105 */ { UD_Ipop, O_R4v, O_NONE, O_NONE, O_NONE, P_oso|P_rexb|P_def64 }, + /* 1106 */ { UD_Ipop, O_R5v, O_NONE, O_NONE, O_NONE, P_oso|P_rexb|P_def64 }, + /* 1107 */ { UD_Ipop, O_R6v, O_NONE, O_NONE, O_NONE, P_oso|P_rexb|P_def64 }, + /* 1108 */ { UD_Ipop, O_R7v, O_NONE, O_NONE, O_NONE, P_oso|P_rexb|P_def64 }, + /* 1109 */ { UD_Ipop, O_Ev, O_NONE, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb|P_def64 }, + /* 1110 */ { UD_Ipopa, O_NONE, O_NONE, O_NONE, O_NONE, P_oso|P_inv64 }, + /* 1111 */ { UD_Ipopad, O_NONE, O_NONE, O_NONE, O_NONE, P_oso|P_inv64 }, + /* 1112 */ { UD_Ipopfw, O_NONE, O_NONE, O_NONE, O_NONE, P_oso }, + /* 1113 */ { UD_Ipopfd, O_NONE, O_NONE, O_NONE, O_NONE, P_oso }, + /* 1114 */ { UD_Ipopfq, O_NONE, O_NONE, O_NONE, O_NONE, P_oso|P_def64 }, + /* 1115 */ { UD_Ipopfq, O_NONE, O_NONE, O_NONE, O_NONE, P_oso|P_def64 }, + /* 1116 */ { UD_Ipor, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1117 */ { UD_Ivpor, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1118 */ { UD_Ipor, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1119 */ { UD_Iprefetch, O_M, O_NONE, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1120 */ { UD_Iprefetch, O_M, O_NONE, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1121 */ { UD_Iprefetch, O_M, O_NONE, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1122 */ { UD_Iprefetch, O_M, O_NONE, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1123 */ { UD_Iprefetch, O_M, O_NONE, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1124 */ { UD_Iprefetch, O_M, O_NONE, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1125 */ { UD_Iprefetch, O_M, O_NONE, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1126 */ { UD_Iprefetch, O_M, O_NONE, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1127 */ { UD_Iprefetchnta, O_M, O_NONE, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1128 */ { UD_Iprefetcht0, O_M, O_NONE, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1129 */ { UD_Iprefetcht1, O_M, O_NONE, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1130 */ { UD_Iprefetcht2, O_M, O_NONE, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1131 */ { UD_Ipsadbw, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1132 */ { UD_Ivpsadbw, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1133 */ { UD_Ipsadbw, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1134 */ { UD_Ipshufw, O_P, O_Q, O_Ib, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1135 */ { UD_Ipsllw, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1136 */ { UD_Ipsllw, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1137 */ { UD_Ipsllw, O_U, O_Ib, O_NONE, O_NONE, P_rexb }, + /* 1138 */ { UD_Ipsllw, O_N, O_Ib, O_NONE, O_NONE, P_none }, + /* 1139 */ { UD_Ipslld, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1140 */ { UD_Ipslld, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1141 */ { UD_Ipslld, O_U, O_Ib, O_NONE, O_NONE, P_rexb }, + /* 1142 */ { UD_Ipslld, O_N, O_Ib, O_NONE, O_NONE, P_none }, + /* 1143 */ { UD_Ipsllq, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1144 */ { UD_Ipsllq, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1145 */ { UD_Ipsllq, O_U, O_Ib, O_NONE, O_NONE, P_rexb }, + /* 1146 */ { UD_Ipsllq, O_N, O_Ib, O_NONE, O_NONE, P_none }, + /* 1147 */ { UD_Ipsraw, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1148 */ { UD_Ipsraw, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1149 */ { UD_Ivpsraw, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1150 */ { UD_Ipsraw, O_U, O_Ib, O_NONE, O_NONE, P_rexb }, + /* 1151 */ { UD_Ivpsraw, O_Hx, O_Ux, O_Ib, O_NONE, P_rexb }, + /* 1152 */ { UD_Ipsraw, O_N, O_Ib, O_NONE, O_NONE, P_none }, + /* 1153 */ { UD_Ipsrad, O_N, O_Ib, O_NONE, O_NONE, P_none }, + /* 1154 */ { UD_Ipsrad, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1155 */ { UD_Ivpsrad, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1156 */ { UD_Ipsrad, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1157 */ { UD_Ipsrad, O_U, O_Ib, O_NONE, O_NONE, P_rexb }, + /* 1158 */ { UD_Ivpsrad, O_Hx, O_Ux, O_Ib, O_NONE, P_rexb }, + /* 1159 */ { UD_Ipsrlw, O_N, O_Ib, O_NONE, O_NONE, P_none }, + /* 1160 */ { UD_Ipsrlw, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1161 */ { UD_Ipsrlw, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1162 */ { UD_Ivpsrlw, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1163 */ { UD_Ipsrlw, O_U, O_Ib, O_NONE, O_NONE, P_rexb }, + /* 1164 */ { UD_Ivpsrlw, O_Hx, O_Ux, O_Ib, O_NONE, P_rexb }, + /* 1165 */ { UD_Ipsrld, O_N, O_Ib, O_NONE, O_NONE, P_none }, + /* 1166 */ { UD_Ipsrld, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1167 */ { UD_Ipsrld, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1168 */ { UD_Ivpsrld, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1169 */ { UD_Ipsrld, O_U, O_Ib, O_NONE, O_NONE, P_rexb }, + /* 1170 */ { UD_Ivpsrld, O_Hx, O_Ux, O_Ib, O_NONE, P_rexb }, + /* 1171 */ { UD_Ipsrlq, O_N, O_Ib, O_NONE, O_NONE, P_none }, + /* 1172 */ { UD_Ipsrlq, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1173 */ { UD_Ipsrlq, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1174 */ { UD_Ivpsrlq, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1175 */ { UD_Ipsrlq, O_U, O_Ib, O_NONE, O_NONE, P_rexb }, + /* 1176 */ { UD_Ivpsrlq, O_Hx, O_Ux, O_Ib, O_NONE, P_rexb }, + /* 1177 */ { UD_Ipsubb, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1178 */ { UD_Ivpsubb, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1179 */ { UD_Ipsubb, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1180 */ { UD_Ipsubw, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1181 */ { UD_Ivpsubw, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1182 */ { UD_Ipsubw, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1183 */ { UD_Ipsubd, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1184 */ { UD_Ipsubd, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1185 */ { UD_Ivpsubd, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1186 */ { UD_Ipsubsb, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1187 */ { UD_Ipsubsb, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1188 */ { UD_Ivpsubsb, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1189 */ { UD_Ipsubsw, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1190 */ { UD_Ipsubsw, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1191 */ { UD_Ivpsubsw, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1192 */ { UD_Ipsubusb, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1193 */ { UD_Ipsubusb, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1194 */ { UD_Ivpsubusb, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1195 */ { UD_Ipsubusw, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1196 */ { UD_Ipsubusw, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1197 */ { UD_Ivpsubusw, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1198 */ { UD_Ipunpckhbw, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1199 */ { UD_Ivpunpckhbw, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1200 */ { UD_Ipunpckhbw, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1201 */ { UD_Ipunpckhwd, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1202 */ { UD_Ivpunpckhwd, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1203 */ { UD_Ipunpckhwd, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1204 */ { UD_Ipunpckhdq, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1205 */ { UD_Ivpunpckhdq, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1206 */ { UD_Ipunpckhdq, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1207 */ { UD_Ipunpcklbw, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1208 */ { UD_Ivpunpcklbw, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1209 */ { UD_Ipunpcklbw, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1210 */ { UD_Ipunpcklwd, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1211 */ { UD_Ivpunpcklwd, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1212 */ { UD_Ipunpcklwd, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1213 */ { UD_Ipunpckldq, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1214 */ { UD_Ivpunpckldq, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1215 */ { UD_Ipunpckldq, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1216 */ { UD_Ipi2fw, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1217 */ { UD_Ipi2fd, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1218 */ { UD_Ipf2iw, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1219 */ { UD_Ipf2id, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1220 */ { UD_Ipfnacc, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1221 */ { UD_Ipfpnacc, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1222 */ { UD_Ipfcmpge, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1223 */ { UD_Ipfmin, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1224 */ { UD_Ipfrcp, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1225 */ { UD_Ipfrsqrt, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1226 */ { UD_Ipfsub, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1227 */ { UD_Ipfadd, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1228 */ { UD_Ipfcmpgt, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1229 */ { UD_Ipfmax, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1230 */ { UD_Ipfrcpit1, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1231 */ { UD_Ipfrsqit1, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1232 */ { UD_Ipfsubr, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1233 */ { UD_Ipfacc, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1234 */ { UD_Ipfcmpeq, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1235 */ { UD_Ipfmul, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1236 */ { UD_Ipfrcpit2, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1237 */ { UD_Ipmulhrw, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1238 */ { UD_Ipswapd, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1239 */ { UD_Ipavgusb, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1240 */ { UD_Ipush, O_ES, O_NONE, O_NONE, O_NONE, P_inv64 }, + /* 1241 */ { UD_Ipush, O_CS, O_NONE, O_NONE, O_NONE, P_inv64 }, + /* 1242 */ { UD_Ipush, O_SS, O_NONE, O_NONE, O_NONE, P_inv64 }, + /* 1243 */ { UD_Ipush, O_DS, O_NONE, O_NONE, O_NONE, P_inv64 }, + /* 1244 */ { UD_Ipush, O_GS, O_NONE, O_NONE, O_NONE, P_none }, + /* 1245 */ { UD_Ipush, O_FS, O_NONE, O_NONE, O_NONE, P_none }, + /* 1246 */ { UD_Ipush, O_R0v, O_NONE, O_NONE, O_NONE, P_oso|P_rexb|P_def64 }, + /* 1247 */ { UD_Ipush, O_R1v, O_NONE, O_NONE, O_NONE, P_oso|P_rexb|P_def64 }, + /* 1248 */ { UD_Ipush, O_R2v, O_NONE, O_NONE, O_NONE, P_oso|P_rexb|P_def64 }, + /* 1249 */ { UD_Ipush, O_R3v, O_NONE, O_NONE, O_NONE, P_oso|P_rexb|P_def64 }, + /* 1250 */ { UD_Ipush, O_R4v, O_NONE, O_NONE, O_NONE, P_oso|P_rexb|P_def64 }, + /* 1251 */ { UD_Ipush, O_R5v, O_NONE, O_NONE, O_NONE, P_oso|P_rexb|P_def64 }, + /* 1252 */ { UD_Ipush, O_R6v, O_NONE, O_NONE, O_NONE, P_oso|P_rexb|P_def64 }, + /* 1253 */ { UD_Ipush, O_R7v, O_NONE, O_NONE, O_NONE, P_oso|P_rexb|P_def64 }, + /* 1254 */ { UD_Ipush, O_sIz, O_NONE, O_NONE, O_NONE, P_oso|P_def64 }, + /* 1255 */ { UD_Ipush, O_Ev, O_NONE, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb|P_def64 }, + /* 1256 */ { UD_Ipush, O_sIb, O_NONE, O_NONE, O_NONE, P_oso|P_def64 }, + /* 1257 */ { UD_Ipusha, O_NONE, O_NONE, O_NONE, O_NONE, P_oso|P_inv64 }, + /* 1258 */ { UD_Ipushad, O_NONE, O_NONE, O_NONE, O_NONE, P_oso|P_inv64 }, + /* 1259 */ { UD_Ipushfw, O_NONE, O_NONE, O_NONE, O_NONE, P_oso }, + /* 1260 */ { UD_Ipushfw, O_NONE, O_NONE, O_NONE, O_NONE, P_oso|P_rexw|P_def64 }, + /* 1261 */ { UD_Ipushfd, O_NONE, O_NONE, O_NONE, O_NONE, P_oso }, + /* 1262 */ { UD_Ipushfq, O_NONE, O_NONE, O_NONE, O_NONE, P_oso|P_rexw|P_def64 }, + /* 1263 */ { UD_Ipushfq, O_NONE, O_NONE, O_NONE, O_NONE, P_oso|P_rexw|P_def64 }, + /* 1264 */ { UD_Ipxor, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1265 */ { UD_Ivpxor, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1266 */ { UD_Ipxor, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1267 */ { UD_Ircl, O_Eb, O_Ib, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1268 */ { UD_Ircl, O_Ev, O_Ib, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1269 */ { UD_Ircl, O_Eb, O_I1, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1270 */ { UD_Ircl, O_Eb, O_CL, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1271 */ { UD_Ircl, O_Ev, O_CL, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1272 */ { UD_Ircl, O_Ev, O_I1, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1273 */ { UD_Ircr, O_Eb, O_I1, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1274 */ { UD_Ircr, O_Ev, O_Ib, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1275 */ { UD_Ircr, O_Eb, O_Ib, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1276 */ { UD_Ircr, O_Ev, O_I1, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1277 */ { UD_Ircr, O_Eb, O_CL, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1278 */ { UD_Ircr, O_Ev, O_CL, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1279 */ { UD_Irol, O_Eb, O_Ib, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1280 */ { UD_Irol, O_Eb, O_I1, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1281 */ { UD_Irol, O_Ev, O_I1, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1282 */ { UD_Irol, O_Eb, O_CL, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1283 */ { UD_Irol, O_Ev, O_CL, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1284 */ { UD_Irol, O_Ev, O_Ib, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1285 */ { UD_Iror, O_Eb, O_I1, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1286 */ { UD_Iror, O_Eb, O_Ib, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1287 */ { UD_Iror, O_Ev, O_Ib, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1288 */ { UD_Iror, O_Ev, O_I1, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1289 */ { UD_Iror, O_Eb, O_CL, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1290 */ { UD_Iror, O_Ev, O_CL, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1291 */ { UD_Ircpps, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1292 */ { UD_Ivrcpps, O_Vx, O_Wx, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 1293 */ { UD_Ircpss, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1294 */ { UD_Ivrcpss, O_Vx, O_Wx, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1295 */ { UD_Irdmsr, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 1296 */ { UD_Irdpmc, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 1297 */ { UD_Irdtsc, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 1298 */ { UD_Irdtscp, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 1299 */ { UD_Irepne, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 1300 */ { UD_Irep, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 1301 */ { UD_Iret, O_Iw, O_NONE, O_NONE, O_NONE, P_none }, + /* 1302 */ { UD_Iret, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 1303 */ { UD_Iretf, O_Iw, O_NONE, O_NONE, O_NONE, P_none }, + /* 1304 */ { UD_Iretf, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 1305 */ { UD_Irsm, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 1306 */ { UD_Irsqrtps, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1307 */ { UD_Ivrsqrtps, O_Vx, O_Wx, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 1308 */ { UD_Irsqrtss, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1309 */ { UD_Ivrsqrtss, O_Vx, O_Wx, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1310 */ { UD_Isahf, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 1311 */ { UD_Isalc, O_NONE, O_NONE, O_NONE, O_NONE, P_inv64 }, + /* 1312 */ { UD_Isar, O_Ev, O_I1, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1313 */ { UD_Isar, O_Eb, O_Ib, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1314 */ { UD_Isar, O_Eb, O_I1, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1315 */ { UD_Isar, O_Ev, O_Ib, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1316 */ { UD_Isar, O_Eb, O_CL, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1317 */ { UD_Isar, O_Ev, O_CL, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1318 */ { UD_Ishl, O_Eb, O_Ib, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1319 */ { UD_Ishl, O_Ev, O_Ib, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1320 */ { UD_Ishl, O_Eb, O_I1, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1321 */ { UD_Ishl, O_Eb, O_CL, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1322 */ { UD_Ishl, O_Ev, O_CL, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1323 */ { UD_Ishl, O_Ev, O_Ib, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1324 */ { UD_Ishl, O_Eb, O_CL, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1325 */ { UD_Ishl, O_Ev, O_I1, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1326 */ { UD_Ishl, O_Eb, O_I1, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1327 */ { UD_Ishl, O_Eb, O_Ib, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1328 */ { UD_Ishl, O_Ev, O_CL, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1329 */ { UD_Ishl, O_Ev, O_I1, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1330 */ { UD_Ishr, O_Ev, O_Ib, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1331 */ { UD_Ishr, O_Eb, O_CL, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1332 */ { UD_Ishr, O_Ev, O_I1, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1333 */ { UD_Ishr, O_Eb, O_I1, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1334 */ { UD_Ishr, O_Eb, O_Ib, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1335 */ { UD_Ishr, O_Ev, O_CL, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1336 */ { UD_Isbb, O_Eb, O_Gb, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1337 */ { UD_Isbb, O_Ev, O_Gv, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1338 */ { UD_Isbb, O_Gb, O_Eb, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1339 */ { UD_Isbb, O_Gv, O_Ev, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1340 */ { UD_Isbb, O_AL, O_Ib, O_NONE, O_NONE, P_none }, + /* 1341 */ { UD_Isbb, O_rAX, O_sIz, O_NONE, O_NONE, P_oso|P_rexw }, + /* 1342 */ { UD_Isbb, O_Eb, O_Ib, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1343 */ { UD_Isbb, O_Ev, O_sIz, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1344 */ { UD_Isbb, O_Eb, O_Ib, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_inv64 }, + /* 1345 */ { UD_Isbb, O_Ev, O_sIb, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1346 */ { UD_Iscasb, O_NONE, O_NONE, O_NONE, O_NONE, P_strz }, + /* 1347 */ { UD_Iscasw, O_NONE, O_NONE, O_NONE, O_NONE, P_strz|P_oso|P_rexw }, + /* 1348 */ { UD_Iscasd, O_NONE, O_NONE, O_NONE, O_NONE, P_strz|P_oso|P_rexw }, + /* 1349 */ { UD_Iscasq, O_NONE, O_NONE, O_NONE, O_NONE, P_strz|P_oso|P_rexw }, + /* 1350 */ { UD_Iseto, O_Eb, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1351 */ { UD_Isetno, O_Eb, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1352 */ { UD_Isetb, O_Eb, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1353 */ { UD_Isetae, O_Eb, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1354 */ { UD_Isetz, O_Eb, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1355 */ { UD_Isetnz, O_Eb, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1356 */ { UD_Isetbe, O_Eb, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1357 */ { UD_Iseta, O_Eb, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1358 */ { UD_Isets, O_Eb, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1359 */ { UD_Isetns, O_Eb, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1360 */ { UD_Isetp, O_Eb, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1361 */ { UD_Isetnp, O_Eb, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1362 */ { UD_Isetl, O_Eb, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1363 */ { UD_Isetge, O_Eb, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1364 */ { UD_Isetle, O_Eb, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1365 */ { UD_Isetg, O_Eb, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1366 */ { UD_Isfence, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 1367 */ { UD_Isfence, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 1368 */ { UD_Isfence, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 1369 */ { UD_Isfence, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 1370 */ { UD_Isfence, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 1371 */ { UD_Isfence, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 1372 */ { UD_Isfence, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 1373 */ { UD_Isfence, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 1374 */ { UD_Isgdt, O_M, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1375 */ { UD_Ishld, O_Ev, O_Gv, O_Ib, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1376 */ { UD_Ishld, O_Ev, O_Gv, O_CL, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1377 */ { UD_Ishrd, O_Ev, O_Gv, O_Ib, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1378 */ { UD_Ishrd, O_Ev, O_Gv, O_CL, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1379 */ { UD_Ishufpd, O_V, O_W, O_Ib, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1380 */ { UD_Ivshufpd, O_Vx, O_Hx, O_Wx, O_Ib, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 1381 */ { UD_Ishufps, O_V, O_W, O_Ib, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1382 */ { UD_Ivshufps, O_Vx, O_Hx, O_Wx, O_Ib, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1383 */ { UD_Isidt, O_M, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1384 */ { UD_Isldt, O_MwRv, O_NONE, O_NONE, O_NONE, P_aso|P_oso|P_rexr|P_rexw|P_rexx|P_rexb }, + /* 1385 */ { UD_Ismsw, O_MwRv, O_NONE, O_NONE, O_NONE, P_aso|P_oso|P_rexr|P_rexw|P_rexx|P_rexb }, + /* 1386 */ { UD_Ismsw, O_MwRv, O_NONE, O_NONE, O_NONE, P_aso|P_oso|P_rexr|P_rexw|P_rexx|P_rexb }, + /* 1387 */ { UD_Isqrtps, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1388 */ { UD_Ivsqrtps, O_Vx, O_Wx, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 1389 */ { UD_Isqrtpd, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1390 */ { UD_Ivsqrtpd, O_Vx, O_Wx, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1391 */ { UD_Isqrtsd, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1392 */ { UD_Ivsqrtsd, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1393 */ { UD_Isqrtss, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1394 */ { UD_Ivsqrtss, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1395 */ { UD_Istc, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 1396 */ { UD_Istd, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 1397 */ { UD_Istgi, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 1398 */ { UD_Isti, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 1399 */ { UD_Iskinit, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 1400 */ { UD_Istmxcsr, O_Md, O_NONE, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1401 */ { UD_Ivstmxcsr, O_Md, O_NONE, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1402 */ { UD_Istosb, O_NONE, O_NONE, O_NONE, O_NONE, P_str|P_seg }, + /* 1403 */ { UD_Istosw, O_NONE, O_NONE, O_NONE, O_NONE, P_str|P_seg|P_oso|P_rexw }, + /* 1404 */ { UD_Istosd, O_NONE, O_NONE, O_NONE, O_NONE, P_str|P_seg|P_oso|P_rexw }, + /* 1405 */ { UD_Istosq, O_NONE, O_NONE, O_NONE, O_NONE, P_str|P_seg|P_oso|P_rexw }, + /* 1406 */ { UD_Istr, O_MwRv, O_NONE, O_NONE, O_NONE, P_aso|P_oso|P_rexr|P_rexw|P_rexx|P_rexb }, + /* 1407 */ { UD_Isub, O_Eb, O_Gb, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1408 */ { UD_Isub, O_Ev, O_Gv, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1409 */ { UD_Isub, O_Gb, O_Eb, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1410 */ { UD_Isub, O_Gv, O_Ev, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1411 */ { UD_Isub, O_AL, O_Ib, O_NONE, O_NONE, P_none }, + /* 1412 */ { UD_Isub, O_rAX, O_sIz, O_NONE, O_NONE, P_oso|P_rexw }, + /* 1413 */ { UD_Isub, O_Eb, O_Ib, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1414 */ { UD_Isub, O_Ev, O_sIz, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1415 */ { UD_Isub, O_Eb, O_Ib, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_inv64 }, + /* 1416 */ { UD_Isub, O_Ev, O_sIb, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1417 */ { UD_Isubpd, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1418 */ { UD_Ivsubpd, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 1419 */ { UD_Isubps, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1420 */ { UD_Ivsubps, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 1421 */ { UD_Isubsd, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1422 */ { UD_Ivsubsd, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1423 */ { UD_Isubss, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1424 */ { UD_Ivsubss, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1425 */ { UD_Iswapgs, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 1426 */ { UD_Isyscall, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 1427 */ { UD_Isysenter, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 1428 */ { UD_Isysenter, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 1429 */ { UD_Isysexit, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 1430 */ { UD_Isysexit, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 1431 */ { UD_Isysret, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 1432 */ { UD_Itest, O_Eb, O_Ib, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1433 */ { UD_Itest, O_Eb, O_Gb, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1434 */ { UD_Itest, O_Ev, O_Gv, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1435 */ { UD_Itest, O_AL, O_Ib, O_NONE, O_NONE, P_none }, + /* 1436 */ { UD_Itest, O_rAX, O_sIz, O_NONE, O_NONE, P_oso|P_rexw }, + /* 1437 */ { UD_Itest, O_Eb, O_Ib, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1438 */ { UD_Itest, O_Ev, O_sIz, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1439 */ { UD_Itest, O_Ev, O_Iz, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1440 */ { UD_Iucomisd, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1441 */ { UD_Ivucomisd, O_Vx, O_Wx, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1442 */ { UD_Iucomiss, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1443 */ { UD_Ivucomiss, O_Vx, O_Wx, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1444 */ { UD_Iud2, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 1445 */ { UD_Iunpckhpd, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1446 */ { UD_Ivunpckhpd, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 1447 */ { UD_Iunpckhps, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1448 */ { UD_Ivunpckhps, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1449 */ { UD_Iunpcklps, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1450 */ { UD_Ivunpcklps, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1451 */ { UD_Iunpcklpd, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1452 */ { UD_Ivunpcklpd, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 1453 */ { UD_Iverr, O_Ew, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1454 */ { UD_Iverw, O_Ew, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1455 */ { UD_Ivmcall, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 1456 */ { UD_Irdrand, O_R, O_NONE, O_NONE, O_NONE, P_oso|P_rexr|P_rexw|P_rexx|P_rexb }, + /* 1457 */ { UD_Ivmclear, O_Mq, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1458 */ { UD_Ivmxon, O_Mq, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1459 */ { UD_Ivmptrld, O_Mq, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1460 */ { UD_Ivmptrst, O_Mq, O_NONE, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1461 */ { UD_Ivmlaunch, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 1462 */ { UD_Ivmresume, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 1463 */ { UD_Ivmxoff, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 1464 */ { UD_Ivmread, O_Ey, O_Gy, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_def64 }, + /* 1465 */ { UD_Ivmwrite, O_Gy, O_Ey, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_def64 }, + /* 1466 */ { UD_Ivmrun, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 1467 */ { UD_Ivmmcall, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 1468 */ { UD_Ivmload, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 1469 */ { UD_Ivmsave, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 1470 */ { UD_Iwait, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 1471 */ { UD_Iwbinvd, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 1472 */ { UD_Iwrmsr, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 1473 */ { UD_Ixadd, O_Eb, O_Gb, O_NONE, O_NONE, P_aso|P_oso|P_rexr|P_rexx|P_rexb }, + /* 1474 */ { UD_Ixadd, O_Ev, O_Gv, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1475 */ { UD_Ixchg, O_Eb, O_Gb, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1476 */ { UD_Ixchg, O_Ev, O_Gv, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1477 */ { UD_Ixchg, O_R0v, O_rAX, O_NONE, O_NONE, P_oso|P_rexw|P_rexb }, + /* 1478 */ { UD_Ixchg, O_R1v, O_rAX, O_NONE, O_NONE, P_oso|P_rexw|P_rexb }, + /* 1479 */ { UD_Ixchg, O_R2v, O_rAX, O_NONE, O_NONE, P_oso|P_rexw|P_rexb }, + /* 1480 */ { UD_Ixchg, O_R3v, O_rAX, O_NONE, O_NONE, P_oso|P_rexw|P_rexb }, + /* 1481 */ { UD_Ixchg, O_R4v, O_rAX, O_NONE, O_NONE, P_oso|P_rexw|P_rexb }, + /* 1482 */ { UD_Ixchg, O_R5v, O_rAX, O_NONE, O_NONE, P_oso|P_rexw|P_rexb }, + /* 1483 */ { UD_Ixchg, O_R6v, O_rAX, O_NONE, O_NONE, P_oso|P_rexw|P_rexb }, + /* 1484 */ { UD_Ixchg, O_R7v, O_rAX, O_NONE, O_NONE, P_oso|P_rexw|P_rexb }, + /* 1485 */ { UD_Ixgetbv, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 1486 */ { UD_Ixlatb, O_NONE, O_NONE, O_NONE, O_NONE, P_rexw|P_seg }, + /* 1487 */ { UD_Ixor, O_Eb, O_Gb, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1488 */ { UD_Ixor, O_Ev, O_Gv, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1489 */ { UD_Ixor, O_Gb, O_Eb, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1490 */ { UD_Ixor, O_Gv, O_Ev, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1491 */ { UD_Ixor, O_AL, O_Ib, O_NONE, O_NONE, P_none }, + /* 1492 */ { UD_Ixor, O_rAX, O_sIz, O_NONE, O_NONE, P_oso|P_rexw }, + /* 1493 */ { UD_Ixor, O_Eb, O_Ib, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1494 */ { UD_Ixor, O_Ev, O_sIz, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1495 */ { UD_Ixor, O_Eb, O_Ib, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_inv64 }, + /* 1496 */ { UD_Ixor, O_Ev, O_sIb, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1497 */ { UD_Ixorpd, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1498 */ { UD_Ivxorpd, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 1499 */ { UD_Ixorps, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1500 */ { UD_Ivxorps, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1501 */ { UD_Ixcryptecb, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 1502 */ { UD_Ixcryptcbc, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 1503 */ { UD_Ixcryptctr, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 1504 */ { UD_Ixcryptcfb, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 1505 */ { UD_Ixcryptofb, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 1506 */ { UD_Ixrstor, O_M, O_NONE, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1507 */ { UD_Ixsave, O_M, O_NONE, O_NONE, O_NONE, P_aso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1508 */ { UD_Ixsetbv, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 1509 */ { UD_Ixsha1, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 1510 */ { UD_Ixsha256, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 1511 */ { UD_Ixstore, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 1512 */ { UD_Ipclmulqdq, O_V, O_W, O_Ib, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1513 */ { UD_Ivpclmulqdq, O_Vx, O_Hx, O_Wx, O_Ib, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1514 */ { UD_Igetsec, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 1515 */ { UD_Imovdqa, O_W, O_V, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1516 */ { UD_Ivmovdqa, O_Wx, O_Vx, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 1517 */ { UD_Imovdqa, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1518 */ { UD_Ivmovdqa, O_Vx, O_Wx, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 1519 */ { UD_Imaskmovdqu, O_V, O_U, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1520 */ { UD_Ivmaskmovdqu, O_Vx, O_Ux, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1521 */ { UD_Imovdq2q, O_P, O_U, O_NONE, O_NONE, P_aso|P_rexb }, + /* 1522 */ { UD_Imovdqu, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1523 */ { UD_Ivmovdqu, O_Vx, O_Wx, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 1524 */ { UD_Imovdqu, O_W, O_V, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1525 */ { UD_Ivmovdqu, O_Wx, O_Vx, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 1526 */ { UD_Imovq2dq, O_V, O_N, O_NONE, O_NONE, P_aso|P_rexr }, + /* 1527 */ { UD_Ipaddq, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1528 */ { UD_Ipaddq, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1529 */ { UD_Ivpaddq, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1530 */ { UD_Ipsubq, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1531 */ { UD_Ivpsubq, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1532 */ { UD_Ipsubq, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1533 */ { UD_Ipmuludq, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1534 */ { UD_Ipmuludq, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1535 */ { UD_Ipshufhw, O_V, O_W, O_Ib, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1536 */ { UD_Ivpshufhw, O_Vx, O_Wx, O_Ib, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1537 */ { UD_Ipshuflw, O_V, O_W, O_Ib, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1538 */ { UD_Ivpshuflw, O_Vx, O_Wx, O_Ib, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1539 */ { UD_Ipshufd, O_V, O_W, O_Ib, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1540 */ { UD_Ivpshufd, O_Vx, O_Wx, O_Ib, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1541 */ { UD_Ipslldq, O_U, O_Ib, O_NONE, O_NONE, P_rexb }, + /* 1542 */ { UD_Ivpslldq, O_Hx, O_Ux, O_Ib, O_NONE, P_rexb }, + /* 1543 */ { UD_Ipsrldq, O_U, O_Ib, O_NONE, O_NONE, P_rexb }, + /* 1544 */ { UD_Ivpsrldq, O_Hx, O_Ux, O_Ib, O_NONE, P_rexb }, + /* 1545 */ { UD_Ipunpckhqdq, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1546 */ { UD_Ivpunpckhqdq, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1547 */ { UD_Ipunpcklqdq, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1548 */ { UD_Ivpunpcklqdq, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1549 */ { UD_Ihaddpd, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1550 */ { UD_Ivhaddpd, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 1551 */ { UD_Ihaddps, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1552 */ { UD_Ivhaddps, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 1553 */ { UD_Ihsubpd, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1554 */ { UD_Ivhsubpd, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 1555 */ { UD_Ihsubps, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1556 */ { UD_Ivhsubps, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 1557 */ { UD_Iinsertps, O_V, O_Md, O_Ib, O_NONE, P_aso|P_rexr|P_rexw|P_rexx|P_rexb }, + /* 1558 */ { UD_Ivinsertps, O_Vx, O_Hx, O_Md, O_Ib, P_aso|P_rexr|P_rexw|P_rexx|P_rexb }, + /* 1559 */ { UD_Ilddqu, O_V, O_M, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1560 */ { UD_Ivlddqu, O_Vx, O_M, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 1561 */ { UD_Imovddup, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1562 */ { UD_Ivmovddup, O_Vx, O_Wx, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1563 */ { UD_Imovddup, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1564 */ { UD_Ivmovddup, O_Vx, O_Wx, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1565 */ { UD_Imovshdup, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1566 */ { UD_Ivmovshdup, O_Vx, O_Wx, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 1567 */ { UD_Imovshdup, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1568 */ { UD_Ivmovshdup, O_Vx, O_Wx, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 1569 */ { UD_Imovsldup, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1570 */ { UD_Ivmovsldup, O_Vx, O_Wx, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 1571 */ { UD_Imovsldup, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1572 */ { UD_Ivmovsldup, O_Vx, O_Wx, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 1573 */ { UD_Ipabsb, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1574 */ { UD_Ipabsb, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1575 */ { UD_Ivpabsb, O_Vx, O_Wx, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 1576 */ { UD_Ipabsw, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1577 */ { UD_Ipabsw, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1578 */ { UD_Ivpabsw, O_Vx, O_Wx, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 1579 */ { UD_Ipabsd, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1580 */ { UD_Ipabsd, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1581 */ { UD_Ivpabsd, O_Vx, O_Wx, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 1582 */ { UD_Ipshufb, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1583 */ { UD_Ipshufb, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1584 */ { UD_Ivpshufb, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1585 */ { UD_Iphaddw, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1586 */ { UD_Iphaddw, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1587 */ { UD_Ivphaddw, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1588 */ { UD_Iphaddd, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1589 */ { UD_Iphaddd, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1590 */ { UD_Ivphaddd, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1591 */ { UD_Iphaddsw, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1592 */ { UD_Iphaddsw, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1593 */ { UD_Ivphaddsw, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1594 */ { UD_Ipmaddubsw, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1595 */ { UD_Ipmaddubsw, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1596 */ { UD_Ivpmaddubsw, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1597 */ { UD_Iphsubw, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1598 */ { UD_Iphsubw, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1599 */ { UD_Ivphsubw, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1600 */ { UD_Iphsubd, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1601 */ { UD_Iphsubd, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1602 */ { UD_Ivphsubd, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1603 */ { UD_Iphsubsw, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1604 */ { UD_Iphsubsw, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1605 */ { UD_Ivphsubsw, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1606 */ { UD_Ipsignb, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1607 */ { UD_Ipsignb, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1608 */ { UD_Ivpsignb, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1609 */ { UD_Ipsignd, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1610 */ { UD_Ipsignd, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1611 */ { UD_Ivpsignd, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1612 */ { UD_Ipsignw, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1613 */ { UD_Ipsignw, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1614 */ { UD_Ivpsignw, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1615 */ { UD_Ipmulhrsw, O_P, O_Q, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1616 */ { UD_Ipmulhrsw, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1617 */ { UD_Ivpmulhrsw, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1618 */ { UD_Ipalignr, O_P, O_Q, O_Ib, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1619 */ { UD_Ipalignr, O_V, O_W, O_Ib, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1620 */ { UD_Ivpalignr, O_Vx, O_Hx, O_Wx, O_Ib, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1621 */ { UD_Ipblendvb, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1622 */ { UD_Ipmuldq, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1623 */ { UD_Ivpmuldq, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1624 */ { UD_Ipminsb, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1625 */ { UD_Ivpminsb, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1626 */ { UD_Ipminsd, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1627 */ { UD_Ivpminsd, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1628 */ { UD_Ipminuw, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1629 */ { UD_Ivpminuw, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1630 */ { UD_Ipminud, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1631 */ { UD_Ivpminud, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1632 */ { UD_Ipmaxsb, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1633 */ { UD_Ivpmaxsb, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1634 */ { UD_Ipmaxsd, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1635 */ { UD_Ivpmaxsd, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1636 */ { UD_Ipmaxud, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1637 */ { UD_Ivpmaxud, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1638 */ { UD_Ipmaxuw, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1639 */ { UD_Ivpmaxuw, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1640 */ { UD_Ipmulld, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1641 */ { UD_Ivpmulld, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1642 */ { UD_Iphminposuw, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1643 */ { UD_Ivphminposuw, O_Vx, O_Wx, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1644 */ { UD_Iroundps, O_V, O_W, O_Ib, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1645 */ { UD_Ivroundps, O_Vx, O_Wx, O_Ib, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 1646 */ { UD_Iroundpd, O_V, O_W, O_Ib, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1647 */ { UD_Ivroundpd, O_Vx, O_Wx, O_Ib, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 1648 */ { UD_Iroundss, O_V, O_W, O_Ib, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1649 */ { UD_Ivroundss, O_Vx, O_Hx, O_Wx, O_Ib, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1650 */ { UD_Iroundsd, O_V, O_W, O_Ib, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1651 */ { UD_Ivroundsd, O_Vx, O_Hx, O_Wx, O_Ib, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1652 */ { UD_Iblendpd, O_V, O_W, O_Ib, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1653 */ { UD_Ivblendpd, O_Vx, O_Hx, O_Wx, O_Ib, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 1654 */ { UD_Iblendps, O_V, O_W, O_Ib, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1655 */ { UD_Ivblendps, O_Vx, O_Hx, O_Wx, O_Ib, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1656 */ { UD_Iblendvpd, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1657 */ { UD_Iblendvps, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1658 */ { UD_Ibound, O_Gv, O_M, O_NONE, O_NONE, P_aso|P_oso }, + /* 1659 */ { UD_Ibsf, O_Gv, O_Ev, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1660 */ { UD_Ibsr, O_Gv, O_Ev, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1661 */ { UD_Ibswap, O_R0y, O_NONE, O_NONE, O_NONE, P_oso|P_rexw|P_rexb }, + /* 1662 */ { UD_Ibswap, O_R1y, O_NONE, O_NONE, O_NONE, P_oso|P_rexw|P_rexb }, + /* 1663 */ { UD_Ibswap, O_R2y, O_NONE, O_NONE, O_NONE, P_oso|P_rexw|P_rexb }, + /* 1664 */ { UD_Ibswap, O_R3y, O_NONE, O_NONE, O_NONE, P_oso|P_rexw|P_rexb }, + /* 1665 */ { UD_Ibswap, O_R4y, O_NONE, O_NONE, O_NONE, P_oso|P_rexw|P_rexb }, + /* 1666 */ { UD_Ibswap, O_R5y, O_NONE, O_NONE, O_NONE, P_oso|P_rexw|P_rexb }, + /* 1667 */ { UD_Ibswap, O_R6y, O_NONE, O_NONE, O_NONE, P_oso|P_rexw|P_rexb }, + /* 1668 */ { UD_Ibswap, O_R7y, O_NONE, O_NONE, O_NONE, P_oso|P_rexw|P_rexb }, + /* 1669 */ { UD_Ibt, O_Ev, O_Ib, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1670 */ { UD_Ibt, O_Ev, O_Gv, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1671 */ { UD_Ibtc, O_Ev, O_Gv, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1672 */ { UD_Ibtc, O_Ev, O_Ib, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1673 */ { UD_Ibtr, O_Ev, O_Gv, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1674 */ { UD_Ibtr, O_Ev, O_Ib, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1675 */ { UD_Ibts, O_Ev, O_Gv, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1676 */ { UD_Ibts, O_Ev, O_Ib, O_NONE, O_NONE, P_aso|P_oso|P_rexw|P_rexr|P_rexx|P_rexb }, + /* 1677 */ { UD_Ipblendw, O_V, O_W, O_Ib, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1678 */ { UD_Ivpblendw, O_Vx, O_Hx, O_Wx, O_Ib, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1679 */ { UD_Impsadbw, O_V, O_W, O_Ib, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1680 */ { UD_Ivmpsadbw, O_Vx, O_Hx, O_Wx, O_Ib, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 1681 */ { UD_Imovntdqa, O_V, O_M, O_NONE, O_NONE, P_aso|P_rexr|P_rexw|P_rexx|P_rexb }, + /* 1682 */ { UD_Ivmovntdqa, O_Vx, O_M, O_NONE, O_NONE, P_aso|P_rexr|P_rexw|P_rexx|P_rexb|P_vexl }, + /* 1683 */ { UD_Ipackusdw, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexw|P_rexx|P_rexb }, + /* 1684 */ { UD_Ivpackusdw, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexw|P_rexx|P_rexb|P_vexl }, + /* 1685 */ { UD_Ipmovsxbw, O_V, O_MqU, O_NONE, O_NONE, P_aso|P_rexr|P_rexw|P_rexx|P_rexb }, + /* 1686 */ { UD_Ivpmovsxbw, O_Vx, O_MqU, O_NONE, O_NONE, P_aso|P_rexr|P_rexw|P_rexx|P_rexb }, + /* 1687 */ { UD_Ipmovsxbd, O_V, O_MdU, O_NONE, O_NONE, P_aso|P_rexr|P_rexw|P_rexx|P_rexb }, + /* 1688 */ { UD_Ivpmovsxbd, O_Vx, O_MdU, O_NONE, O_NONE, P_aso|P_rexr|P_rexw|P_rexx|P_rexb }, + /* 1689 */ { UD_Ipmovsxbq, O_V, O_MwU, O_NONE, O_NONE, P_aso|P_rexr|P_rexw|P_rexx|P_rexb }, + /* 1690 */ { UD_Ivpmovsxbq, O_Vx, O_MwU, O_NONE, O_NONE, P_aso|P_rexr|P_rexw|P_rexx|P_rexb }, + /* 1691 */ { UD_Ipmovsxwd, O_V, O_MqU, O_NONE, O_NONE, P_aso|P_rexr|P_rexw|P_rexx|P_rexb }, + /* 1692 */ { UD_Ivpmovsxwd, O_Vx, O_MqU, O_NONE, O_NONE, P_aso|P_rexr|P_rexw|P_rexx|P_rexb }, + /* 1693 */ { UD_Ipmovsxwq, O_V, O_MdU, O_NONE, O_NONE, P_aso|P_rexr|P_rexw|P_rexx|P_rexb }, + /* 1694 */ { UD_Ivpmovsxwq, O_Vx, O_MdU, O_NONE, O_NONE, P_aso|P_rexr|P_rexw|P_rexx|P_rexb }, + /* 1695 */ { UD_Ipmovsxdq, O_V, O_MqU, O_NONE, O_NONE, P_aso|P_rexr|P_rexw|P_rexx|P_rexb }, + /* 1696 */ { UD_Ipmovzxbw, O_V, O_MqU, O_NONE, O_NONE, P_aso|P_rexr|P_rexw|P_rexx|P_rexb }, + /* 1697 */ { UD_Ivpmovzxbw, O_Vx, O_MqU, O_NONE, O_NONE, P_aso|P_rexr|P_rexw|P_rexx|P_rexb }, + /* 1698 */ { UD_Ipmovzxbd, O_V, O_MdU, O_NONE, O_NONE, P_aso|P_rexr|P_rexw|P_rexx|P_rexb }, + /* 1699 */ { UD_Ivpmovzxbd, O_Vx, O_MdU, O_NONE, O_NONE, P_aso|P_rexr|P_rexw|P_rexx|P_rexb }, + /* 1700 */ { UD_Ipmovzxbq, O_V, O_MwU, O_NONE, O_NONE, P_aso|P_rexr|P_rexw|P_rexx|P_rexb }, + /* 1701 */ { UD_Ivpmovzxbq, O_Vx, O_MwU, O_NONE, O_NONE, P_aso|P_rexr|P_rexw|P_rexx|P_rexb }, + /* 1702 */ { UD_Ipmovzxwd, O_V, O_MqU, O_NONE, O_NONE, P_aso|P_rexr|P_rexw|P_rexx|P_rexb }, + /* 1703 */ { UD_Ivpmovzxwd, O_Vx, O_MqU, O_NONE, O_NONE, P_aso|P_rexr|P_rexw|P_rexx|P_rexb }, + /* 1704 */ { UD_Ipmovzxwq, O_V, O_MdU, O_NONE, O_NONE, P_aso|P_rexr|P_rexw|P_rexx|P_rexb }, + /* 1705 */ { UD_Ivpmovzxwq, O_Vx, O_MdU, O_NONE, O_NONE, P_aso|P_rexr|P_rexw|P_rexx|P_rexb }, + /* 1706 */ { UD_Ipmovzxdq, O_V, O_MqU, O_NONE, O_NONE, P_aso|P_rexr|P_rexw|P_rexx|P_rexb }, + /* 1707 */ { UD_Ivpmovzxdq, O_Vx, O_MqU, O_NONE, O_NONE, P_aso|P_rexr|P_rexw|P_rexx|P_rexb }, + /* 1708 */ { UD_Ipcmpeqq, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexw|P_rexx|P_rexb }, + /* 1709 */ { UD_Ivpcmpeqq, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexw|P_rexx|P_rexb }, + /* 1710 */ { UD_Ipopcnt, O_Gv, O_Ev, O_NONE, O_NONE, P_aso|P_oso|P_rexr|P_rexw|P_rexx|P_rexb }, + /* 1711 */ { UD_Iptest, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexw|P_rexx|P_rexb }, + /* 1712 */ { UD_Ivptest, O_Vx, O_Wx, O_NONE, O_NONE, P_aso|P_rexr|P_rexw|P_rexx|P_rexb|P_vexl }, + /* 1713 */ { UD_Ipcmpestri, O_V, O_W, O_Ib, O_NONE, P_aso|P_rexr|P_rexw|P_rexx|P_rexb }, + /* 1714 */ { UD_Ivpcmpestri, O_Vx, O_Wx, O_Ib, O_NONE, P_aso|P_rexr|P_rexw|P_rexx|P_rexb }, + /* 1715 */ { UD_Ipcmpestrm, O_V, O_W, O_Ib, O_NONE, P_aso|P_rexr|P_rexw|P_rexx|P_rexb }, + /* 1716 */ { UD_Ivpcmpestrm, O_Vx, O_Wx, O_Ib, O_NONE, P_aso|P_rexr|P_rexw|P_rexx|P_rexb }, + /* 1717 */ { UD_Ipcmpgtq, O_V, O_W, O_NONE, O_NONE, P_aso|P_rexr|P_rexw|P_rexx|P_rexb }, + /* 1718 */ { UD_Ivpcmpgtq, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexw|P_rexx|P_rexb }, + /* 1719 */ { UD_Ipcmpistri, O_V, O_W, O_Ib, O_NONE, P_aso|P_rexr|P_rexw|P_rexx|P_rexb }, + /* 1720 */ { UD_Ivpcmpistri, O_Vx, O_Wx, O_Ib, O_NONE, P_aso|P_rexr|P_rexw|P_rexx|P_rexb }, + /* 1721 */ { UD_Ipcmpistrm, O_V, O_W, O_Ib, O_NONE, P_aso|P_rexr|P_rexw|P_rexx|P_rexb }, + /* 1722 */ { UD_Ivpcmpistrm, O_Vx, O_Wx, O_Ib, O_NONE, P_aso|P_rexr|P_rexw|P_rexx|P_rexb }, + /* 1723 */ { UD_Imovbe, O_Gv, O_Mv, O_NONE, O_NONE, P_aso|P_oso|P_rexr|P_rexw|P_rexx|P_rexb }, + /* 1724 */ { UD_Imovbe, O_Mv, O_Gv, O_NONE, O_NONE, P_aso|P_oso|P_rexr|P_rexw|P_rexx|P_rexb }, + /* 1725 */ { UD_Icrc32, O_Gy, O_Eb, O_NONE, O_NONE, P_aso|P_oso|P_rexr|P_rexw|P_rexx|P_rexb }, + /* 1726 */ { UD_Icrc32, O_Gy, O_Ev, O_NONE, O_NONE, P_aso|P_oso|P_rexr|P_rexw|P_rexx|P_rexb }, + /* 1727 */ { UD_Ivbroadcastss, O_V, O_Md, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 1728 */ { UD_Ivbroadcastsd, O_Vqq, O_Mq, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 1729 */ { UD_Ivextractf128, O_Wdq, O_Vqq, O_Ib, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 1730 */ { UD_Ivinsertf128, O_Vqq, O_Hqq, O_Wdq, O_Ib, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 1731 */ { UD_Ivmaskmovps, O_V, O_H, O_M, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 1732 */ { UD_Ivmaskmovps, O_M, O_H, O_V, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 1733 */ { UD_Ivmaskmovpd, O_V, O_H, O_M, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 1734 */ { UD_Ivmaskmovpd, O_M, O_H, O_V, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 1735 */ { UD_Ivpermilpd, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 1736 */ { UD_Ivpermilpd, O_V, O_W, O_Ib, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 1737 */ { UD_Ivpermilps, O_Vx, O_Hx, O_Wx, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 1738 */ { UD_Ivpermilps, O_Vx, O_Wx, O_Ib, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 1739 */ { UD_Ivperm2f128, O_Vqq, O_Hqq, O_Wqq, O_Ib, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 1740 */ { UD_Ivtestps, O_Vx, O_Wx, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 1741 */ { UD_Ivtestpd, O_Vx, O_Wx, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 1742 */ { UD_Ivzeroupper, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 1743 */ { UD_Ivzeroall, O_NONE, O_NONE, O_NONE, O_NONE, P_none }, + /* 1744 */ { UD_Ivblendvpd, O_Vx, O_Hx, O_Wx, O_Lx, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 1745 */ { UD_Ivblendvps, O_Vx, O_Hx, O_Wx, O_Lx, P_aso|P_rexr|P_rexx|P_rexb|P_vexl }, + /* 1746 */ { UD_Ivmovsd, O_V, O_H, O_U, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1747 */ { UD_Ivmovsd, O_V, O_Mq, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1748 */ { UD_Ivmovsd, O_U, O_H, O_V, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1749 */ { UD_Ivmovsd, O_Mq, O_V, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1750 */ { UD_Ivmovss, O_V, O_H, O_U, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1751 */ { UD_Ivmovss, O_V, O_Md, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1752 */ { UD_Ivmovss, O_U, O_H, O_V, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1753 */ { UD_Ivmovss, O_Md, O_V, O_NONE, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1754 */ { UD_Ivpblendvb, O_V, O_H, O_W, O_L, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1755 */ { UD_Ivpsllw, O_V, O_H, O_W, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1756 */ { UD_Ivpsllw, O_H, O_V, O_W, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1757 */ { UD_Ivpslld, O_V, O_H, O_W, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1758 */ { UD_Ivpslld, O_H, O_V, O_W, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1759 */ { UD_Ivpsllq, O_V, O_H, O_W, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, + /* 1760 */ { UD_Ivpsllq, O_H, O_V, O_W, O_NONE, P_aso|P_rexr|P_rexx|P_rexb }, +}; + + +const char* ud_mnemonics_str[] = { + "aaa", + "aad", + "aam", + "aas", + "adc", + "add", + "addpd", + "addps", + "addsd", + "addss", + "addsubpd", + "addsubps", + "aesdec", + "aesdeclast", + "aesenc", + "aesenclast", + "aesimc", + "aeskeygenassist", + "and", + "andnpd", + "andnps", + "andpd", + "andps", + "arpl", + "blendpd", + "blendps", + "blendvpd", + "blendvps", + "bound", + "bsf", + "bsr", + "bswap", + "bt", + "btc", + "btr", + "bts", + "call", + "cbw", + "cdq", + "cdqe", + "clc", + "cld", + "clflush", + "clgi", + "cli", + "clts", + "cmc", + "cmova", + "cmovae", + "cmovb", + "cmovbe", + "cmovg", + "cmovge", + "cmovl", + "cmovle", + "cmovno", + "cmovnp", + "cmovns", + "cmovnz", + "cmovo", + "cmovp", + "cmovs", + "cmovz", + "cmp", + "cmppd", + "cmpps", + "cmpsb", + "cmpsd", + "cmpsq", + "cmpss", + "cmpsw", + "cmpxchg", + "cmpxchg16b", + "cmpxchg8b", + "comisd", + "comiss", + "cpuid", + "cqo", + "crc32", + "cvtdq2pd", + "cvtdq2ps", + "cvtpd2dq", + "cvtpd2pi", + "cvtpd2ps", + "cvtpi2pd", + "cvtpi2ps", + "cvtps2dq", + "cvtps2pd", + "cvtps2pi", + "cvtsd2si", + "cvtsd2ss", + "cvtsi2sd", + "cvtsi2ss", + "cvtss2sd", + "cvtss2si", + "cvttpd2dq", + "cvttpd2pi", + "cvttps2dq", + "cvttps2pi", + "cvttsd2si", + "cvttss2si", + "cwd", + "cwde", + "daa", + "das", + "dec", + "div", + "divpd", + "divps", + "divsd", + "divss", + "dppd", + "dpps", + "emms", + "enter", + "extractps", + "f2xm1", + "fabs", + "fadd", + "faddp", + "fbld", + "fbstp", + "fchs", + "fclex", + "fcmovb", + "fcmovbe", + "fcmove", + "fcmovnb", + "fcmovnbe", + "fcmovne", + "fcmovnu", + "fcmovu", + "fcom", + "fcom2", + "fcomi", + "fcomip", + "fcomp", + "fcomp3", + "fcomp5", + "fcompp", + "fcos", + "fdecstp", + "fdiv", + "fdivp", + "fdivr", + "fdivrp", + "femms", + "ffree", + "ffreep", + "fiadd", + "ficom", + "ficomp", + "fidiv", + "fidivr", + "fild", + "fimul", + "fincstp", + "fist", + "fistp", + "fisttp", + "fisub", + "fisubr", + "fld", + "fld1", + "fldcw", + "fldenv", + "fldl2e", + "fldl2t", + "fldlg2", + "fldln2", + "fldpi", + "fldz", + "fmul", + "fmulp", + "fndisi", + "fneni", + "fninit", + "fnop", + "fnsave", + "fnsetpm", + "fnstcw", + "fnstenv", + "fnstsw", + "fpatan", + "fprem", + "fprem1", + "fptan", + "frndint", + "frstor", + "frstpm", + "fscale", + "fsin", + "fsincos", + "fsqrt", + "fst", + "fstp", + "fstp1", + "fstp8", + "fstp9", + "fsub", + "fsubp", + "fsubr", + "fsubrp", + "ftst", + "fucom", + "fucomi", + "fucomip", + "fucomp", + "fucompp", + "fxam", + "fxch", + "fxch4", + "fxch7", + "fxrstor", + "fxsave", + "fxtract", + "fyl2x", + "fyl2xp1", + "getsec", + "haddpd", + "haddps", + "hlt", + "hsubpd", + "hsubps", + "idiv", + "imul", + "in", + "inc", + "insb", + "insd", + "insertps", + "insw", + "int", + "int1", + "int3", + "into", + "invd", + "invept", + "invlpg", + "invlpga", + "invvpid", + "iretd", + "iretq", + "iretw", + "ja", + "jae", + "jb", + "jbe", + "jcxz", + "jecxz", + "jg", + "jge", + "jl", + "jle", + "jmp", + "jno", + "jnp", + "jns", + "jnz", + "jo", + "jp", + "jrcxz", + "js", + "jz", + "lahf", + "lar", + "lddqu", + "ldmxcsr", + "lds", + "lea", + "leave", + "les", + "lfence", + "lfs", + "lgdt", + "lgs", + "lidt", + "lldt", + "lmsw", + "lock", + "lodsb", + "lodsd", + "lodsq", + "lodsw", + "loop", + "loope", + "loopne", + "lsl", + "lss", + "ltr", + "maskmovdqu", + "maskmovq", + "maxpd", + "maxps", + "maxsd", + "maxss", + "mfence", + "minpd", + "minps", + "minsd", + "minss", + "monitor", + "montmul", + "mov", + "movapd", + "movaps", + "movbe", + "movd", + "movddup", + "movdq2q", + "movdqa", + "movdqu", + "movhlps", + "movhpd", + "movhps", + "movlhps", + "movlpd", + "movlps", + "movmskpd", + "movmskps", + "movntdq", + "movntdqa", + "movnti", + "movntpd", + "movntps", + "movntq", + "movq", + "movq2dq", + "movsb", + "movsd", + "movshdup", + "movsldup", + "movsq", + "movss", + "movsw", + "movsx", + "movsxd", + "movupd", + "movups", + "movzx", + "mpsadbw", + "mul", + "mulpd", + "mulps", + "mulsd", + "mulss", + "mwait", + "neg", + "nop", + "not", + "or", + "orpd", + "orps", + "out", + "outsb", + "outsd", + "outsw", + "pabsb", + "pabsd", + "pabsw", + "packssdw", + "packsswb", + "packusdw", + "packuswb", + "paddb", + "paddd", + "paddq", + "paddsb", + "paddsw", + "paddusb", + "paddusw", + "paddw", + "palignr", + "pand", + "pandn", + "pavgb", + "pavgusb", + "pavgw", + "pblendvb", + "pblendw", + "pclmulqdq", + "pcmpeqb", + "pcmpeqd", + "pcmpeqq", + "pcmpeqw", + "pcmpestri", + "pcmpestrm", + "pcmpgtb", + "pcmpgtd", + "pcmpgtq", + "pcmpgtw", + "pcmpistri", + "pcmpistrm", + "pextrb", + "pextrd", + "pextrq", + "pextrw", + "pf2id", + "pf2iw", + "pfacc", + "pfadd", + "pfcmpeq", + "pfcmpge", + "pfcmpgt", + "pfmax", + "pfmin", + "pfmul", + "pfnacc", + "pfpnacc", + "pfrcp", + "pfrcpit1", + "pfrcpit2", + "pfrsqit1", + "pfrsqrt", + "pfsub", + "pfsubr", + "phaddd", + "phaddsw", + "phaddw", + "phminposuw", + "phsubd", + "phsubsw", + "phsubw", + "pi2fd", + "pi2fw", + "pinsrb", + "pinsrd", + "pinsrq", + "pinsrw", + "pmaddubsw", + "pmaddwd", + "pmaxsb", + "pmaxsd", + "pmaxsw", + "pmaxub", + "pmaxud", + "pmaxuw", + "pminsb", + "pminsd", + "pminsw", + "pminub", + "pminud", + "pminuw", + "pmovmskb", + "pmovsxbd", + "pmovsxbq", + "pmovsxbw", + "pmovsxdq", + "pmovsxwd", + "pmovsxwq", + "pmovzxbd", + "pmovzxbq", + "pmovzxbw", + "pmovzxdq", + "pmovzxwd", + "pmovzxwq", + "pmuldq", + "pmulhrsw", + "pmulhrw", + "pmulhuw", + "pmulhw", + "pmulld", + "pmullw", + "pmuludq", + "pop", + "popa", + "popad", + "popcnt", + "popfd", + "popfq", + "popfw", + "por", + "prefetch", + "prefetchnta", + "prefetcht0", + "prefetcht1", + "prefetcht2", + "psadbw", + "pshufb", + "pshufd", + "pshufhw", + "pshuflw", + "pshufw", + "psignb", + "psignd", + "psignw", + "pslld", + "pslldq", + "psllq", + "psllw", + "psrad", + "psraw", + "psrld", + "psrldq", + "psrlq", + "psrlw", + "psubb", + "psubd", + "psubq", + "psubsb", + "psubsw", + "psubusb", + "psubusw", + "psubw", + "pswapd", + "ptest", + "punpckhbw", + "punpckhdq", + "punpckhqdq", + "punpckhwd", + "punpcklbw", + "punpckldq", + "punpcklqdq", + "punpcklwd", + "push", + "pusha", + "pushad", + "pushfd", + "pushfq", + "pushfw", + "pxor", + "rcl", + "rcpps", + "rcpss", + "rcr", + "rdmsr", + "rdpmc", + "rdrand", + "rdtsc", + "rdtscp", + "rep", + "repne", + "ret", + "retf", + "rol", + "ror", + "roundpd", + "roundps", + "roundsd", + "roundss", + "rsm", + "rsqrtps", + "rsqrtss", + "sahf", + "salc", + "sar", + "sbb", + "scasb", + "scasd", + "scasq", + "scasw", + "seta", + "setae", + "setb", + "setbe", + "setg", + "setge", + "setl", + "setle", + "setno", + "setnp", + "setns", + "setnz", + "seto", + "setp", + "sets", + "setz", + "sfence", + "sgdt", + "shl", + "shld", + "shr", + "shrd", + "shufpd", + "shufps", + "sidt", + "skinit", + "sldt", + "smsw", + "sqrtpd", + "sqrtps", + "sqrtsd", + "sqrtss", + "stc", + "std", + "stgi", + "sti", + "stmxcsr", + "stosb", + "stosd", + "stosq", + "stosw", + "str", + "sub", + "subpd", + "subps", + "subsd", + "subss", + "swapgs", + "syscall", + "sysenter", + "sysexit", + "sysret", + "test", + "ucomisd", + "ucomiss", + "ud2", + "unpckhpd", + "unpckhps", + "unpcklpd", + "unpcklps", + "vaddpd", + "vaddps", + "vaddsd", + "vaddss", + "vaddsubpd", + "vaddsubps", + "vaesdec", + "vaesdeclast", + "vaesenc", + "vaesenclast", + "vaesimc", + "vaeskeygenassist", + "vandnpd", + "vandnps", + "vandpd", + "vandps", + "vblendpd", + "vblendps", + "vblendvpd", + "vblendvps", + "vbroadcastsd", + "vbroadcastss", + "vcmppd", + "vcmpps", + "vcmpsd", + "vcmpss", + "vcomisd", + "vcomiss", + "vcvtdq2pd", + "vcvtdq2ps", + "vcvtpd2dq", + "vcvtpd2ps", + "vcvtps2dq", + "vcvtps2pd", + "vcvtsd2si", + "vcvtsd2ss", + "vcvtsi2sd", + "vcvtsi2ss", + "vcvtss2sd", + "vcvtss2si", + "vcvttpd2dq", + "vcvttps2dq", + "vcvttsd2si", + "vcvttss2si", + "vdivpd", + "vdivps", + "vdivsd", + "vdivss", + "vdppd", + "vdpps", + "verr", + "verw", + "vextractf128", + "vextractps", + "vhaddpd", + "vhaddps", + "vhsubpd", + "vhsubps", + "vinsertf128", + "vinsertps", + "vlddqu", + "vmaskmovdqu", + "vmaskmovpd", + "vmaskmovps", + "vmaxpd", + "vmaxps", + "vmaxsd", + "vmaxss", + "vmcall", + "vmclear", + "vminpd", + "vminps", + "vminsd", + "vminss", + "vmlaunch", + "vmload", + "vmmcall", + "vmovapd", + "vmovaps", + "vmovd", + "vmovddup", + "vmovdqa", + "vmovdqu", + "vmovhlps", + "vmovhpd", + "vmovhps", + "vmovlhps", + "vmovlpd", + "vmovlps", + "vmovmskpd", + "vmovmskps", + "vmovntdq", + "vmovntdqa", + "vmovntpd", + "vmovntps", + "vmovq", + "vmovsd", + "vmovshdup", + "vmovsldup", + "vmovss", + "vmovupd", + "vmovups", + "vmpsadbw", + "vmptrld", + "vmptrst", + "vmread", + "vmresume", + "vmrun", + "vmsave", + "vmulpd", + "vmulps", + "vmulsd", + "vmulss", + "vmwrite", + "vmxoff", + "vmxon", + "vorpd", + "vorps", + "vpabsb", + "vpabsd", + "vpabsw", + "vpackssdw", + "vpacksswb", + "vpackusdw", + "vpackuswb", + "vpaddb", + "vpaddd", + "vpaddq", + "vpaddsb", + "vpaddsw", + "vpaddusb", + "vpaddusw", + "vpaddw", + "vpalignr", + "vpand", + "vpandn", + "vpavgb", + "vpavgw", + "vpblendvb", + "vpblendw", + "vpclmulqdq", + "vpcmpeqb", + "vpcmpeqd", + "vpcmpeqq", + "vpcmpeqw", + "vpcmpestri", + "vpcmpestrm", + "vpcmpgtb", + "vpcmpgtd", + "vpcmpgtq", + "vpcmpgtw", + "vpcmpistri", + "vpcmpistrm", + "vperm2f128", + "vpermilpd", + "vpermilps", + "vpextrb", + "vpextrd", + "vpextrq", + "vpextrw", + "vphaddd", + "vphaddsw", + "vphaddw", + "vphminposuw", + "vphsubd", + "vphsubsw", + "vphsubw", + "vpinsrb", + "vpinsrd", + "vpinsrq", + "vpinsrw", + "vpmaddubsw", + "vpmaddwd", + "vpmaxsb", + "vpmaxsd", + "vpmaxsw", + "vpmaxub", + "vpmaxud", + "vpmaxuw", + "vpminsb", + "vpminsd", + "vpminsw", + "vpminub", + "vpminud", + "vpminuw", + "vpmovmskb", + "vpmovsxbd", + "vpmovsxbq", + "vpmovsxbw", + "vpmovsxwd", + "vpmovsxwq", + "vpmovzxbd", + "vpmovzxbq", + "vpmovzxbw", + "vpmovzxdq", + "vpmovzxwd", + "vpmovzxwq", + "vpmuldq", + "vpmulhrsw", + "vpmulhuw", + "vpmulhw", + "vpmulld", + "vpmullw", + "vpor", + "vpsadbw", + "vpshufb", + "vpshufd", + "vpshufhw", + "vpshuflw", + "vpsignb", + "vpsignd", + "vpsignw", + "vpslld", + "vpslldq", + "vpsllq", + "vpsllw", + "vpsrad", + "vpsraw", + "vpsrld", + "vpsrldq", + "vpsrlq", + "vpsrlw", + "vpsubb", + "vpsubd", + "vpsubq", + "vpsubsb", + "vpsubsw", + "vpsubusb", + "vpsubusw", + "vpsubw", + "vptest", + "vpunpckhbw", + "vpunpckhdq", + "vpunpckhqdq", + "vpunpckhwd", + "vpunpcklbw", + "vpunpckldq", + "vpunpcklqdq", + "vpunpcklwd", + "vpxor", + "vrcpps", + "vrcpss", + "vroundpd", + "vroundps", + "vroundsd", + "vroundss", + "vrsqrtps", + "vrsqrtss", + "vshufpd", + "vshufps", + "vsqrtpd", + "vsqrtps", + "vsqrtsd", + "vsqrtss", + "vstmxcsr", + "vsubpd", + "vsubps", + "vsubsd", + "vsubss", + "vtestpd", + "vtestps", + "vucomisd", + "vucomiss", + "vunpckhpd", + "vunpckhps", + "vunpcklpd", + "vunpcklps", + "vxorpd", + "vxorps", + "vzeroall", + "vzeroupper", + "wait", + "wbinvd", + "wrmsr", + "xadd", + "xchg", + "xcryptcbc", + "xcryptcfb", + "xcryptctr", + "xcryptecb", + "xcryptofb", + "xgetbv", + "xlatb", + "xor", + "xorpd", + "xorps", + "xrstor", + "xsave", + "xsetbv", + "xsha1", + "xsha256", + "xstore", + "invalid", + "3dnow", + "none", + "db", + "pause" +}; diff --git a/ext/opcache/jit/libudis86/itab.h b/ext/opcache/jit/libudis86/itab.h new file mode 100644 index 0000000000000..3d54c43546203 --- /dev/null +++ b/ext/opcache/jit/libudis86/itab.h @@ -0,0 +1,939 @@ +#ifndef UD_ITAB_H +#define UD_ITAB_H + +/* itab.h -- generated by udis86:scripts/ud_itab.py, do no edit */ + +/* ud_table_type -- lookup table types (see decode.c) */ +enum ud_table_type { + UD_TAB__OPC_VEX, + UD_TAB__OPC_TABLE, + UD_TAB__OPC_X87, + UD_TAB__OPC_MOD, + UD_TAB__OPC_RM, + UD_TAB__OPC_OSIZE, + UD_TAB__OPC_MODE, + UD_TAB__OPC_VEX_L, + UD_TAB__OPC_3DNOW, + UD_TAB__OPC_REG, + UD_TAB__OPC_ASIZE, + UD_TAB__OPC_VEX_W, + UD_TAB__OPC_SSE, + UD_TAB__OPC_VENDOR +}; + +/* ud_mnemonic -- mnemonic constants */ +enum ud_mnemonic_code { + UD_Iaaa, + UD_Iaad, + UD_Iaam, + UD_Iaas, + UD_Iadc, + UD_Iadd, + UD_Iaddpd, + UD_Iaddps, + UD_Iaddsd, + UD_Iaddss, + UD_Iaddsubpd, + UD_Iaddsubps, + UD_Iaesdec, + UD_Iaesdeclast, + UD_Iaesenc, + UD_Iaesenclast, + UD_Iaesimc, + UD_Iaeskeygenassist, + UD_Iand, + UD_Iandnpd, + UD_Iandnps, + UD_Iandpd, + UD_Iandps, + UD_Iarpl, + UD_Iblendpd, + UD_Iblendps, + UD_Iblendvpd, + UD_Iblendvps, + UD_Ibound, + UD_Ibsf, + UD_Ibsr, + UD_Ibswap, + UD_Ibt, + UD_Ibtc, + UD_Ibtr, + UD_Ibts, + UD_Icall, + UD_Icbw, + UD_Icdq, + UD_Icdqe, + UD_Iclc, + UD_Icld, + UD_Iclflush, + UD_Iclgi, + UD_Icli, + UD_Iclts, + UD_Icmc, + UD_Icmova, + UD_Icmovae, + UD_Icmovb, + UD_Icmovbe, + UD_Icmovg, + UD_Icmovge, + UD_Icmovl, + UD_Icmovle, + UD_Icmovno, + UD_Icmovnp, + UD_Icmovns, + UD_Icmovnz, + UD_Icmovo, + UD_Icmovp, + UD_Icmovs, + UD_Icmovz, + UD_Icmp, + UD_Icmppd, + UD_Icmpps, + UD_Icmpsb, + UD_Icmpsd, + UD_Icmpsq, + UD_Icmpss, + UD_Icmpsw, + UD_Icmpxchg, + UD_Icmpxchg16b, + UD_Icmpxchg8b, + UD_Icomisd, + UD_Icomiss, + UD_Icpuid, + UD_Icqo, + UD_Icrc32, + UD_Icvtdq2pd, + UD_Icvtdq2ps, + UD_Icvtpd2dq, + UD_Icvtpd2pi, + UD_Icvtpd2ps, + UD_Icvtpi2pd, + UD_Icvtpi2ps, + UD_Icvtps2dq, + UD_Icvtps2pd, + UD_Icvtps2pi, + UD_Icvtsd2si, + UD_Icvtsd2ss, + UD_Icvtsi2sd, + UD_Icvtsi2ss, + UD_Icvtss2sd, + UD_Icvtss2si, + UD_Icvttpd2dq, + UD_Icvttpd2pi, + UD_Icvttps2dq, + UD_Icvttps2pi, + UD_Icvttsd2si, + UD_Icvttss2si, + UD_Icwd, + UD_Icwde, + UD_Idaa, + UD_Idas, + UD_Idec, + UD_Idiv, + UD_Idivpd, + UD_Idivps, + UD_Idivsd, + UD_Idivss, + UD_Idppd, + UD_Idpps, + UD_Iemms, + UD_Ienter, + UD_Iextractps, + UD_If2xm1, + UD_Ifabs, + UD_Ifadd, + UD_Ifaddp, + UD_Ifbld, + UD_Ifbstp, + UD_Ifchs, + UD_Ifclex, + UD_Ifcmovb, + UD_Ifcmovbe, + UD_Ifcmove, + UD_Ifcmovnb, + UD_Ifcmovnbe, + UD_Ifcmovne, + UD_Ifcmovnu, + UD_Ifcmovu, + UD_Ifcom, + UD_Ifcom2, + UD_Ifcomi, + UD_Ifcomip, + UD_Ifcomp, + UD_Ifcomp3, + UD_Ifcomp5, + UD_Ifcompp, + UD_Ifcos, + UD_Ifdecstp, + UD_Ifdiv, + UD_Ifdivp, + UD_Ifdivr, + UD_Ifdivrp, + UD_Ifemms, + UD_Iffree, + UD_Iffreep, + UD_Ifiadd, + UD_Ificom, + UD_Ificomp, + UD_Ifidiv, + UD_Ifidivr, + UD_Ifild, + UD_Ifimul, + UD_Ifincstp, + UD_Ifist, + UD_Ifistp, + UD_Ifisttp, + UD_Ifisub, + UD_Ifisubr, + UD_Ifld, + UD_Ifld1, + UD_Ifldcw, + UD_Ifldenv, + UD_Ifldl2e, + UD_Ifldl2t, + UD_Ifldlg2, + UD_Ifldln2, + UD_Ifldpi, + UD_Ifldz, + UD_Ifmul, + UD_Ifmulp, + UD_Ifndisi, + UD_Ifneni, + UD_Ifninit, + UD_Ifnop, + UD_Ifnsave, + UD_Ifnsetpm, + UD_Ifnstcw, + UD_Ifnstenv, + UD_Ifnstsw, + UD_Ifpatan, + UD_Ifprem, + UD_Ifprem1, + UD_Ifptan, + UD_Ifrndint, + UD_Ifrstor, + UD_Ifrstpm, + UD_Ifscale, + UD_Ifsin, + UD_Ifsincos, + UD_Ifsqrt, + UD_Ifst, + UD_Ifstp, + UD_Ifstp1, + UD_Ifstp8, + UD_Ifstp9, + UD_Ifsub, + UD_Ifsubp, + UD_Ifsubr, + UD_Ifsubrp, + UD_Iftst, + UD_Ifucom, + UD_Ifucomi, + UD_Ifucomip, + UD_Ifucomp, + UD_Ifucompp, + UD_Ifxam, + UD_Ifxch, + UD_Ifxch4, + UD_Ifxch7, + UD_Ifxrstor, + UD_Ifxsave, + UD_Ifxtract, + UD_Ifyl2x, + UD_Ifyl2xp1, + UD_Igetsec, + UD_Ihaddpd, + UD_Ihaddps, + UD_Ihlt, + UD_Ihsubpd, + UD_Ihsubps, + UD_Iidiv, + UD_Iimul, + UD_Iin, + UD_Iinc, + UD_Iinsb, + UD_Iinsd, + UD_Iinsertps, + UD_Iinsw, + UD_Iint, + UD_Iint1, + UD_Iint3, + UD_Iinto, + UD_Iinvd, + UD_Iinvept, + UD_Iinvlpg, + UD_Iinvlpga, + UD_Iinvvpid, + UD_Iiretd, + UD_Iiretq, + UD_Iiretw, + UD_Ija, + UD_Ijae, + UD_Ijb, + UD_Ijbe, + UD_Ijcxz, + UD_Ijecxz, + UD_Ijg, + UD_Ijge, + UD_Ijl, + UD_Ijle, + UD_Ijmp, + UD_Ijno, + UD_Ijnp, + UD_Ijns, + UD_Ijnz, + UD_Ijo, + UD_Ijp, + UD_Ijrcxz, + UD_Ijs, + UD_Ijz, + UD_Ilahf, + UD_Ilar, + UD_Ilddqu, + UD_Ildmxcsr, + UD_Ilds, + UD_Ilea, + UD_Ileave, + UD_Iles, + UD_Ilfence, + UD_Ilfs, + UD_Ilgdt, + UD_Ilgs, + UD_Ilidt, + UD_Illdt, + UD_Ilmsw, + UD_Ilock, + UD_Ilodsb, + UD_Ilodsd, + UD_Ilodsq, + UD_Ilodsw, + UD_Iloop, + UD_Iloope, + UD_Iloopne, + UD_Ilsl, + UD_Ilss, + UD_Iltr, + UD_Imaskmovdqu, + UD_Imaskmovq, + UD_Imaxpd, + UD_Imaxps, + UD_Imaxsd, + UD_Imaxss, + UD_Imfence, + UD_Iminpd, + UD_Iminps, + UD_Iminsd, + UD_Iminss, + UD_Imonitor, + UD_Imontmul, + UD_Imov, + UD_Imovapd, + UD_Imovaps, + UD_Imovbe, + UD_Imovd, + UD_Imovddup, + UD_Imovdq2q, + UD_Imovdqa, + UD_Imovdqu, + UD_Imovhlps, + UD_Imovhpd, + UD_Imovhps, + UD_Imovlhps, + UD_Imovlpd, + UD_Imovlps, + UD_Imovmskpd, + UD_Imovmskps, + UD_Imovntdq, + UD_Imovntdqa, + UD_Imovnti, + UD_Imovntpd, + UD_Imovntps, + UD_Imovntq, + UD_Imovq, + UD_Imovq2dq, + UD_Imovsb, + UD_Imovsd, + UD_Imovshdup, + UD_Imovsldup, + UD_Imovsq, + UD_Imovss, + UD_Imovsw, + UD_Imovsx, + UD_Imovsxd, + UD_Imovupd, + UD_Imovups, + UD_Imovzx, + UD_Impsadbw, + UD_Imul, + UD_Imulpd, + UD_Imulps, + UD_Imulsd, + UD_Imulss, + UD_Imwait, + UD_Ineg, + UD_Inop, + UD_Inot, + UD_Ior, + UD_Iorpd, + UD_Iorps, + UD_Iout, + UD_Ioutsb, + UD_Ioutsd, + UD_Ioutsw, + UD_Ipabsb, + UD_Ipabsd, + UD_Ipabsw, + UD_Ipackssdw, + UD_Ipacksswb, + UD_Ipackusdw, + UD_Ipackuswb, + UD_Ipaddb, + UD_Ipaddd, + UD_Ipaddq, + UD_Ipaddsb, + UD_Ipaddsw, + UD_Ipaddusb, + UD_Ipaddusw, + UD_Ipaddw, + UD_Ipalignr, + UD_Ipand, + UD_Ipandn, + UD_Ipavgb, + UD_Ipavgusb, + UD_Ipavgw, + UD_Ipblendvb, + UD_Ipblendw, + UD_Ipclmulqdq, + UD_Ipcmpeqb, + UD_Ipcmpeqd, + UD_Ipcmpeqq, + UD_Ipcmpeqw, + UD_Ipcmpestri, + UD_Ipcmpestrm, + UD_Ipcmpgtb, + UD_Ipcmpgtd, + UD_Ipcmpgtq, + UD_Ipcmpgtw, + UD_Ipcmpistri, + UD_Ipcmpistrm, + UD_Ipextrb, + UD_Ipextrd, + UD_Ipextrq, + UD_Ipextrw, + UD_Ipf2id, + UD_Ipf2iw, + UD_Ipfacc, + UD_Ipfadd, + UD_Ipfcmpeq, + UD_Ipfcmpge, + UD_Ipfcmpgt, + UD_Ipfmax, + UD_Ipfmin, + UD_Ipfmul, + UD_Ipfnacc, + UD_Ipfpnacc, + UD_Ipfrcp, + UD_Ipfrcpit1, + UD_Ipfrcpit2, + UD_Ipfrsqit1, + UD_Ipfrsqrt, + UD_Ipfsub, + UD_Ipfsubr, + UD_Iphaddd, + UD_Iphaddsw, + UD_Iphaddw, + UD_Iphminposuw, + UD_Iphsubd, + UD_Iphsubsw, + UD_Iphsubw, + UD_Ipi2fd, + UD_Ipi2fw, + UD_Ipinsrb, + UD_Ipinsrd, + UD_Ipinsrq, + UD_Ipinsrw, + UD_Ipmaddubsw, + UD_Ipmaddwd, + UD_Ipmaxsb, + UD_Ipmaxsd, + UD_Ipmaxsw, + UD_Ipmaxub, + UD_Ipmaxud, + UD_Ipmaxuw, + UD_Ipminsb, + UD_Ipminsd, + UD_Ipminsw, + UD_Ipminub, + UD_Ipminud, + UD_Ipminuw, + UD_Ipmovmskb, + UD_Ipmovsxbd, + UD_Ipmovsxbq, + UD_Ipmovsxbw, + UD_Ipmovsxdq, + UD_Ipmovsxwd, + UD_Ipmovsxwq, + UD_Ipmovzxbd, + UD_Ipmovzxbq, + UD_Ipmovzxbw, + UD_Ipmovzxdq, + UD_Ipmovzxwd, + UD_Ipmovzxwq, + UD_Ipmuldq, + UD_Ipmulhrsw, + UD_Ipmulhrw, + UD_Ipmulhuw, + UD_Ipmulhw, + UD_Ipmulld, + UD_Ipmullw, + UD_Ipmuludq, + UD_Ipop, + UD_Ipopa, + UD_Ipopad, + UD_Ipopcnt, + UD_Ipopfd, + UD_Ipopfq, + UD_Ipopfw, + UD_Ipor, + UD_Iprefetch, + UD_Iprefetchnta, + UD_Iprefetcht0, + UD_Iprefetcht1, + UD_Iprefetcht2, + UD_Ipsadbw, + UD_Ipshufb, + UD_Ipshufd, + UD_Ipshufhw, + UD_Ipshuflw, + UD_Ipshufw, + UD_Ipsignb, + UD_Ipsignd, + UD_Ipsignw, + UD_Ipslld, + UD_Ipslldq, + UD_Ipsllq, + UD_Ipsllw, + UD_Ipsrad, + UD_Ipsraw, + UD_Ipsrld, + UD_Ipsrldq, + UD_Ipsrlq, + UD_Ipsrlw, + UD_Ipsubb, + UD_Ipsubd, + UD_Ipsubq, + UD_Ipsubsb, + UD_Ipsubsw, + UD_Ipsubusb, + UD_Ipsubusw, + UD_Ipsubw, + UD_Ipswapd, + UD_Iptest, + UD_Ipunpckhbw, + UD_Ipunpckhdq, + UD_Ipunpckhqdq, + UD_Ipunpckhwd, + UD_Ipunpcklbw, + UD_Ipunpckldq, + UD_Ipunpcklqdq, + UD_Ipunpcklwd, + UD_Ipush, + UD_Ipusha, + UD_Ipushad, + UD_Ipushfd, + UD_Ipushfq, + UD_Ipushfw, + UD_Ipxor, + UD_Ircl, + UD_Ircpps, + UD_Ircpss, + UD_Ircr, + UD_Irdmsr, + UD_Irdpmc, + UD_Irdrand, + UD_Irdtsc, + UD_Irdtscp, + UD_Irep, + UD_Irepne, + UD_Iret, + UD_Iretf, + UD_Irol, + UD_Iror, + UD_Iroundpd, + UD_Iroundps, + UD_Iroundsd, + UD_Iroundss, + UD_Irsm, + UD_Irsqrtps, + UD_Irsqrtss, + UD_Isahf, + UD_Isalc, + UD_Isar, + UD_Isbb, + UD_Iscasb, + UD_Iscasd, + UD_Iscasq, + UD_Iscasw, + UD_Iseta, + UD_Isetae, + UD_Isetb, + UD_Isetbe, + UD_Isetg, + UD_Isetge, + UD_Isetl, + UD_Isetle, + UD_Isetno, + UD_Isetnp, + UD_Isetns, + UD_Isetnz, + UD_Iseto, + UD_Isetp, + UD_Isets, + UD_Isetz, + UD_Isfence, + UD_Isgdt, + UD_Ishl, + UD_Ishld, + UD_Ishr, + UD_Ishrd, + UD_Ishufpd, + UD_Ishufps, + UD_Isidt, + UD_Iskinit, + UD_Isldt, + UD_Ismsw, + UD_Isqrtpd, + UD_Isqrtps, + UD_Isqrtsd, + UD_Isqrtss, + UD_Istc, + UD_Istd, + UD_Istgi, + UD_Isti, + UD_Istmxcsr, + UD_Istosb, + UD_Istosd, + UD_Istosq, + UD_Istosw, + UD_Istr, + UD_Isub, + UD_Isubpd, + UD_Isubps, + UD_Isubsd, + UD_Isubss, + UD_Iswapgs, + UD_Isyscall, + UD_Isysenter, + UD_Isysexit, + UD_Isysret, + UD_Itest, + UD_Iucomisd, + UD_Iucomiss, + UD_Iud2, + UD_Iunpckhpd, + UD_Iunpckhps, + UD_Iunpcklpd, + UD_Iunpcklps, + UD_Ivaddpd, + UD_Ivaddps, + UD_Ivaddsd, + UD_Ivaddss, + UD_Ivaddsubpd, + UD_Ivaddsubps, + UD_Ivaesdec, + UD_Ivaesdeclast, + UD_Ivaesenc, + UD_Ivaesenclast, + UD_Ivaesimc, + UD_Ivaeskeygenassist, + UD_Ivandnpd, + UD_Ivandnps, + UD_Ivandpd, + UD_Ivandps, + UD_Ivblendpd, + UD_Ivblendps, + UD_Ivblendvpd, + UD_Ivblendvps, + UD_Ivbroadcastsd, + UD_Ivbroadcastss, + UD_Ivcmppd, + UD_Ivcmpps, + UD_Ivcmpsd, + UD_Ivcmpss, + UD_Ivcomisd, + UD_Ivcomiss, + UD_Ivcvtdq2pd, + UD_Ivcvtdq2ps, + UD_Ivcvtpd2dq, + UD_Ivcvtpd2ps, + UD_Ivcvtps2dq, + UD_Ivcvtps2pd, + UD_Ivcvtsd2si, + UD_Ivcvtsd2ss, + UD_Ivcvtsi2sd, + UD_Ivcvtsi2ss, + UD_Ivcvtss2sd, + UD_Ivcvtss2si, + UD_Ivcvttpd2dq, + UD_Ivcvttps2dq, + UD_Ivcvttsd2si, + UD_Ivcvttss2si, + UD_Ivdivpd, + UD_Ivdivps, + UD_Ivdivsd, + UD_Ivdivss, + UD_Ivdppd, + UD_Ivdpps, + UD_Iverr, + UD_Iverw, + UD_Ivextractf128, + UD_Ivextractps, + UD_Ivhaddpd, + UD_Ivhaddps, + UD_Ivhsubpd, + UD_Ivhsubps, + UD_Ivinsertf128, + UD_Ivinsertps, + UD_Ivlddqu, + UD_Ivmaskmovdqu, + UD_Ivmaskmovpd, + UD_Ivmaskmovps, + UD_Ivmaxpd, + UD_Ivmaxps, + UD_Ivmaxsd, + UD_Ivmaxss, + UD_Ivmcall, + UD_Ivmclear, + UD_Ivminpd, + UD_Ivminps, + UD_Ivminsd, + UD_Ivminss, + UD_Ivmlaunch, + UD_Ivmload, + UD_Ivmmcall, + UD_Ivmovapd, + UD_Ivmovaps, + UD_Ivmovd, + UD_Ivmovddup, + UD_Ivmovdqa, + UD_Ivmovdqu, + UD_Ivmovhlps, + UD_Ivmovhpd, + UD_Ivmovhps, + UD_Ivmovlhps, + UD_Ivmovlpd, + UD_Ivmovlps, + UD_Ivmovmskpd, + UD_Ivmovmskps, + UD_Ivmovntdq, + UD_Ivmovntdqa, + UD_Ivmovntpd, + UD_Ivmovntps, + UD_Ivmovq, + UD_Ivmovsd, + UD_Ivmovshdup, + UD_Ivmovsldup, + UD_Ivmovss, + UD_Ivmovupd, + UD_Ivmovups, + UD_Ivmpsadbw, + UD_Ivmptrld, + UD_Ivmptrst, + UD_Ivmread, + UD_Ivmresume, + UD_Ivmrun, + UD_Ivmsave, + UD_Ivmulpd, + UD_Ivmulps, + UD_Ivmulsd, + UD_Ivmulss, + UD_Ivmwrite, + UD_Ivmxoff, + UD_Ivmxon, + UD_Ivorpd, + UD_Ivorps, + UD_Ivpabsb, + UD_Ivpabsd, + UD_Ivpabsw, + UD_Ivpackssdw, + UD_Ivpacksswb, + UD_Ivpackusdw, + UD_Ivpackuswb, + UD_Ivpaddb, + UD_Ivpaddd, + UD_Ivpaddq, + UD_Ivpaddsb, + UD_Ivpaddsw, + UD_Ivpaddusb, + UD_Ivpaddusw, + UD_Ivpaddw, + UD_Ivpalignr, + UD_Ivpand, + UD_Ivpandn, + UD_Ivpavgb, + UD_Ivpavgw, + UD_Ivpblendvb, + UD_Ivpblendw, + UD_Ivpclmulqdq, + UD_Ivpcmpeqb, + UD_Ivpcmpeqd, + UD_Ivpcmpeqq, + UD_Ivpcmpeqw, + UD_Ivpcmpestri, + UD_Ivpcmpestrm, + UD_Ivpcmpgtb, + UD_Ivpcmpgtd, + UD_Ivpcmpgtq, + UD_Ivpcmpgtw, + UD_Ivpcmpistri, + UD_Ivpcmpistrm, + UD_Ivperm2f128, + UD_Ivpermilpd, + UD_Ivpermilps, + UD_Ivpextrb, + UD_Ivpextrd, + UD_Ivpextrq, + UD_Ivpextrw, + UD_Ivphaddd, + UD_Ivphaddsw, + UD_Ivphaddw, + UD_Ivphminposuw, + UD_Ivphsubd, + UD_Ivphsubsw, + UD_Ivphsubw, + UD_Ivpinsrb, + UD_Ivpinsrd, + UD_Ivpinsrq, + UD_Ivpinsrw, + UD_Ivpmaddubsw, + UD_Ivpmaddwd, + UD_Ivpmaxsb, + UD_Ivpmaxsd, + UD_Ivpmaxsw, + UD_Ivpmaxub, + UD_Ivpmaxud, + UD_Ivpmaxuw, + UD_Ivpminsb, + UD_Ivpminsd, + UD_Ivpminsw, + UD_Ivpminub, + UD_Ivpminud, + UD_Ivpminuw, + UD_Ivpmovmskb, + UD_Ivpmovsxbd, + UD_Ivpmovsxbq, + UD_Ivpmovsxbw, + UD_Ivpmovsxwd, + UD_Ivpmovsxwq, + UD_Ivpmovzxbd, + UD_Ivpmovzxbq, + UD_Ivpmovzxbw, + UD_Ivpmovzxdq, + UD_Ivpmovzxwd, + UD_Ivpmovzxwq, + UD_Ivpmuldq, + UD_Ivpmulhrsw, + UD_Ivpmulhuw, + UD_Ivpmulhw, + UD_Ivpmulld, + UD_Ivpmullw, + UD_Ivpor, + UD_Ivpsadbw, + UD_Ivpshufb, + UD_Ivpshufd, + UD_Ivpshufhw, + UD_Ivpshuflw, + UD_Ivpsignb, + UD_Ivpsignd, + UD_Ivpsignw, + UD_Ivpslld, + UD_Ivpslldq, + UD_Ivpsllq, + UD_Ivpsllw, + UD_Ivpsrad, + UD_Ivpsraw, + UD_Ivpsrld, + UD_Ivpsrldq, + UD_Ivpsrlq, + UD_Ivpsrlw, + UD_Ivpsubb, + UD_Ivpsubd, + UD_Ivpsubq, + UD_Ivpsubsb, + UD_Ivpsubsw, + UD_Ivpsubusb, + UD_Ivpsubusw, + UD_Ivpsubw, + UD_Ivptest, + UD_Ivpunpckhbw, + UD_Ivpunpckhdq, + UD_Ivpunpckhqdq, + UD_Ivpunpckhwd, + UD_Ivpunpcklbw, + UD_Ivpunpckldq, + UD_Ivpunpcklqdq, + UD_Ivpunpcklwd, + UD_Ivpxor, + UD_Ivrcpps, + UD_Ivrcpss, + UD_Ivroundpd, + UD_Ivroundps, + UD_Ivroundsd, + UD_Ivroundss, + UD_Ivrsqrtps, + UD_Ivrsqrtss, + UD_Ivshufpd, + UD_Ivshufps, + UD_Ivsqrtpd, + UD_Ivsqrtps, + UD_Ivsqrtsd, + UD_Ivsqrtss, + UD_Ivstmxcsr, + UD_Ivsubpd, + UD_Ivsubps, + UD_Ivsubsd, + UD_Ivsubss, + UD_Ivtestpd, + UD_Ivtestps, + UD_Ivucomisd, + UD_Ivucomiss, + UD_Ivunpckhpd, + UD_Ivunpckhps, + UD_Ivunpcklpd, + UD_Ivunpcklps, + UD_Ivxorpd, + UD_Ivxorps, + UD_Ivzeroall, + UD_Ivzeroupper, + UD_Iwait, + UD_Iwbinvd, + UD_Iwrmsr, + UD_Ixadd, + UD_Ixchg, + UD_Ixcryptcbc, + UD_Ixcryptcfb, + UD_Ixcryptctr, + UD_Ixcryptecb, + UD_Ixcryptofb, + UD_Ixgetbv, + UD_Ixlatb, + UD_Ixor, + UD_Ixorpd, + UD_Ixorps, + UD_Ixrstor, + UD_Ixsave, + UD_Ixsetbv, + UD_Ixsha1, + UD_Ixsha256, + UD_Ixstore, + UD_Iinvalid, + UD_I3dnow, + UD_Inone, + UD_Idb, + UD_Ipause, + UD_MAX_MNEMONIC_CODE +}; + +extern const char * ud_mnemonics_str[]; + +#endif /* UD_ITAB_H */ diff --git a/ext/opcache/jit/libudis86/syn-att.c b/ext/opcache/jit/libudis86/syn-att.c new file mode 100644 index 0000000000000..d1ba89b7ae217 --- /dev/null +++ b/ext/opcache/jit/libudis86/syn-att.c @@ -0,0 +1,228 @@ +/* udis86 - libudis86/syn-att.c + * + * Copyright (c) 2002-2009 Vivek Thampi + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#include "types.h" +#include "extern.h" +#include "decode.h" +#include "itab.h" +#include "syn.h" +#include "udint.h" + +/* ----------------------------------------------------------------------------- + * opr_cast() - Prints an operand cast. + * ----------------------------------------------------------------------------- + */ +static void +opr_cast(struct ud* u, struct ud_operand* op) +{ + switch(op->size) { + case 16 : case 32 : + ud_asmprintf(u, "*"); break; + default: break; + } +} + +/* ----------------------------------------------------------------------------- + * gen_operand() - Generates assembly output for each operand. + * ----------------------------------------------------------------------------- + */ +static void +gen_operand(struct ud* u, struct ud_operand* op) +{ + switch(op->type) { + case UD_OP_CONST: + ud_asmprintf(u, "$0x%x", op->lval.udword); + break; + + case UD_OP_REG: + ud_asmprintf(u, "%%%s", ud_reg_tab[op->base - UD_R_AL]); + break; + + case UD_OP_MEM: + if (u->br_far) { + opr_cast(u, op); + } + if (u->pfx_seg) { + ud_asmprintf(u, "%%%s:", ud_reg_tab[u->pfx_seg - UD_R_AL]); + } + if (op->offset != 0) { + ud_syn_print_mem_disp(u, op, 0); + } + if (op->base) { + ud_asmprintf(u, "(%%%s", ud_reg_tab[op->base - UD_R_AL]); + } + if (op->index) { + if (op->base) { + ud_asmprintf(u, ","); + } else { + ud_asmprintf(u, "("); + } + ud_asmprintf(u, "%%%s", ud_reg_tab[op->index - UD_R_AL]); + } + if (op->scale) { + ud_asmprintf(u, ",%d", op->scale); + } + if (op->base || op->index) { + ud_asmprintf(u, ")"); + } + break; + + case UD_OP_IMM: + ud_asmprintf(u, "$"); + ud_syn_print_imm(u, op); + break; + + case UD_OP_JIMM: + ud_syn_print_addr(u, ud_syn_rel_target(u, op)); + break; + + case UD_OP_PTR: + switch (op->size) { + case 32: + ud_asmprintf(u, "$0x%x, $0x%x", op->lval.ptr.seg, + op->lval.ptr.off & 0xFFFF); + break; + case 48: + ud_asmprintf(u, "$0x%x, $0x%x", op->lval.ptr.seg, + op->lval.ptr.off); + break; + } + break; + + default: return; + } +} + +/* ============================================================================= + * translates to AT&T syntax + * ============================================================================= + */ +extern void +ud_translate_att(struct ud *u) +{ + int size = 0; + int star = 0; + + /* check if P_OSO prefix is used */ + if (! P_OSO(u->itab_entry->prefix) && u->pfx_opr) { + switch (u->dis_mode) { + case 16: + ud_asmprintf(u, "o32 "); + break; + case 32: + case 64: + ud_asmprintf(u, "o16 "); + break; + } + } + + /* check if P_ASO prefix was used */ + if (! P_ASO(u->itab_entry->prefix) && u->pfx_adr) { + switch (u->dis_mode) { + case 16: + ud_asmprintf(u, "a32 "); + break; + case 32: + ud_asmprintf(u, "a16 "); + break; + case 64: + ud_asmprintf(u, "a32 "); + break; + } + } + + if (u->pfx_lock) + ud_asmprintf(u, "lock "); + if (u->pfx_rep) { + ud_asmprintf(u, "rep "); + } else if (u->pfx_repe) { + ud_asmprintf(u, "repe "); + } else if (u->pfx_repne) { + ud_asmprintf(u, "repne "); + } + + /* special instructions */ + switch (u->mnemonic) { + case UD_Iretf: + ud_asmprintf(u, "lret "); + break; + case UD_Idb: + ud_asmprintf(u, ".byte 0x%x", u->operand[0].lval.ubyte); + return; + case UD_Ijmp: + case UD_Icall: + if (u->br_far) ud_asmprintf(u, "l"); + if (u->operand[0].type == UD_OP_REG) { + star = 1; + } + ud_asmprintf(u, "%s", ud_lookup_mnemonic(u->mnemonic)); + break; + case UD_Ibound: + case UD_Ienter: + if (u->operand[0].type != UD_NONE) + gen_operand(u, &u->operand[0]); + if (u->operand[1].type != UD_NONE) { + ud_asmprintf(u, ","); + gen_operand(u, &u->operand[1]); + } + return; + default: + ud_asmprintf(u, "%s", ud_lookup_mnemonic(u->mnemonic)); + } + + if (size == 8) { + ud_asmprintf(u, "b"); + } else if (size == 16) { + ud_asmprintf(u, "w"); + } else if (size == 64) { + ud_asmprintf(u, "q"); + } + + if (star) { + ud_asmprintf(u, " *"); + } else { + ud_asmprintf(u, " "); + } + + if (u->operand[3].type != UD_NONE) { + gen_operand(u, &u->operand[3]); + ud_asmprintf(u, ", "); + } + if (u->operand[2].type != UD_NONE) { + gen_operand(u, &u->operand[2]); + ud_asmprintf(u, ", "); + } + if (u->operand[1].type != UD_NONE) { + gen_operand(u, &u->operand[1]); + ud_asmprintf(u, ", "); + } + if (u->operand[0].type != UD_NONE) { + gen_operand(u, &u->operand[0]); + } +} + +/* +vim: set ts=2 sw=2 expandtab +*/ diff --git a/ext/opcache/jit/libudis86/syn-intel.c b/ext/opcache/jit/libudis86/syn-intel.c new file mode 100644 index 0000000000000..0664fea092770 --- /dev/null +++ b/ext/opcache/jit/libudis86/syn-intel.c @@ -0,0 +1,224 @@ +/* udis86 - libudis86/syn-intel.c + * + * Copyright (c) 2002-2013 Vivek Thampi + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#include "types.h" +#include "extern.h" +#include "decode.h" +#include "itab.h" +#include "syn.h" +#include "udint.h" + +/* ----------------------------------------------------------------------------- + * opr_cast() - Prints an operand cast. + * ----------------------------------------------------------------------------- + */ +static void +opr_cast(struct ud* u, struct ud_operand* op) +{ + if (u->br_far) { + ud_asmprintf(u, "far "); + } + switch(op->size) { + case 8: ud_asmprintf(u, "byte " ); break; + case 16: ud_asmprintf(u, "word " ); break; + case 32: ud_asmprintf(u, "dword "); break; + case 64: ud_asmprintf(u, "qword "); break; + case 80: ud_asmprintf(u, "tword "); break; + case 128: ud_asmprintf(u, "oword "); break; + case 256: ud_asmprintf(u, "yword "); break; + default: break; + } +} + +/* ----------------------------------------------------------------------------- + * gen_operand() - Generates assembly output for each operand. + * ----------------------------------------------------------------------------- + */ +static void gen_operand(struct ud* u, struct ud_operand* op, int syn_cast) +{ + switch(op->type) { + case UD_OP_REG: + ud_asmprintf(u, "%s", ud_reg_tab[op->base - UD_R_AL]); + break; + + case UD_OP_MEM: + if (syn_cast) { + opr_cast(u, op); + } + ud_asmprintf(u, "["); + if (u->pfx_seg) { + ud_asmprintf(u, "%s:", ud_reg_tab[u->pfx_seg - UD_R_AL]); + } + if (op->base) { + ud_asmprintf(u, "%s", ud_reg_tab[op->base - UD_R_AL]); + } + if (op->index) { + ud_asmprintf(u, "%s%s", op->base != UD_NONE? "+" : "", + ud_reg_tab[op->index - UD_R_AL]); + if (op->scale) { + ud_asmprintf(u, "*%d", op->scale); + } + } + if (op->offset != 0) { + ud_syn_print_mem_disp(u, op, (op->base != UD_NONE || + op->index != UD_NONE) ? 1 : 0); + } + ud_asmprintf(u, "]"); + break; + + case UD_OP_IMM: + ud_syn_print_imm(u, op); + break; + + + case UD_OP_JIMM: + ud_syn_print_addr(u, ud_syn_rel_target(u, op)); + break; + + case UD_OP_PTR: + switch (op->size) { + case 32: + ud_asmprintf(u, "word 0x%x:0x%x", op->lval.ptr.seg, + op->lval.ptr.off & 0xFFFF); + break; + case 48: + ud_asmprintf(u, "dword 0x%x:0x%x", op->lval.ptr.seg, + op->lval.ptr.off); + break; + } + break; + + case UD_OP_CONST: + if (syn_cast) opr_cast(u, op); + ud_asmprintf(u, "%d", op->lval.udword); + break; + + default: return; + } +} + +/* ============================================================================= + * translates to intel syntax + * ============================================================================= + */ +extern void +ud_translate_intel(struct ud* u) +{ + /* check if P_OSO prefix is used */ + if (!P_OSO(u->itab_entry->prefix) && u->pfx_opr) { + switch (u->dis_mode) { + case 16: ud_asmprintf(u, "o32 "); break; + case 32: + case 64: ud_asmprintf(u, "o16 "); break; + } + } + + /* check if P_ASO prefix was used */ + if (!P_ASO(u->itab_entry->prefix) && u->pfx_adr) { + switch (u->dis_mode) { + case 16: ud_asmprintf(u, "a32 "); break; + case 32: ud_asmprintf(u, "a16 "); break; + case 64: ud_asmprintf(u, "a32 "); break; + } + } + + if (u->pfx_seg && + u->operand[0].type != UD_OP_MEM && + u->operand[1].type != UD_OP_MEM ) { + ud_asmprintf(u, "%s ", ud_reg_tab[u->pfx_seg - UD_R_AL]); + } + + if (u->pfx_lock) { + ud_asmprintf(u, "lock "); + } + if (u->pfx_rep) { + ud_asmprintf(u, "rep "); + } else if (u->pfx_repe) { + ud_asmprintf(u, "repe "); + } else if (u->pfx_repne) { + ud_asmprintf(u, "repne "); + } + + /* print the instruction mnemonic */ + ud_asmprintf(u, "%s", ud_lookup_mnemonic(u->mnemonic)); + + if (u->operand[0].type != UD_NONE) { + int cast = 0; + ud_asmprintf(u, " "); + if (u->operand[0].type == UD_OP_MEM) { + if (u->operand[1].type == UD_OP_IMM || + u->operand[1].type == UD_OP_CONST || + u->operand[1].type == UD_NONE || + (u->operand[0].size != u->operand[1].size)) { + cast = 1; + } else if (u->operand[1].type == UD_OP_REG && + u->operand[1].base == UD_R_CL) { + switch (u->mnemonic) { + case UD_Ircl: + case UD_Irol: + case UD_Iror: + case UD_Ircr: + case UD_Ishl: + case UD_Ishr: + case UD_Isar: + cast = 1; + break; + default: break; + } + } + } + gen_operand(u, &u->operand[0], cast); + } + + if (u->operand[1].type != UD_NONE) { + int cast = 0; + ud_asmprintf(u, ", "); + if (u->operand[1].type == UD_OP_MEM && + u->operand[0].size != u->operand[1].size && + !ud_opr_is_sreg(&u->operand[0])) { + cast = 1; + } + gen_operand(u, &u->operand[1], cast); + } + + if (u->operand[2].type != UD_NONE) { + int cast = 0; + ud_asmprintf(u, ", "); + if (u->operand[2].type == UD_OP_MEM && + u->operand[2].size != u->operand[1].size) { + cast = 1; + } + gen_operand(u, &u->operand[2], cast); + } + + if (u->operand[3].type != UD_NONE) { + ud_asmprintf(u, ", "); + gen_operand(u, &u->operand[3], 0); + } +} + +/* +vim: set ts=2 sw=2 expandtab +*/ diff --git a/ext/opcache/jit/libudis86/syn.c b/ext/opcache/jit/libudis86/syn.c new file mode 100644 index 0000000000000..1b9e1d42a5961 --- /dev/null +++ b/ext/opcache/jit/libudis86/syn.c @@ -0,0 +1,212 @@ +/* udis86 - libudis86/syn.c + * + * Copyright (c) 2002-2013 Vivek Thampi + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#include "types.h" +#include "decode.h" +#include "syn.h" +#include "udint.h" + +/* + * Register Table - Order Matters (types.h)! + * + */ +const char* ud_reg_tab[] = +{ + "al", "cl", "dl", "bl", + "ah", "ch", "dh", "bh", + "spl", "bpl", "sil", "dil", + "r8b", "r9b", "r10b", "r11b", + "r12b", "r13b", "r14b", "r15b", + + "ax", "cx", "dx", "bx", + "sp", "bp", "si", "di", + "r8w", "r9w", "r10w", "r11w", + "r12w", "r13w", "r14w", "r15w", + + "eax", "ecx", "edx", "ebx", + "esp", "ebp", "esi", "edi", + "r8d", "r9d", "r10d", "r11d", + "r12d", "r13d", "r14d", "r15d", + + "rax", "rcx", "rdx", "rbx", + "rsp", "rbp", "rsi", "rdi", + "r8", "r9", "r10", "r11", + "r12", "r13", "r14", "r15", + + "es", "cs", "ss", "ds", + "fs", "gs", + + "cr0", "cr1", "cr2", "cr3", + "cr4", "cr5", "cr6", "cr7", + "cr8", "cr9", "cr10", "cr11", + "cr12", "cr13", "cr14", "cr15", + + "dr0", "dr1", "dr2", "dr3", + "dr4", "dr5", "dr6", "dr7", + "dr8", "dr9", "dr10", "dr11", + "dr12", "dr13", "dr14", "dr15", + + "mm0", "mm1", "mm2", "mm3", + "mm4", "mm5", "mm6", "mm7", + + "st0", "st1", "st2", "st3", + "st4", "st5", "st6", "st7", + + "xmm0", "xmm1", "xmm2", "xmm3", + "xmm4", "xmm5", "xmm6", "xmm7", + "xmm8", "xmm9", "xmm10", "xmm11", + "xmm12", "xmm13", "xmm14", "xmm15", + + "ymm0", "ymm1", "ymm2", "ymm3", + "ymm4", "ymm5", "ymm6", "ymm7", + "ymm8", "ymm9", "ymm10", "ymm11", + "ymm12", "ymm13", "ymm14", "ymm15", + + "rip" +}; + + +uint64_t +ud_syn_rel_target(struct ud *u, struct ud_operand *opr) +{ + const uint64_t trunc_mask = 0xffffffffffffffffull >> (64 - u->opr_mode); + switch (opr->size) { + case 8 : return (u->pc + opr->lval.sbyte) & trunc_mask; + case 16: return (u->pc + opr->lval.sword) & trunc_mask; + case 32: return (u->pc + opr->lval.sdword) & trunc_mask; + default: UD_ASSERT(!"invalid relative offset size."); + return 0ull; + } +} + + +/* + * asmprintf + * Printf style function for printing translated assembly + * output. Returns the number of characters written and + * moves the buffer pointer forward. On an overflow, + * returns a negative number and truncates the output. + */ +int +ud_asmprintf(struct ud *u, const char *fmt, ...) +{ + int ret; + int avail; + va_list ap; + va_start(ap, fmt); + avail = u->asm_buf_size - u->asm_buf_fill - 1 /* nullchar */; + ret = vsnprintf((char*) u->asm_buf + u->asm_buf_fill, avail, fmt, ap); + if (ret < 0 || ret > avail) { + u->asm_buf_fill = u->asm_buf_size - 1; + } else { + u->asm_buf_fill += ret; + } + va_end(ap); + return ret; +} + + +void +ud_syn_print_addr(struct ud *u, uint64_t addr) +{ + const char *name = NULL; + if (u->sym_resolver) { + int64_t offset = 0; + name = u->sym_resolver(u, addr, &offset); + if (name) { + if (offset) { + ud_asmprintf(u, "%s%+" FMT64 "d", name, offset); + } else { + ud_asmprintf(u, "%s", name); + } + return; + } + } + ud_asmprintf(u, "0x%" FMT64 "x", addr); +} + + +void +ud_syn_print_imm(struct ud* u, const struct ud_operand *op) +{ + uint64_t v; + if (op->_oprcode == OP_sI && op->size != u->opr_mode) { + if (op->size == 8) { + v = (int64_t)op->lval.sbyte; + } else { + UD_ASSERT(op->size == 32); + v = (int64_t)op->lval.sdword; + } + if (u->opr_mode < 64) { + v = v & ((1ull << u->opr_mode) - 1ull); + } + } else { + switch (op->size) { + case 8 : v = op->lval.ubyte; break; + case 16: v = op->lval.uword; break; + case 32: v = op->lval.udword; break; + case 64: v = op->lval.uqword; break; + default: UD_ASSERT(!"invalid offset"); v = 0; /* keep cc happy */ + } + } + ud_asmprintf(u, "0x%" FMT64 "x", v); +} + + +void +ud_syn_print_mem_disp(struct ud* u, const struct ud_operand *op, int sign) +{ + UD_ASSERT(op->offset != 0); + if (op->base == UD_NONE && op->index == UD_NONE) { + uint64_t v; + UD_ASSERT(op->scale == UD_NONE && op->offset != 8); + /* unsigned mem-offset */ + switch (op->offset) { + case 16: v = op->lval.uword; break; + case 32: v = op->lval.udword; break; + case 64: v = op->lval.uqword; break; + default: UD_ASSERT(!"invalid offset"); v = 0; /* keep cc happy */ + } + ud_asmprintf(u, "0x%" FMT64 "x", v); + } else { + int64_t v; + UD_ASSERT(op->offset != 64); + switch (op->offset) { + case 8 : v = op->lval.sbyte; break; + case 16: v = op->lval.sword; break; + case 32: v = op->lval.sdword; break; + default: UD_ASSERT(!"invalid offset"); v = 0; /* keep cc happy */ + } + if (v < 0) { + ud_asmprintf(u, "-0x%" FMT64 "x", -v); + } else if (v > 0) { + ud_asmprintf(u, "%s0x%" FMT64 "x", sign? "+" : "", v); + } + } +} + +/* +vim: set ts=2 sw=2 expandtab +*/ diff --git a/ext/opcache/jit/libudis86/syn.h b/ext/opcache/jit/libudis86/syn.h new file mode 100644 index 0000000000000..d3b1e3fe04441 --- /dev/null +++ b/ext/opcache/jit/libudis86/syn.h @@ -0,0 +1,53 @@ +/* udis86 - libudis86/syn.h + * + * Copyright (c) 2002-2009 + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef UD_SYN_H +#define UD_SYN_H + +#include "types.h" +#ifndef __UD_STANDALONE__ +# include +#endif /* __UD_STANDALONE__ */ + +extern const char* ud_reg_tab[]; + +uint64_t ud_syn_rel_target(struct ud*, struct ud_operand*); + +#ifdef __GNUC__ +int ud_asmprintf(struct ud *u, const char *fmt, ...) + __attribute__ ((format (printf, 2, 3))); +#else +int ud_asmprintf(struct ud *u, const char *fmt, ...); +#endif + +void ud_syn_print_addr(struct ud *u, uint64_t addr); +void ud_syn_print_imm(struct ud* u, const struct ud_operand *op); +void ud_syn_print_mem_disp(struct ud* u, const struct ud_operand *, int sign); + +#endif /* UD_SYN_H */ + +/* +vim: set ts=2 sw=2 expandtab +*/ diff --git a/ext/opcache/jit/libudis86/types.h b/ext/opcache/jit/libudis86/types.h new file mode 100644 index 0000000000000..69072ca480517 --- /dev/null +++ b/ext/opcache/jit/libudis86/types.h @@ -0,0 +1,260 @@ +/* udis86 - libudis86/types.h + * + * Copyright (c) 2002-2013 Vivek Thampi + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef UD_TYPES_H +#define UD_TYPES_H + +#ifdef __KERNEL__ + /* + * -D__KERNEL__ is automatically passed on the command line when + * building something as part of the Linux kernel. Assume standalone + * mode. + */ +# include +# include +# ifndef __UD_STANDALONE__ +# define __UD_STANDALONE__ 1 +# endif +#endif /* __KERNEL__ */ + +#if !defined(__UD_STANDALONE__) +# include +# include +#endif + +/* gcc specific extensions */ +#ifdef __GNUC__ +# define UD_ATTR_PACKED __attribute__((packed)) +#else +# define UD_ATTR_PACKED +#endif /* UD_ATTR_PACKED */ + + +/* ----------------------------------------------------------------------------- + * All possible "types" of objects in udis86. Order is Important! + * ----------------------------------------------------------------------------- + */ +enum ud_type +{ + UD_NONE, + + /* 8 bit GPRs */ + UD_R_AL, UD_R_CL, UD_R_DL, UD_R_BL, + UD_R_AH, UD_R_CH, UD_R_DH, UD_R_BH, + UD_R_SPL, UD_R_BPL, UD_R_SIL, UD_R_DIL, + UD_R_R8B, UD_R_R9B, UD_R_R10B, UD_R_R11B, + UD_R_R12B, UD_R_R13B, UD_R_R14B, UD_R_R15B, + + /* 16 bit GPRs */ + UD_R_AX, UD_R_CX, UD_R_DX, UD_R_BX, + UD_R_SP, UD_R_BP, UD_R_SI, UD_R_DI, + UD_R_R8W, UD_R_R9W, UD_R_R10W, UD_R_R11W, + UD_R_R12W, UD_R_R13W, UD_R_R14W, UD_R_R15W, + + /* 32 bit GPRs */ + UD_R_EAX, UD_R_ECX, UD_R_EDX, UD_R_EBX, + UD_R_ESP, UD_R_EBP, UD_R_ESI, UD_R_EDI, + UD_R_R8D, UD_R_R9D, UD_R_R10D, UD_R_R11D, + UD_R_R12D, UD_R_R13D, UD_R_R14D, UD_R_R15D, + + /* 64 bit GPRs */ + UD_R_RAX, UD_R_RCX, UD_R_RDX, UD_R_RBX, + UD_R_RSP, UD_R_RBP, UD_R_RSI, UD_R_RDI, + UD_R_R8, UD_R_R9, UD_R_R10, UD_R_R11, + UD_R_R12, UD_R_R13, UD_R_R14, UD_R_R15, + + /* segment registers */ + UD_R_ES, UD_R_CS, UD_R_SS, UD_R_DS, + UD_R_FS, UD_R_GS, + + /* control registers*/ + UD_R_CR0, UD_R_CR1, UD_R_CR2, UD_R_CR3, + UD_R_CR4, UD_R_CR5, UD_R_CR6, UD_R_CR7, + UD_R_CR8, UD_R_CR9, UD_R_CR10, UD_R_CR11, + UD_R_CR12, UD_R_CR13, UD_R_CR14, UD_R_CR15, + + /* debug registers */ + UD_R_DR0, UD_R_DR1, UD_R_DR2, UD_R_DR3, + UD_R_DR4, UD_R_DR5, UD_R_DR6, UD_R_DR7, + UD_R_DR8, UD_R_DR9, UD_R_DR10, UD_R_DR11, + UD_R_DR12, UD_R_DR13, UD_R_DR14, UD_R_DR15, + + /* mmx registers */ + UD_R_MM0, UD_R_MM1, UD_R_MM2, UD_R_MM3, + UD_R_MM4, UD_R_MM5, UD_R_MM6, UD_R_MM7, + + /* x87 registers */ + UD_R_ST0, UD_R_ST1, UD_R_ST2, UD_R_ST3, + UD_R_ST4, UD_R_ST5, UD_R_ST6, UD_R_ST7, + + /* extended multimedia registers */ + UD_R_XMM0, UD_R_XMM1, UD_R_XMM2, UD_R_XMM3, + UD_R_XMM4, UD_R_XMM5, UD_R_XMM6, UD_R_XMM7, + UD_R_XMM8, UD_R_XMM9, UD_R_XMM10, UD_R_XMM11, + UD_R_XMM12, UD_R_XMM13, UD_R_XMM14, UD_R_XMM15, + + /* 256B multimedia registers */ + UD_R_YMM0, UD_R_YMM1, UD_R_YMM2, UD_R_YMM3, + UD_R_YMM4, UD_R_YMM5, UD_R_YMM6, UD_R_YMM7, + UD_R_YMM8, UD_R_YMM9, UD_R_YMM10, UD_R_YMM11, + UD_R_YMM12, UD_R_YMM13, UD_R_YMM14, UD_R_YMM15, + + UD_R_RIP, + + /* Operand Types */ + UD_OP_REG, UD_OP_MEM, UD_OP_PTR, UD_OP_IMM, + UD_OP_JIMM, UD_OP_CONST +}; + +#include "itab.h" + +union ud_lval { + int8_t sbyte; + uint8_t ubyte; + int16_t sword; + uint16_t uword; + int32_t sdword; + uint32_t udword; + int64_t sqword; + uint64_t uqword; + struct { + uint16_t seg; + uint32_t off; + } ptr; +}; + +/* ----------------------------------------------------------------------------- + * struct ud_operand - Disassembled instruction Operand. + * ----------------------------------------------------------------------------- + */ +struct ud_operand { + enum ud_type type; + uint16_t size; + enum ud_type base; + enum ud_type index; + uint8_t scale; + uint8_t offset; + union ud_lval lval; + /* + * internal use only + */ + uint64_t _legacy; /* this will be removed in 1.8 */ + uint8_t _oprcode; +}; + +/* ----------------------------------------------------------------------------- + * struct ud - The udis86 object. + * ----------------------------------------------------------------------------- + */ +struct ud +{ + /* + * input buffering + */ + int (*inp_hook) (struct ud*); +#ifndef __UD_STANDALONE__ + FILE* inp_file; +#endif + const uint8_t* inp_buf; + size_t inp_buf_size; + size_t inp_buf_index; + uint8_t inp_curr; + size_t inp_ctr; + uint8_t inp_sess[64]; + int inp_end; + int inp_peek; + + void (*translator)(struct ud*); + uint64_t insn_offset; + char insn_hexcode[64]; + + /* + * Assembly output buffer + */ + char *asm_buf; + size_t asm_buf_size; + size_t asm_buf_fill; + char asm_buf_int[128]; + + /* + * Symbol resolver for use in the translation phase. + */ + const char* (*sym_resolver)(struct ud*, uint64_t addr, int64_t *offset); + + uint8_t dis_mode; + uint64_t pc; + uint8_t vendor; + enum ud_mnemonic_code mnemonic; + struct ud_operand operand[4]; + uint8_t error; + uint8_t _rex; + uint8_t pfx_rex; + uint8_t pfx_seg; + uint8_t pfx_opr; + uint8_t pfx_adr; + uint8_t pfx_lock; + uint8_t pfx_str; + uint8_t pfx_rep; + uint8_t pfx_repe; + uint8_t pfx_repne; + uint8_t opr_mode; + uint8_t adr_mode; + uint8_t br_far; + uint8_t br_near; + uint8_t have_modrm; + uint8_t modrm; + uint8_t modrm_offset; + uint8_t vex_op; + uint8_t vex_b1; + uint8_t vex_b2; + uint8_t primary_opcode; + void * user_opaque_data; + struct ud_itab_entry * itab_entry; + struct ud_lookup_table_list_entry *le; +}; + +/* ----------------------------------------------------------------------------- + * Type-definitions + * ----------------------------------------------------------------------------- + */ +typedef enum ud_type ud_type_t; +typedef enum ud_mnemonic_code ud_mnemonic_code_t; + +typedef struct ud ud_t; +typedef struct ud_operand ud_operand_t; + +#define UD_SYN_INTEL ud_translate_intel +#define UD_SYN_ATT ud_translate_att +#define UD_EOI (-1) +#define UD_INP_CACHE_SZ 32 +#define UD_VENDOR_AMD 0 +#define UD_VENDOR_INTEL 1 +#define UD_VENDOR_ANY 2 + +#endif + +/* +vim: set ts=2 sw=2 expandtab +*/ diff --git a/ext/opcache/jit/libudis86/udint.h b/ext/opcache/jit/libudis86/udint.h new file mode 100644 index 0000000000000..734f0eaa82db1 --- /dev/null +++ b/ext/opcache/jit/libudis86/udint.h @@ -0,0 +1,99 @@ +/* udis86 - libudis86/udint.h -- definitions for internal use only + * + * Copyright (c) 2002-2009 Vivek Thampi + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef _UDINT_H_ +#define _UDINT_H_ + +#include "types.h" + +#ifdef HAVE_CONFIG_H +# include +#endif /* HAVE_CONFIG_H */ + +#if defined(UD_DEBUG) && HAVE_ASSERT_H +# include +# define UD_ASSERT(_x) assert(_x) +#else +# define UD_ASSERT(_x) +#endif /* !HAVE_ASSERT_H */ + +#if defined(UD_DEBUG) + #define UDERR(u, msg) \ + do { \ + (u)->error = 1; \ + fprintf(stderr, "decode-error: %s:%d: %s", \ + __FILE__, __LINE__, (msg)); \ + } while (0) +#else + #define UDERR(u, m) \ + do { \ + (u)->error = 1; \ + } while (0) +#endif /* !LOGERR */ + +#define UD_RETURN_ON_ERROR(u) \ + do { \ + if ((u)->error != 0) { \ + return (u)->error; \ + } \ + } while (0) + +#define UD_RETURN_WITH_ERROR(u, m) \ + do { \ + UDERR(u, m); \ + return (u)->error; \ + } while (0) + +#ifndef __UD_STANDALONE__ +# define UD_NON_STANDALONE(x) x +#else +# define UD_NON_STANDALONE(x) +#endif + +/* printf formatting int64 specifier */ +#ifdef FMT64 +# undef FMT64 +#endif +#if defined(_MSC_VER) || defined(__BORLANDC__) +# define FMT64 "I64" +#else +# if defined(__APPLE__) +# define FMT64 "ll" +# elif defined(__amd64__) || defined(__x86_64__) +# define FMT64 "l" +# else +# define FMT64 "ll" +# endif /* !x64 */ +#endif + +/* define an inline macro */ +#if defined(_MSC_VER) || defined(__BORLANDC__) +# define UD_INLINE __inline /* MS Visual Studio requires __inline + instead of inline for C code */ +#else +# define UD_INLINE inline +#endif + +#endif /* _UDINT_H_ */ diff --git a/ext/opcache/jit/libudis86/udis86.c b/ext/opcache/jit/libudis86/udis86.c new file mode 100644 index 0000000000000..e039c4e838cdf --- /dev/null +++ b/ext/opcache/jit/libudis86/udis86.c @@ -0,0 +1,458 @@ +/* udis86 - libudis86/udis86.c + * + * Copyright (c) 2002-2013 Vivek Thampi + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "udint.h" +#include "extern.h" +#include "decode.h" + +#if !defined(__UD_STANDALONE__) +# if HAVE_STRING_H +# include +# endif +#endif /* !__UD_STANDALONE__ */ + +static void ud_inp_init(struct ud *u); + +/* ============================================================================= + * ud_init + * Initializes ud_t object. + * ============================================================================= + */ +extern void +ud_init(struct ud* u) +{ + memset((void*)u, 0, sizeof(struct ud)); + ud_set_mode(u, 16); + u->mnemonic = UD_Iinvalid; + ud_set_pc(u, 0); +#ifndef __UD_STANDALONE__ + ud_set_input_file(u, stdin); +#endif /* __UD_STANDALONE__ */ + + ud_set_asm_buffer(u, u->asm_buf_int, sizeof(u->asm_buf_int)); +} + + +/* ============================================================================= + * ud_disassemble + * Disassembles one instruction and returns the number of + * bytes disassembled. A zero means end of disassembly. + * ============================================================================= + */ +extern unsigned int +ud_disassemble(struct ud* u) +{ + int len; + if (u->inp_end) { + return 0; + } + if ((len = ud_decode(u)) > 0) { + if (u->translator != NULL) { + u->asm_buf[0] = '\0'; + u->translator(u); + } + } + return len; +} + + +/* ============================================================================= + * ud_set_mode() - Set Disassemly Mode. + * ============================================================================= + */ +extern void +ud_set_mode(struct ud* u, uint8_t m) +{ + switch(m) { + case 16: + case 32: + case 64: u->dis_mode = m ; return; + default: u->dis_mode = 16; return; + } +} + +/* ============================================================================= + * ud_set_vendor() - Set vendor. + * ============================================================================= + */ +extern void +ud_set_vendor(struct ud* u, unsigned v) +{ + switch(v) { + case UD_VENDOR_INTEL: + u->vendor = v; + break; + case UD_VENDOR_ANY: + u->vendor = v; + break; + default: + u->vendor = UD_VENDOR_AMD; + } +} + +/* ============================================================================= + * ud_set_pc() - Sets code origin. + * ============================================================================= + */ +extern void +ud_set_pc(struct ud* u, uint64_t o) +{ + u->pc = o; +} + +/* ============================================================================= + * ud_set_syntax() - Sets the output syntax. + * ============================================================================= + */ +extern void +ud_set_syntax(struct ud* u, void (*t)(struct ud*)) +{ + u->translator = t; +} + +/* ============================================================================= + * ud_insn() - returns the disassembled instruction + * ============================================================================= + */ +const char* +ud_insn_asm(const struct ud* u) +{ + return u->asm_buf; +} + +/* ============================================================================= + * ud_insn_offset() - Returns the offset. + * ============================================================================= + */ +uint64_t +ud_insn_off(const struct ud* u) +{ + return u->insn_offset; +} + + +/* ============================================================================= + * ud_insn_hex() - Returns hex form of disassembled instruction. + * ============================================================================= + */ +const char* +ud_insn_hex(struct ud* u) +{ + u->insn_hexcode[0] = 0; + if (!u->error) { + unsigned int i; + const unsigned char *src_ptr = ud_insn_ptr(u); + char* src_hex; + src_hex = (char*) u->insn_hexcode; + /* for each byte used to decode instruction */ + for (i = 0; i < ud_insn_len(u) && i < sizeof(u->insn_hexcode) / 2; + ++i, ++src_ptr) { + sprintf(src_hex, "%02x", *src_ptr & 0xFF); + src_hex += 2; + } + } + return u->insn_hexcode; +} + + +/* ============================================================================= + * ud_insn_ptr + * Returns a pointer to buffer containing the bytes that were + * disassembled. + * ============================================================================= + */ +extern const uint8_t* +ud_insn_ptr(const struct ud* u) +{ + return (u->inp_buf == NULL) ? + u->inp_sess : u->inp_buf + (u->inp_buf_index - u->inp_ctr); +} + + +/* ============================================================================= + * ud_insn_len + * Returns the count of bytes disassembled. + * ============================================================================= + */ +extern unsigned int +ud_insn_len(const struct ud* u) +{ + return u->inp_ctr; +} + + +/* ============================================================================= + * ud_insn_get_opr + * Return the operand struct representing the nth operand of + * the currently disassembled instruction. Returns NULL if + * there's no such operand. + * ============================================================================= + */ +const struct ud_operand* +ud_insn_opr(const struct ud *u, unsigned int n) +{ + if (n > 3 || u->operand[n].type == UD_NONE) { + return NULL; + } else { + return &u->operand[n]; + } +} + + +/* ============================================================================= + * ud_opr_is_sreg + * Returns non-zero if the given operand is of a segment register type. + * ============================================================================= + */ +int +ud_opr_is_sreg(const struct ud_operand *opr) +{ + return opr->type == UD_OP_REG && + opr->base >= UD_R_ES && + opr->base <= UD_R_GS; +} + + +/* ============================================================================= + * ud_opr_is_sreg + * Returns non-zero if the given operand is of a general purpose + * register type. + * ============================================================================= + */ +int +ud_opr_is_gpr(const struct ud_operand *opr) +{ + return opr->type == UD_OP_REG && + opr->base >= UD_R_AL && + opr->base <= UD_R_R15; +} + + +/* ============================================================================= + * ud_set_user_opaque_data + * ud_get_user_opaque_data + * Get/set user opaqute data pointer + * ============================================================================= + */ +void +ud_set_user_opaque_data(struct ud * u, void* opaque) +{ + u->user_opaque_data = opaque; +} + +void* +ud_get_user_opaque_data(const struct ud *u) +{ + return u->user_opaque_data; +} + + +/* ============================================================================= + * ud_set_asm_buffer + * Allow the user to set an assembler output buffer. If `buf` is NULL, + * we switch back to the internal buffer. + * ============================================================================= + */ +void +ud_set_asm_buffer(struct ud *u, char *buf, size_t size) +{ + if (buf == NULL) { + ud_set_asm_buffer(u, u->asm_buf_int, sizeof(u->asm_buf_int)); + } else { + u->asm_buf = buf; + u->asm_buf_size = size; + } +} + + +/* ============================================================================= + * ud_set_sym_resolver + * Set symbol resolver for relative targets used in the translation + * phase. + * + * The resolver is a function that takes a uint64_t address and returns a + * symbolic name for the that address. The function also takes a second + * argument pointing to an integer that the client can optionally set to a + * non-zero value for offsetted targets. (symbol+offset) The function may + * also return NULL, in which case the translator only prints the target + * address. + * + * The function pointer maybe NULL which resets symbol resolution. + * ============================================================================= + */ +void +ud_set_sym_resolver(struct ud *u, const char* (*resolver)(struct ud*, + uint64_t addr, + int64_t *offset)) +{ + u->sym_resolver = resolver; +} + + +/* ============================================================================= + * ud_insn_mnemonic + * Return the current instruction mnemonic. + * ============================================================================= + */ +enum ud_mnemonic_code +ud_insn_mnemonic(const struct ud *u) +{ + return u->mnemonic; +} + + +/* ============================================================================= + * ud_lookup_mnemonic + * Looks up mnemonic code in the mnemonic string table. + * Returns NULL if the mnemonic code is invalid. + * ============================================================================= + */ +const char* +ud_lookup_mnemonic(enum ud_mnemonic_code c) +{ + if (c < UD_MAX_MNEMONIC_CODE) { + return ud_mnemonics_str[c]; + } else { + return NULL; + } +} + + +/* + * ud_inp_init + * Initializes the input system. + */ +static void +ud_inp_init(struct ud *u) +{ + u->inp_hook = NULL; + u->inp_buf = NULL; + u->inp_buf_size = 0; + u->inp_buf_index = 0; + u->inp_curr = 0; + u->inp_ctr = 0; + u->inp_end = 0; + u->inp_peek = UD_EOI; + UD_NON_STANDALONE(u->inp_file = NULL); +} + + +/* ============================================================================= + * ud_inp_set_hook + * Sets input hook. + * ============================================================================= + */ +void +ud_set_input_hook(register struct ud* u, int (*hook)(struct ud*)) +{ + ud_inp_init(u); + u->inp_hook = hook; +} + +/* ============================================================================= + * ud_inp_set_buffer + * Set buffer as input. + * ============================================================================= + */ +void +ud_set_input_buffer(register struct ud* u, const uint8_t* buf, size_t len) +{ + ud_inp_init(u); + u->inp_buf = buf; + u->inp_buf_size = len; + u->inp_buf_index = 0; +} + + +#ifndef __UD_STANDALONE__ +/* ============================================================================= + * ud_input_set_file + * Set FILE as input. + * ============================================================================= + */ +static int +inp_file_hook(struct ud* u) +{ + return fgetc(u->inp_file); +} + +void +ud_set_input_file(register struct ud* u, FILE* f) +{ + ud_inp_init(u); + u->inp_hook = inp_file_hook; + u->inp_file = f; +} +#endif /* __UD_STANDALONE__ */ + + +/* ============================================================================= + * ud_input_skip + * Skip n input bytes. + * ============================================================================ + */ +void +ud_input_skip(struct ud* u, size_t n) +{ + if (u->inp_end) { + return; + } + if (u->inp_buf == NULL) { + while (n--) { + int c = u->inp_hook(u); + if (c == UD_EOI) { + goto eoi; + } + } + return; + } else { + if (n > u->inp_buf_size || + u->inp_buf_index > u->inp_buf_size - n) { + u->inp_buf_index = u->inp_buf_size; + goto eoi; + } + u->inp_buf_index += n; + return; + } +eoi: + u->inp_end = 1; + UDERR(u, "cannot skip, eoi received\b"); + return; +} + + +/* ============================================================================= + * ud_input_end + * Returns non-zero on end-of-input. + * ============================================================================= + */ +int +ud_input_end(const struct ud *u) +{ + return u->inp_end; +} + +/* vim:set ts=2 sw=2 expandtab */ diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 06876c857d7a2..a7d7a2cb4dc5e 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -1,6 +1,6 @@ /* +----------------------------------------------------------------------+ - | Zend OPcache | + | Zend JIT | +----------------------------------------------------------------------+ | Copyright (c) 1998-2016 The PHP Group | +----------------------------------------------------------------------+ @@ -29,6 +29,7 @@ #include "dynasm/dasm_proto.h" #include "dynasm/dasm_x86.h" #include "jit/zend_jit_x86.c" +#include "jit/zend_jit_disasm_x86.c" #if _WIN32 # include @@ -130,7 +131,7 @@ static void jit_free(void *p, size_t size) #endif } -ZEND_API int zend_jit(zend_op_array *op_array, zend_persistent_script* main_persistent_script) +ZEND_API int zend_jit(zend_op_array *op_array, zend_script *script) { uint32_t flags; zend_ssa ssa; @@ -333,6 +334,13 @@ ZEND_API int zend_jit(zend_op_array *op_array, zend_persistent_script* main_pers } } dasm_free(&dasm_state); + +#ifdef HAVE_DISASM + if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_ASM) { + zend_jit_disasm(op_array, handler, dasm_ptr); + } +#endif + zend_arena_release(&CG(arena), checkpoint); return SUCCESS; @@ -382,6 +390,15 @@ ZEND_API int zend_jit_startup(size_t size) dasm_buf = dasm_ptr = buf; dasm_end = (void*)(((char*)dasm_buf)+size); +#ifdef HAVE_DISASM + if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_ASM) { + if (!zend_jit_disasm_init()) { + // TODO: error reporting and cleanup ??? + return FAILURE; + } + } +#endif + zend_jit_unprotect(); dasm_init(&dasm_state, DASM_MAXSECTION); @@ -392,7 +409,16 @@ ZEND_API int zend_jit_startup(size_t size) zend_jit_protect(); +#ifdef HAVE_DISASM + if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_ASM) { + if (dasm_buf != dasm_ptr) { + zend_jit_disasm(NULL, dasm_buf, dasm_ptr); + } + } +#endif + if (!ret) { + // TODO: error reporting and cleanup ??? return FAILURE; } @@ -408,7 +434,7 @@ ZEND_API void zend_jit_shutdown(void) #else /* HAVE_JIT */ -ZEND_API int zend_jit(zend_op_array *op_array, zend_persistent_script* main_persistent_script) +ZEND_API int zend_jit(zend_op_array *op_array, zend_script *script) { return FAILURE; } diff --git a/ext/opcache/jit/zend_jit.h b/ext/opcache/jit/zend_jit.h index 9e93f456dc695..b8183c4e2430c 100644 --- a/ext/opcache/jit/zend_jit.h +++ b/ext/opcache/jit/zend_jit.h @@ -1,6 +1,6 @@ /* +----------------------------------------------------------------------+ - | Zend OPcache | + | Zend JIT | +----------------------------------------------------------------------+ | Copyright (c) 1998-2016 The PHP Group | +----------------------------------------------------------------------+ @@ -21,7 +21,9 @@ #ifndef HAVE_JIT_H #define HAVE_JIT_H -ZEND_API int zend_jit(zend_op_array *op_array, zend_persistent_script* main_persistent_script); +#define ZEND_JIT_DEBUG_ASM (1<<0) + +ZEND_API int zend_jit(zend_op_array *op_array, zend_script *script); ZEND_API void zend_jit_unprotect(void); ZEND_API void zend_jit_protect(void); ZEND_API int zend_jit_startup(size_t size); diff --git a/ext/opcache/jit/zend_jit_disasm_x86.c b/ext/opcache/jit/zend_jit_disasm_x86.c new file mode 100644 index 0000000000000..3f3539ab5380b --- /dev/null +++ b/ext/opcache/jit/zend_jit_disasm_x86.c @@ -0,0 +1,105 @@ +/* + +----------------------------------------------------------------------+ + | Zend JIT | + +----------------------------------------------------------------------+ + | Copyright (c) 1998-2016 The PHP Group | + +----------------------------------------------------------------------+ + | This source file is subject to version 3.01 of the PHP license, | + | that is bundled with this package in the file LICENSE, and is | + | available through the world-wide-web at the following url: | + | http://www.php.net/license/3_01.txt | + | If you did not receive a copy of the PHP license and are unable to | + | obtain it through the world-wide-web, please send a note to | + | license@php.net so we can mail you a copy immediately. | + +----------------------------------------------------------------------+ + | Authors: Dmitry Stogov | + +----------------------------------------------------------------------+ +*/ + +#define HAVE_DISASM 1 +#define DISASM_INTEL_SYNTAX 0 + +#include "jit/libudis86/itab.c" +#include "jit/libudis86/decode.c" +#include "jit/libudis86/syn.c" +#if DISASM_INTEL_SYNTAX +# include "jit/libudis86/syn-intel.c" +#else +# include "jit/libudis86/syn-att.c" +#endif +#include "jit/libudis86/udis86.c" + +#ifndef _GNU_SOURCE +# define _GNU_SOURCE +#endif + +#include + +static struct ud ud; + +static const char* zend_jit_disasm_resolver(struct ud *ud, + uint64_t addr, + int64_t *offset) +{ + void *a = (void*)(zend_uintptr_t)(addr); + Dl_info info; + + if (dladdr(a, &info) + && info.dli_sname != NULL + && info.dli_saddr == a) { + return info.dli_sname; + } + + return NULL; +} + +static int zend_jit_disasm(zend_op_array *op_array, + const void *start, + const void *end) +{ + if (op_array) { + if (op_array->function_name) { + if (op_array->scope) { + fprintf(stderr, "$_%s::%s: ; (%s)\n", ZSTR_VAL(op_array->scope->name), ZSTR_VAL(op_array->function_name), op_array->filename ? ZSTR_VAL(op_array->filename) : "unknown"); + } else { + fprintf(stderr, "$_%s: ; (%s)\n", ZSTR_VAL(op_array->function_name), ZSTR_VAL(op_array->filename), op_array->filename ? ZSTR_VAL(op_array->filename) : "unknown"); + } + } else if (op_array->filename) { + fprintf(stderr, "$_: ; (%s)\n", ZSTR_VAL(op_array->filename)); + } + } + + ud_set_input_buffer(&ud, (uint8_t*)start, (uint8_t*)end - (uint8_t*)start); + ud_set_pc(&ud, (uint64_t)(uintptr_t)start); + + while (ud_disassemble(&ud)) { + fprintf(stderr, "0x%" PRIx64 ":\t%s\n", ud_insn_off(&ud), ud_insn_asm(&ud)); + } + fprintf(stderr, "\n"); +} + +static int zend_jit_disasm_init(void) +{ + ud_init(&ud); +#ifdef __x86_64__ + ud_set_mode(&ud, 64); +#else + ud_set_mode(&ud, 32); +#endif +#if DISASM_INTEL_SYNTAX + ud_set_syntax(&ud, UD_SYN_INTEL); +#else + ud_set_syntax(&ud, UD_SYN_ATT); +#endif + ud_set_sym_resolver(&ud, zend_jit_disasm_resolver); + + return 1; +} + +/* + * Local variables: + * tab-width: 4 + * c-basic-offset: 4 + * indent-tabs-mode: t + * End: + */ diff --git a/ext/opcache/jit/zend_jit_gdb.c b/ext/opcache/jit/zend_jit_gdb.c index db66b3cd193b4..654159b10eb9f 100644 --- a/ext/opcache/jit/zend_jit_gdb.c +++ b/ext/opcache/jit/zend_jit_gdb.c @@ -1,6 +1,6 @@ /* +----------------------------------------------------------------------+ - | Zend OPcache | + | Zend JIT | +----------------------------------------------------------------------+ | Copyright (c) 1998-2016 The PHP Group | +----------------------------------------------------------------------+ diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index f38a2f756bdcb..507dcf3074800 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -1,6 +1,6 @@ /* * +----------------------------------------------------------------------+ - * | Zend OPcache | + * | Zend JIT | * +----------------------------------------------------------------------+ * | Copyright (c) 1998-2016 The PHP Group | * +----------------------------------------------------------------------+ diff --git a/ext/opcache/zend_persist.c b/ext/opcache/zend_persist.c index ce9d9ceebcdc8..f47cabaf1336f 100644 --- a/ext/opcache/zend_persist.c +++ b/ext/opcache/zend_persist.c @@ -571,7 +571,7 @@ static void zend_persist_op_array_ex(zend_op_array *op_array, zend_persistent_sc #ifdef HAVE_JIT if (do_jit && ZCG(accel_directives).jit_buffer_size) { - zend_jit(op_array, main_persistent_script TSRMLS_CC); + zend_jit(op_array, &main_persistent_script->script); } #endif } From 2bb79d29ab25c3a6b07fa02549ba9d8763494651 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 19 Aug 2016 11:57:01 +0300 Subject: [PATCH 009/569] Remove useless file and fix white spaces --- ext/opcache/jit/libudis86/Makefile.am | 51 --------------------------- ext/opcache/jit/zend_jit.c | 24 ++++++------- ext/opcache/jit/zend_jit_disasm_x86.c | 6 ++-- 3 files changed, 15 insertions(+), 66 deletions(-) delete mode 100644 ext/opcache/jit/libudis86/Makefile.am diff --git a/ext/opcache/jit/libudis86/Makefile.am b/ext/opcache/jit/libudis86/Makefile.am deleted file mode 100644 index c61cf9d04ff0d..0000000000000 --- a/ext/opcache/jit/libudis86/Makefile.am +++ /dev/null @@ -1,51 +0,0 @@ -# -# -- udis86/libudis86 -# - -PYTHON = @PYTHON@ -OPTABLE = @top_srcdir@/docs/x86/optable.xml - -MAINTAINERCLEANFILES = Makefile.in - -lib_LTLIBRARIES = libudis86.la - -libudis86_la_SOURCES = \ - itab.c \ - decode.c \ - syn.c \ - syn-intel.c \ - syn-att.c \ - udis86.c \ - udint.h \ - syn.h \ - decode.h - -include_ladir = ${includedir}/libudis86 -include_la_HEADERS = \ - types.h \ - extern.h \ - itab.h - - -BUILT_SOURCES = \ - itab.c \ - itab.h - -# -# DLLs may not contain undefined symbol references. -# We have the linker check this explicitly. -# -if TARGET_WINDOWS -libudis86_la_LDFLAGS = -no-undefined -version-info 0:0:0 -endif - -itab.c itab.h: $(OPTABLE) \ - $(top_srcdir)/scripts/ud_itab.py \ - $(top_srcdir)/scripts/ud_opcode.py - $(PYTHON) $(top_srcdir)/scripts/ud_itab.py $(OPTABLE) $(srcdir) - - -clean-local: - rm -rf $(BUILT_SOURCES) - -maintainer-clean-local: diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index a7d7a2cb4dc5e..ad0deab81d028 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -56,7 +56,7 @@ static void *dasm_link_and_encode(dasm_State **dasm_state) // TODO: dasm_link() failed ??? return NULL; } - + size = ZEND_MM_ALIGNED_SIZE_EX(size, DASM_ALIGNMENT); if ((void*)((char*)dasm_ptr + size) > dasm_end) { @@ -70,10 +70,10 @@ static void *dasm_link_and_encode(dasm_State **dasm_state) // TODO: dasm_encode() failed ??? return NULL; } - + entry = dasm_ptr; dasm_ptr = (void*)((char*)dasm_ptr + size); - + return entry; } @@ -88,7 +88,7 @@ static size_t jit_page_size(void) #endif } -static void *jit_alloc(size_t size, int shared) +static void *jit_alloc(size_t size, int shared) { #ifdef _WIN32 return VirtualAlloc(0, size, MEM_COMMIT | MEM_RESERVE, PAGE_EXECUTE_READWRITE); @@ -104,7 +104,7 @@ static void *jit_alloc(size_t size, int shared) # endif (shared ? MAP_SHARED : MAP_PRIVATE) | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0); if (p != MAP_FAILED) { - return (void*)p; + return (void*)p; } # endif p = mmap(NULL, size, @@ -115,7 +115,7 @@ static void *jit_alloc(size_t size, int shared) # endif (shared ? MAP_SHARED : MAP_PRIVATE) | MAP_ANONYMOUS, -1, 0); if (p == MAP_FAILED) { - return NULL; + return NULL; } return (void*)p; @@ -125,14 +125,14 @@ static void *jit_alloc(size_t size, int shared) static void jit_free(void *p, size_t size) { #ifdef _WIN32 - VirtualFree(p, 0, MEM_RELEASE); + VirtualFree(p, 0, MEM_RELEASE); #else munmap(p, size); #endif } - + ZEND_API int zend_jit(zend_op_array *op_array, zend_script *script) -{ +{ uint32_t flags; zend_ssa ssa; void *checkpoint; @@ -273,8 +273,8 @@ ZEND_API int zend_jit(zend_op_array *op_array, zend_script *script) /* smart branch */ if (!zend_jit_cond_jmp(&dasm_state, opline + 1, ssa.cfg.blocks[b].successors[0])) { goto jit_failure; - } - break; + } + break; } } /* break missing intentionally */ @@ -333,7 +333,7 @@ ZEND_API int zend_jit(zend_op_array *op_array, zend_script *script) dasm_getpclabel(&dasm_state, ssa.cfg.blocks_count + b)); } } - dasm_free(&dasm_state); + dasm_free(&dasm_state); #ifdef HAVE_DISASM if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_ASM) { diff --git a/ext/opcache/jit/zend_jit_disasm_x86.c b/ext/opcache/jit/zend_jit_disasm_x86.c index 3f3539ab5380b..934a7b99e21f3 100644 --- a/ext/opcache/jit/zend_jit_disasm_x86.c +++ b/ext/opcache/jit/zend_jit_disasm_x86.c @@ -37,12 +37,12 @@ static struct ud ud; -static const char* zend_jit_disasm_resolver(struct ud *ud, +static const char* zend_jit_disasm_resolver(struct ud *ud, uint64_t addr, int64_t *offset) { void *a = (void*)(zend_uintptr_t)(addr); - Dl_info info; + Dl_info info; if (dladdr(a, &info) && info.dli_sname != NULL @@ -68,7 +68,7 @@ static int zend_jit_disasm(zend_op_array *op_array, fprintf(stderr, "$_: ; (%s)\n", ZSTR_VAL(op_array->filename)); } } - + ud_set_input_buffer(&ud, (uint8_t*)start, (uint8_t*)end - (uint8_t*)start); ud_set_pc(&ud, (uint64_t)(uintptr_t)start); From 50f42aa0b5b2229c8a1fcb4e6f1530df48dd82ac Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 19 Aug 2016 14:55:38 +0300 Subject: [PATCH 010/569] Added support for oprofile and Linux perf --- ext/opcache/config.m4 | 41 ++++++++++-- ext/opcache/jit/Makefile.frag | 4 +- ext/opcache/jit/zend_jit.c | 93 ++++++++++++++++++++++++++- ext/opcache/jit/zend_jit.h | 7 +- ext/opcache/jit/zend_jit_disasm_x86.c | 19 ++---- ext/opcache/jit/zend_jit_gdb.c | 2 +- ext/opcache/jit/zend_jit_oprofile.c | 58 +++++++++++++++++ ext/opcache/jit/zend_jit_perf_dump.c | 47 ++++++++++++++ 8 files changed, 247 insertions(+), 24 deletions(-) create mode 100644 ext/opcache/jit/zend_jit_oprofile.c create mode 100644 ext/opcache/jit/zend_jit_perf_dump.c diff --git a/ext/opcache/config.m4 b/ext/opcache/config.m4 index ec59369752ce3..6dad85dc97afb 100644 --- a/ext/opcache/config.m4 +++ b/ext/opcache/config.m4 @@ -17,6 +17,9 @@ PHP_ARG_ENABLE(opcache-jit, whether to enable JIT, if test "$PHP_OPCACHE" != "no"; then + dnl Always build as shared extension + ext_shared=yes + if test "$PHP_OPCACHE_FILE" = "yes"; then AC_DEFINE(HAVE_OPCACHE_FILE_CACHE, 1, [Define to enable file based caching (experimental)]) fi @@ -28,7 +31,8 @@ if test "$PHP_OPCACHE" != "no"; then if test "$PHP_OPCACHE_JIT" = "yes"; then AC_DEFINE(HAVE_JIT, 1, [Define to enable JIT]) ZEND_JIT_SRC=jit/zend_jit.c - # Find out which ABI we are using. + + dnl Find out which ABI we are using. echo 'int i;' > conftest.$ac_ext if AC_TRY_EVAL(ac_compile); then case `/usr/bin/file conftest.o` in @@ -39,6 +43,31 @@ if test "$PHP_OPCACHE" != "no"; then esac fi rm -rf conftest* + + AC_MSG_CHECKING(for opagent in default path) + for i in /usr/local /usr; do + if test -r $i/include/opagent.h; then + OPAGENT_DIR=$i + AC_MSG_RESULT(found in $i) + break + fi + done + if test -z "$OPAGENT_DIR"; then + AC_MSG_RESULT(not found) + else + PHP_CHECK_LIBRARY(opagent, op_write_native_code, + [ + AC_DEFINE(HAVE_OPROFILE,1,[ ]) + PHP_ADD_INCLUDE($OPAGENT_DIR/include) + PHP_ADD_LIBRARY_WITH_PATH(opagent, $OPAGENT_DIR/$PHP_LIBDIR/oprofile, OPCACHE_SHARED_LIBADD) + PHP_SUBST(OPCACHE_SHARED_LIBADD) + ],[ + AC_MSG_RESULT(not found) + ],[ + -L$OPAGENT_DIR/$PHP_LIBDIR/oprofile + ]) + fi + fi AC_CHECK_FUNC(mprotect,[ @@ -367,12 +396,12 @@ AC_MSG_CHECKING("whether flock struct is linux ordered") AC_TRY_RUN([ #include struct flock lock = { 1, 2, 3, 4, 5 }; - int main() { + int main() { if(lock.l_type == 1 && lock.l_whence == 2 && lock.l_start == 3 && lock.l_len == 4) { return 0; } return 1; - } + } ], [ flock_type=linux AC_DEFINE([HAVE_FLOCK_LINUX], [], [Struct flock is Linux-type]) @@ -383,15 +412,15 @@ AC_MSG_CHECKING("whether flock struct is BSD ordered") AC_TRY_RUN([ #include struct flock lock = { 1, 2, 3, 4, 5 }; - int main() { + int main() { if(lock.l_start == 1 && lock.l_len == 2 && lock.l_type == 4 && lock.l_whence == 5) { return 0; } return 1; - } + } ], [ flock_type=bsd - AC_DEFINE([HAVE_FLOCK_BSD], [], [Struct flock is BSD-type]) + AC_DEFINE([HAVE_FLOCK_BSD], [], [Struct flock is BSD-type]) AC_MSG_RESULT("yes") ], AC_MSG_RESULT("no") ) diff --git a/ext/opcache/jit/Makefile.frag b/ext/opcache/jit/Makefile.frag index a9e175406a6f4..dcacd3ad9041d 100644 --- a/ext/opcache/jit/Makefile.frag +++ b/ext/opcache/jit/Makefile.frag @@ -8,5 +8,7 @@ $(builddir)/jit/zend_jit_x86.c: $(srcdir)/jit/zend_jit_x86.dasc $(srcdir)/jit/dy $(builddir)/jit/zend_jit.lo: \ $(builddir)/jit/zend_jit_x86.c \ $(srcdir)/jit/zend_jit_disasm_x86.c \ - $(srcdir)/jit/zend_jit_gdb.c + $(srcdir)/jit/zend_jit_gdb.c \ + $(srcdir)/jit/zend_jit_perf_dump.c \ + $(srcdir)/jit/zend_jit_oprofile.c diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index ad0deab81d028..759e5e7b20be7 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -18,6 +18,7 @@ #include #include "Zend/zend_execute.h" +#include "zend_smart_str.h" #include "jit/zend_jit.h" #ifdef HAVE_JIT @@ -30,6 +31,11 @@ #include "dynasm/dasm_x86.h" #include "jit/zend_jit_x86.c" #include "jit/zend_jit_disasm_x86.c" +#include "jit/zend_jit_gdb.c" +#include "jit/zend_jit_perf_dump.c" +#ifdef HAVE_OPROFILE +# include "jit/zend_jit_oprofile.c" +#endif #if _WIN32 # include @@ -46,6 +52,34 @@ static void *dasm_buf = NULL; static void *dasm_ptr = NULL; static void *dasm_end = NULL; +static zend_string *zend_jit_func_name(zend_op_array *op_array) +{ + smart_str buf = {0}; + + if (op_array->function_name) { + if (op_array->scope) { + smart_str_appends(&buf, "JIT$"); + smart_str_appendl(&buf, ZSTR_VAL(op_array->scope->name), ZSTR_LEN(op_array->scope->name)); + smart_str_appends(&buf, "::"); + smart_str_appendl(&buf, ZSTR_VAL(op_array->function_name), ZSTR_LEN(op_array->function_name)); + smart_str_0(&buf); + return buf.s; + } else { + smart_str_appends(&buf, "JIT$"); + smart_str_appendl(&buf, ZSTR_VAL(op_array->function_name), ZSTR_LEN(op_array->function_name)); + smart_str_0(&buf); + return buf.s; + } + } else if (op_array->filename) { + smart_str_appends(&buf, "JIT$"); + smart_str_appendl(&buf, ZSTR_VAL(op_array->filename), ZSTR_LEN(op_array->filename)); + smart_str_0(&buf); + return buf.s; + } else { + return NULL; + } +} + static void *dasm_link_and_encode(dasm_State **dasm_state) { size_t size; @@ -337,7 +371,43 @@ ZEND_API int zend_jit(zend_op_array *op_array, zend_script *script) #ifdef HAVE_DISASM if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_ASM) { - zend_jit_disasm(op_array, handler, dasm_ptr); + zend_string *name = zend_jit_func_name(op_array); + + zend_jit_disasm( + name ? ZSTR_VAL(name) : NULL, + op_array->filename ? ZSTR_VAL(op_array->filename) : NULL, + handler, dasm_ptr); + if (name) { + zend_string_release(name); + } + } +#endif + +#ifdef HAVE_OPROFILE + if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_OPROFILE) { + zend_string *name = zend_jit_func_name(op_array); + + zend_jit_oprofile_register( + name ? ZSTR_VAL(name) : NULL, + handler, + (char*)dasm_ptr - (char*)handler); + if (name) { + zend_string_release(name); + } + } +#endif + +#ifdef HAVE_PERFTOOLS + if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_PERF) { + zend_string *name = zend_jit_func_name(op_array); + + if (name) { + zend_jit_perf_dump( + ZSTR_VAL(name), + handler, + (char*)dasm_ptr - (char*)handler); + zend_string_release(name); + } } #endif @@ -374,6 +444,12 @@ ZEND_API int zend_jit_startup(size_t size) dasm_State* dasm_state = NULL; int ret; +#ifdef HAVE_OPROFILE + if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_OPROFILE) { + shared = 0; + } +#endif + /* Round up to the page size, which should be a power of two. */ page_size = jit_page_size(); if (!page_size || (page_size & (page_size - 1))) { @@ -398,6 +474,14 @@ ZEND_API int zend_jit_startup(size_t size) } } #endif +#ifdef HAVE_OPROFILE + if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_OPROFILE) { + if (!zend_jit_oprofile_startup()) { + // TODO: error reporting and cleanup ??? + return FAILURE; + } + } +#endif zend_jit_unprotect(); @@ -412,7 +496,7 @@ ZEND_API int zend_jit_startup(size_t size) #ifdef HAVE_DISASM if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_ASM) { if (dasm_buf != dasm_ptr) { - zend_jit_disasm(NULL, dasm_buf, dasm_ptr); + zend_jit_disasm(NULL, NULL, dasm_buf, dasm_ptr); } } #endif @@ -427,6 +511,11 @@ ZEND_API int zend_jit_startup(size_t size) ZEND_API void zend_jit_shutdown(void) { +#ifdef HAVE_OPROFILE + if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_OPROFILE) { + zend_jit_oprofile_shutdown(); + } +#endif if (dasm_buf) { jit_free(dasm_buf, ((char*)dasm_end) - ((char*)dasm_buf)); } diff --git a/ext/opcache/jit/zend_jit.h b/ext/opcache/jit/zend_jit.h index b8183c4e2430c..c51c8e0ee753b 100644 --- a/ext/opcache/jit/zend_jit.h +++ b/ext/opcache/jit/zend_jit.h @@ -21,7 +21,12 @@ #ifndef HAVE_JIT_H #define HAVE_JIT_H -#define ZEND_JIT_DEBUG_ASM (1<<0) +#define ZEND_JIT_DEBUG_ASM (1<<0) + +#define ZEND_JIT_DEBUG_GDB (1<<4) +#define ZEND_JIT_DEBUG_PERF (1<<5) +#define ZEND_JIT_DEBUG_OPROFILE (1<<6) + ZEND_API int zend_jit(zend_op_array *op_array, zend_script *script); ZEND_API void zend_jit_unprotect(void); diff --git a/ext/opcache/jit/zend_jit_disasm_x86.c b/ext/opcache/jit/zend_jit_disasm_x86.c index 934a7b99e21f3..11f583fdcd298 100644 --- a/ext/opcache/jit/zend_jit_disasm_x86.c +++ b/ext/opcache/jit/zend_jit_disasm_x86.c @@ -53,20 +53,13 @@ static const char* zend_jit_disasm_resolver(struct ud *ud, return NULL; } -static int zend_jit_disasm(zend_op_array *op_array, - const void *start, - const void *end) +static int zend_jit_disasm(const char *name, + const char *filename, + const void *start, + const void *end) { - if (op_array) { - if (op_array->function_name) { - if (op_array->scope) { - fprintf(stderr, "$_%s::%s: ; (%s)\n", ZSTR_VAL(op_array->scope->name), ZSTR_VAL(op_array->function_name), op_array->filename ? ZSTR_VAL(op_array->filename) : "unknown"); - } else { - fprintf(stderr, "$_%s: ; (%s)\n", ZSTR_VAL(op_array->function_name), ZSTR_VAL(op_array->filename), op_array->filename ? ZSTR_VAL(op_array->filename) : "unknown"); - } - } else if (op_array->filename) { - fprintf(stderr, "$_: ; (%s)\n", ZSTR_VAL(op_array->filename)); - } + if (name) { + fprintf(stderr, "%s: ; (%s)\n", name, filename ? filename : "unknown"); } ud_set_input_buffer(&ud, (uint8_t*)start, (uint8_t*)end - (uint8_t*)start); diff --git a/ext/opcache/jit/zend_jit_gdb.c b/ext/opcache/jit/zend_jit_gdb.c index 654159b10eb9f..453e7f1ccc6bf 100644 --- a/ext/opcache/jit/zend_jit_gdb.c +++ b/ext/opcache/jit/zend_jit_gdb.c @@ -35,7 +35,7 @@ struct _gdb_jit_descriptor { }; struct _gdb_jit_descriptor __jit_debug_descriptor = { - 1, GDBJIT_NOACTION, NULL, NULL + 1, GDB_JIT_NOACTION, NULL, NULL }; zend_never_inline void __jit_debug_register_code() diff --git a/ext/opcache/jit/zend_jit_oprofile.c b/ext/opcache/jit/zend_jit_oprofile.c new file mode 100644 index 0000000000000..a6221515f3854 --- /dev/null +++ b/ext/opcache/jit/zend_jit_oprofile.c @@ -0,0 +1,58 @@ +/* + +----------------------------------------------------------------------+ + | Zend JIT | + +----------------------------------------------------------------------+ + | Copyright (c) 1998-2016 The PHP Group | + +----------------------------------------------------------------------+ + | This source file is subject to version 3.01 of the PHP license, | + | that is bundled with this package in the file LICENSE, and is | + | available through the world-wide-web at the following url: | + | http://www.php.net/license/3_01.txt | + | If you did not receive a copy of the PHP license and are unable to | + | obtain it through the world-wide-web, please send a note to | + | license@php.net so we can mail you a copy immediately. | + +----------------------------------------------------------------------+ + | Authors: Dmitry Stogov | + +----------------------------------------------------------------------+ +*/ + +#define HAVE_OPROFILE 1 + +#include + +static op_agent_t op_agent = NULL; + +static void zend_jit_oprofile_register(const char *name, + const void *start, + size_t size) +{ + if (op_agent) { + op_write_native_code(op_agent, name, (uint64_t)(zend_uintptr_t)start, start, size); + } +} + +static int zend_jit_oprofile_startup(void) +{ + op_agent = op_open_agent(); + if (!op_agent) { + fprintf(stderr, "OpAgent initialization failed [%d]!\n", errno); + return 0; + } + return 1; +} + +static void zend_jit_oprofile_shutdown(void) +{ + if (op_agent) { +//??? sleep(60); + op_close_agent(op_agent); + } +} + +/* + * Local variables: + * tab-width: 4 + * c-basic-offset: 4 + * indent-tabs-mode: t + * End: + */ diff --git a/ext/opcache/jit/zend_jit_perf_dump.c b/ext/opcache/jit/zend_jit_perf_dump.c new file mode 100644 index 0000000000000..65b149d86d412 --- /dev/null +++ b/ext/opcache/jit/zend_jit_perf_dump.c @@ -0,0 +1,47 @@ +/* + +----------------------------------------------------------------------+ + | Zend JIT | + +----------------------------------------------------------------------+ + | Copyright (c) 1998-2016 The PHP Group | + +----------------------------------------------------------------------+ + | This source file is subject to version 3.01 of the PHP license, | + | that is bundled with this package in the file LICENSE, and is | + | available through the world-wide-web at the following url: | + | http://www.php.net/license/3_01.txt | + | If you did not receive a copy of the PHP license and are unable to | + | obtain it through the world-wide-web, please send a note to | + | license@php.net so we can mail you a copy immediately. | + +----------------------------------------------------------------------+ + | Authors: Dmitry Stogov | + +----------------------------------------------------------------------+ +*/ + +#define HAVE_PERFTOOLS 1 + +#include +#include + +static void zend_jit_perf_dump(const char *name, void *start, size_t size) +{ + static FILE *fp = NULL; + + if (!fp) { + char filename[64]; + + sprintf(filename, "/tmp/perf-%d.map", getpid()); + fp = fopen(filename, "w"); + if (!fp) { + return; + } + setlinebuf(fp); + } + fprintf(fp, "%lx %x %s\n", start, size, name); +} + +/* + * Local variables: + * tab-width: 4 + * c-basic-offset: 4 + * indent-tabs-mode: t + * End: + */ From 6981cd77c8cada05f07e6af5203661a70ff8c36e Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 19 Aug 2016 16:38:44 +0300 Subject: [PATCH 011/569] GDB support (incomplete) --- ext/opcache/jit/zend_jit.c | 14 +++ ext/opcache/jit/zend_jit_gdb.c | 162 +++++++++++++++++++++++++++++++-- 2 files changed, 170 insertions(+), 6 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 759e5e7b20be7..ff39e23e6812c 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -383,6 +383,20 @@ ZEND_API int zend_jit(zend_op_array *op_array, zend_script *script) } #endif +#ifdef HAVE_GDB + if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_GDB) { + zend_string *name = zend_jit_func_name(op_array); + + if (name) { + zend_jit_perf_dump( + ZSTR_VAL(name), + handler, + (char*)dasm_ptr - (char*)handler); + zend_string_release(name); + } + } +#endif + #ifdef HAVE_OPROFILE if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_OPROFILE) { zend_string *name = zend_jit_func_name(op_array); diff --git a/ext/opcache/jit/zend_jit_gdb.c b/ext/opcache/jit/zend_jit_gdb.c index 453e7f1ccc6bf..fd8461647dca5 100644 --- a/ext/opcache/jit/zend_jit_gdb.c +++ b/ext/opcache/jit/zend_jit_gdb.c @@ -16,6 +16,146 @@ +----------------------------------------------------------------------+ */ +#define HAVE_GDB + +#if SIZEOF_SIZE_T == 8 +# define ELF64 +#else +# undef ELF64 +#endif + +struct _elf_header { + uint8_t emagic[4]; + uint8_t eclass; + uint8_t eendian; + uint8_t eversion; + uint8_t eosabi; + uint8_t eabiversion; + uint8_t epad[7]; + uint16_t type; + uint16_t machine; + uint32_t version; + uintptr_t entry; + uintptr_t phofs; + uintptr_t shofs; + uint32_t flags; + uint16_t ehsize; + uint16_t phentsize; + uint16_t phnum; + uint16_t shentsize; + uint16_t shnum; + uint16_t shstridx; +}; + +struct _elf_sectheader { + uint32_t name; + uint32_t type; + uintptr_t flags; + uintptr_t addr; + uintptr_t ofs; + uintptr_t size; + uint32_t link; + uint32_t info; + uintptr_t align; + uintptr_t entsize; +}; + +struct _elf_symbol { +#ifdef ELF64 + uint32_t name; + uint8_t info; + uint8_t other; + uint16_t sectidx; + uintptr_t value; + uint64_t size; +#else + uint32_t name; + uintptr_t value; + uint32_t size; + uint8_t info; + uint8_t other; + uint16_t sectidx; +#endif +}; + +enum { + GDB_JIT_SECT_NULL, + GDB_JIT_SECT_text, + GDB_JIT_SECT_eh_frame, + GDB_JIT_SECT_shstrtab, + GDB_JIT_SECT_strtab, + GDB_JIT_SECT_symtab, + GDB_JIT_SECT_debug_info, + GDB_JIT_SECT_debug_abbrev, + GDB_JIT_SECT_debug_line, + GDB_JIT_SECT__MAX +}; + +enum { + GDB_JIT_SYM_UNDEF, + GDB_JIT_SYM_FILE, + GDB_JIT_SYM_FUNC, + GDB_JIT_SYM__MAX +}; + +struct _gdb_jit_obj { + struct _elf_header hdr; + struct _elf_sectheader sect[GDB_JIT_SECT__MAX]; + struct _elf_symbol sym[GDB_JIT_SYM__MAX]; + uint8_t space[4096]; +}; + +static const struct _elf_header template = { + .emagic = { 0x7f, 'E', 'L', 'F' }, +#ifdef ELF64 + .eclass = 2, +#else + .eclass = 1, +#endif +#ifdef WORDS_BIGENDIAN + .eendian = 2, +#else + .eendian = 1, +#endif + .eversion = 1, +#if deined(Linux) + .eosabi = 0, /* Nope, it's not 3. ??? */ +#elif defined(__FreeBSD__) + .eosabi = 9, +#elif defined(__NetBSD__) + .eosabi = 2, +#elif defined(__OpenBSD__) + .eosabi = 12, +#elif defined(__DragonFly__) + .eosabi = 0, +#elif (defined(__sun__) && defined(__svr4__)) + .eosabi = 6, +#else + .eosabi = 0, +#endif + .eabiversion = 0, + .epad = { 0, 0, 0, 0, 0, 0, 0 }, + .type = 1, +#if defind(__i386__) + .machine = 3, +#elif defined(__x86_64__) + .machine = 62, +#else +# error "Unsupported target architecture" +#endif + .version = 1, + .entry = 0, + .phofs = 0, + .shofs = offsetof(struct _gdb_jit_obj, sect), + .flags = 0, + .ehsize = sizeof(struct _elf_header), + .phentsize = 0, + .phnum = 0, + .shentsize = sizeof(struct _elf_sectheader), + .shnum = GDB_JIT_SECT__MAX, + .shstridx = GDB_JIT_SECT_shstrtab +}; + #define GDB_JIT_NOACTION 0 #define GDB_JIT_REGISTER 1 #define GDB_JIT_UNREGISTYER 2 @@ -43,16 +183,26 @@ zend_never_inline void __jit_debug_register_code() __asm__ __volatile__(""); } -int gdb_jit_register(const char*symfile_addr, uint64_t symfile_size) +static int zend_jit_gdb_register(const char *name, + const void *start, + size_t size) { struct _gdb_jit_code_entry *entry; + struct _gdb_jit_obj obj; + size_t obj_size; - entry = (struct _gdb_jit_code_entry*) malloc(sizeof(struct _gdb_jit_code_entry)); + // TODO: create in-memory ELF object file ??? + return 0; + + entry = malloc(sizeof(struct _gdb_jit_code_entry) + size); if (!entry) { - return -1; + return 0; } - entry->symfile_addr = symfile_addr; - entry->symfile_size = symfile_size; + + entry->symfile_addr = ((char*)entry) + sizeof(struct _gdb_jit_code_entry); + entry->symfile_size = obj_size; + memcpy(entry->symfile_addr, &obj, obj_size); + entry->prev_entry = NULL; entry->next_entry = __jit_debug_descriptor.first_entry; if (__jit_debug_descriptor.first_entry) { @@ -64,7 +214,7 @@ int gdb_jit_register(const char*symfile_addr, uint64_t symfile_size) __jit_debug_descriptor.action_flag = GDB_JIT_REGISTER; __jit_debug_register_code(); - return 0; + return 1; } /* From 547299b05bae27bc16715645ed126594fda64c95 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Sat, 20 Aug 2016 22:13:15 +0800 Subject: [PATCH 012/569] Fixed build --- ext/opcache/jit/zend_jit_disasm_x86.c | 2 ++ ext/opcache/jit/zend_jit_gdb.c | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ext/opcache/jit/zend_jit_disasm_x86.c b/ext/opcache/jit/zend_jit_disasm_x86.c index 11f583fdcd298..0c5ac88445482 100644 --- a/ext/opcache/jit/zend_jit_disasm_x86.c +++ b/ext/opcache/jit/zend_jit_disasm_x86.c @@ -69,6 +69,8 @@ static int zend_jit_disasm(const char *name, fprintf(stderr, "0x%" PRIx64 ":\t%s\n", ud_insn_off(&ud), ud_insn_asm(&ud)); } fprintf(stderr, "\n"); + + return 1; } static int zend_jit_disasm_init(void) diff --git a/ext/opcache/jit/zend_jit_gdb.c b/ext/opcache/jit/zend_jit_gdb.c index fd8461647dca5..399a4b4b2b83f 100644 --- a/ext/opcache/jit/zend_jit_gdb.c +++ b/ext/opcache/jit/zend_jit_gdb.c @@ -118,7 +118,7 @@ static const struct _elf_header template = { .eendian = 1, #endif .eversion = 1, -#if deined(Linux) +#if defined(Linux) .eosabi = 0, /* Nope, it's not 3. ??? */ #elif defined(__FreeBSD__) .eosabi = 9, @@ -136,7 +136,7 @@ static const struct _elf_header template = { .eabiversion = 0, .epad = { 0, 0, 0, 0, 0, 0, 0 }, .type = 1, -#if defind(__i386__) +#if defined(__i386__) .machine = 3, #elif defined(__x86_64__) .machine = 62, From 08a37e9104246ad6903a88c11efbbf6f3b5aba4b Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Mon, 22 Aug 2016 17:00:51 +0800 Subject: [PATCH 013/569] GDB support (not finish yet) --- ext/opcache/jit/zend_jit.c | 1 + ext/opcache/jit/zend_jit_gdb.c | 499 +++++++++++++++++++++++---- ext/opcache/jit/zend_jit_perf_dump.c | 2 +- 3 files changed, 443 insertions(+), 59 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index ff39e23e6812c..3bc9e125eeae4 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -388,6 +388,7 @@ ZEND_API int zend_jit(zend_op_array *op_array, zend_script *script) zend_string *name = zend_jit_func_name(op_array); if (name) { + /* TODO: register gdb as a whole file? */ zend_jit_perf_dump( ZSTR_VAL(name), handler, diff --git a/ext/opcache/jit/zend_jit_gdb.c b/ext/opcache/jit/zend_jit_gdb.c index 399a4b4b2b83f..f221244977eda 100644 --- a/ext/opcache/jit/zend_jit_gdb.c +++ b/ext/opcache/jit/zend_jit_gdb.c @@ -24,7 +24,7 @@ # undef ELF64 #endif -struct _elf_header { +typedef struct _zend_elf_header { uint8_t emagic[4]; uint8_t eclass; uint8_t eendian; @@ -45,9 +45,9 @@ struct _elf_header { uint16_t shentsize; uint16_t shnum; uint16_t shstridx; -}; +} zend_elf_header; -struct _elf_sectheader { +typedef struct zend_elf_sectheader { uint32_t name; uint32_t type; uintptr_t flags; @@ -58,9 +58,22 @@ struct _elf_sectheader { uint32_t info; uintptr_t align; uintptr_t entsize; +} zend_elf_sectheader; + +#define ELFSECT_IDX_ABS 0xfff1 + +enum { + ELFSECT_TYPE_PROGBITS = 1, + ELFSECT_TYPE_SYMTAB = 2, + ELFSECT_TYPE_STRTAB = 3, + ELFSECT_TYPE_NOBITS = 8 }; -struct _elf_symbol { +#define ELFSECT_FLAGS_WRITE 1 +#define ELFSECT_FLAGS_ALLOC 2 +#define ELFSECT_FLAGS_EXEC 4 + +typedef struct zend_elf_symbol { #ifdef ELF64 uint32_t name; uint8_t info; @@ -76,36 +89,119 @@ struct _elf_symbol { uint8_t other; uint16_t sectidx; #endif +} zend_elf_symbol; + +enum { + ELFSYM_TYPE_FUNC = 2, + ELFSYM_TYPE_FILE = 4, + ELFSYM_BIND_LOCAL = 0 << 4, + ELFSYM_BIND_GLOBAL = 1 << 4, +}; + +/* DWARF definitions. */ +#define DW_CIE_VERSION 1 + +enum { + DW_CFA_nop = 0x0, + DW_CFA_offset_extended = 0x5, + DW_CFA_def_cfa = 0xc, + DW_CFA_def_cfa_offset = 0xe, + DW_CFA_offset_extended_sf = 0x11, + DW_CFA_advance_loc = 0x40, + DW_CFA_offset = 0x80 +}; + +enum { + DW_EH_PE_udata4 = 3, + DW_EH_PE_textrel = 0x20 }; enum { - GDB_JIT_SECT_NULL, - GDB_JIT_SECT_text, - GDB_JIT_SECT_eh_frame, - GDB_JIT_SECT_shstrtab, - GDB_JIT_SECT_strtab, - GDB_JIT_SECT_symtab, - GDB_JIT_SECT_debug_info, - GDB_JIT_SECT_debug_abbrev, - GDB_JIT_SECT_debug_line, - GDB_JIT_SECT__MAX + DW_TAG_compile_unit = 0x11 }; enum { - GDB_JIT_SYM_UNDEF, - GDB_JIT_SYM_FILE, - GDB_JIT_SYM_FUNC, - GDB_JIT_SYM__MAX + DW_children_no = 0, + DW_children_yes = 1 }; -struct _gdb_jit_obj { - struct _elf_header hdr; - struct _elf_sectheader sect[GDB_JIT_SECT__MAX]; - struct _elf_symbol sym[GDB_JIT_SYM__MAX]; - uint8_t space[4096]; +enum { + DW_AT_name = 0x03, + DW_AT_stmt_list = 0x10, + DW_AT_low_pc = 0x11, + DW_AT_high_pc = 0x12 +}; + +enum { + DW_FORM_addr = 0x01, + DW_FORM_data4 = 0x06, + DW_FORM_string = 0x08 +}; + +enum { + DW_LNS_extended_op = 0, + DW_LNS_copy = 1, + DW_LNS_advance_pc = 2, + DW_LNS_advance_line = 3 +}; + +enum { + DW_LNE_end_sequence = 1, + DW_LNE_set_address = 2 +}; + +enum { +#if defined(__i386__) + DW_REG_AX, DW_REG_CX, DW_REG_DX, DW_REG_BX, + DW_REG_SP, DW_REG_BP, DW_REG_SI, DW_REG_DI, + DW_REG_RA, +#elif defined(__x86_64__) + /* Yes, the order is strange, but correct. */ + DW_REG_AX, DW_REG_DX, DW_REG_CX, DW_REG_BX, + DW_REG_SI, DW_REG_DI, DW_REG_BP, DW_REG_SP, + DW_REG_8, DW_REG_9, DW_REG_10, DW_REG_11, + DW_REG_12, DW_REG_13, DW_REG_14, DW_REG_15, + DW_REG_RA, + /* TODO: ARM supports? */ +#else +#error "Unsupported target architecture" +#endif }; -static const struct _elf_header template = { +enum { + GDBJIT_NOACTION, + GDBJIT_REGISTER, + GDBJIT_UNREGISTYER +}; + +enum { + GDBJIT_SECT_NULL, + GDBJIT_SECT_text, + GDBJIT_SECT_eh_frame, + GDBJIT_SECT_shstrtab, + GDBJIT_SECT_strtab, + GDBJIT_SECT_symtab, + GDBJIT_SECT_debug_info, + GDBJIT_SECT_debug_abbrev, + GDBJIT_SECT_debug_line, + GDBJIT_SECT__MAX +}; + +enum { + GDBJIT_SYM_UNDEF, + GDBJIT_SYM_FILE, + GDBJIT_SYM_FUNC, + GDBJIT_SYM__MAX +}; + +typedef struct _zend_gdbjit_obj { + zend_elf_header hdr; + zend_elf_sectheader sect[GDBJIT_SECT__MAX]; + zend_elf_symbol sym[GDBJIT_SYM__MAX]; + uint8_t space[4096]; +} zend_gdbjit_obj; + +static const zend_elf_header zend_elfhdr_template = { .emagic = { 0x7f, 'E', 'L', 'F' }, #ifdef ELF64 .eclass = 2, @@ -122,8 +218,6 @@ static const struct _elf_header template = { .eosabi = 0, /* Nope, it's not 3. ??? */ #elif defined(__FreeBSD__) .eosabi = 9, -#elif defined(__NetBSD__) - .eosabi = 2, #elif defined(__OpenBSD__) .eosabi = 12, #elif defined(__DragonFly__) @@ -146,36 +240,47 @@ static const struct _elf_header template = { .version = 1, .entry = 0, .phofs = 0, - .shofs = offsetof(struct _gdb_jit_obj, sect), + .shofs = offsetof(zend_gdbjit_obj, sect), .flags = 0, - .ehsize = sizeof(struct _elf_header), + .ehsize = sizeof(zend_elf_header), .phentsize = 0, .phnum = 0, - .shentsize = sizeof(struct _elf_sectheader), - .shnum = GDB_JIT_SECT__MAX, - .shstridx = GDB_JIT_SECT_shstrtab + .shentsize = sizeof(zend_elf_sectheader), + .shnum = GDBJIT_SECT__MAX, + .shstridx = GDBJIT_SECT_shstrtab }; -#define GDB_JIT_NOACTION 0 -#define GDB_JIT_REGISTER 1 -#define GDB_JIT_UNREGISTYER 2 - -struct _gdb_jit_code_entry { - struct _gdb_jit_code_entry *next_entry; - struct _gdb_jit_code_entry *prev_entry; +typedef struct _zend_gdbjit_code_entry { + struct _zend_gdbjit_code_entry *next_entry; + struct _zend_gdbjit_code_entry *prev_entry; const char *symfile_addr; uint64_t symfile_size; -}; +} zend_gdbjit_code_entry; -struct _gdb_jit_descriptor { +typedef struct _zend_gdbjit_descriptor { uint32_t version; uint32_t action_flag; - struct _gdb_jit_code_entry *relevant_entry; - struct _gdb_jit_code_entry *first_entry; -}; + struct _zend_gdbjit_code_entry *relevant_entry; + struct _zend_gdbjit_code_entry *first_entry; +} zend_gdbjit_descriptor; + +/* Context for generating the ELF object for the GDB JIT API. */ +typedef struct _zend_gdbjit_ctx { + uint8_t *p; /* Pointer to next address in obj.space. */ + uint8_t *startp; /* Pointer to start address in obj.space. */ + void *T; /* Generate symbols for this trace. */ + uintptr_t mcaddr; /* Machine code address. */ + uint32_t szmcode; /* Size of machine code. */ + uint32_t spadjp; /* Stack adjustment for parent trace or interpreter. */ + uint32_t spadj; /* Stack adjustment for trace itself. */ + int32_t lineno; /* Starting line number. */ + const char *filename; /* Starting file name. */ + size_t objsize; /* Final size of ELF object. */ + zend_gdbjit_obj obj; /* In-memory ELF object. */ +} zend_gdbjit_ctx; -struct _gdb_jit_descriptor __jit_debug_descriptor = { - 1, GDB_JIT_NOACTION, NULL, NULL +zend_gdbjit_descriptor __jit_debug_descriptor = { + 1, GDBJIT_NOACTION, NULL, NULL }; zend_never_inline void __jit_debug_register_code() @@ -183,25 +288,296 @@ zend_never_inline void __jit_debug_register_code() __asm__ __volatile__(""); } +/* Add a zero-terminated string */ +static uint32_t zend_gdbjit_strz(zend_gdbjit_ctx *ctx, const char *str) +{ + uint8_t *p = ctx->p; + uint32_t ofs = (uint32_t)(p - ctx->startp); + do { + *p++ = (uint8_t)*str; + } while (*str++); + ctx->p = p; + return ofs; +} + +/* Add a ULEB128 value */ +static void zend_gdbjit_uleb128(zend_gdbjit_ctx *ctx, uint32_t v) +{ + uint8_t *p = ctx->p; + for (; v >= 0x80; v >>= 7) + *p++ = (uint8_t)((v & 0x7f) | 0x80); + *p++ = (uint8_t)v; + ctx->p = p; +} + +/* Add a SLEB128 value */ +static void zend_gdbjit_sleb128(zend_gdbjit_ctx *ctx, int32_t v) +{ + uint8_t *p = ctx->p; + for (; (uint32_t)(v+0x40) >= 0x80; v >>= 7) + *p++ = (uint8_t)((v & 0x7f) | 0x80); + *p++ = (uint8_t)(v & 0x7f); + ctx->p = p; +} + +static void zend_gdbjit_secthdr(zend_gdbjit_ctx *ctx) { + zend_elf_sectheader *sect; + + *ctx->p++ = '\0'; + +#define SECTDEF(id, tp, al) \ + sect = &ctx->obj.sect[GDBJIT_SECT_##id]; \ + sect->name = zend_gdbjit_strz(ctx, "." #id); \ + sect->type = ELFSECT_TYPE_##tp; \ + sect->align = (al) + + SECTDEF(text, NOBITS, 16); + sect->flags = ELFSECT_FLAGS_ALLOC|ELFSECT_FLAGS_EXEC; + sect->addr = ctx->mcaddr; + sect->ofs = 0; + sect->size = ctx->szmcode; + + SECTDEF(eh_frame, PROGBITS, sizeof(uintptr_t)); + sect->flags = ELFSECT_FLAGS_ALLOC; + + SECTDEF(shstrtab, STRTAB, 1); + SECTDEF(strtab, STRTAB, 1); + + SECTDEF(symtab, SYMTAB, sizeof(uintptr_t)); + sect->ofs = offsetof(zend_gdbjit_obj, sym); + sect->size = sizeof(ctx->obj.sym); + sect->link = GDBJIT_SECT_strtab; + sect->entsize = sizeof(zend_elf_symbol); + sect->info = GDBJIT_SYM_FUNC; + + SECTDEF(debug_info, PROGBITS, 1); + SECTDEF(debug_abbrev, PROGBITS, 1); + SECTDEF(debug_line, PROGBITS, 1); + +#undef SECTDEF +} + +static void zend_gdbjit_symtab(zend_gdbjit_ctx *ctx) +{ + zend_elf_symbol *sym; + + *ctx->p++ = '\0'; + + sym = &ctx->obj.sym[GDBJIT_SYM_FILE]; + sym->name = zend_gdbjit_strz(ctx, "JIT mcode"); + sym->sectidx = ELFSECT_IDX_ABS; + sym->info = ELFSYM_TYPE_FILE|ELFSYM_BIND_LOCAL; + + sym = &ctx->obj.sym[GDBJIT_SYM_FUNC]; + sym->name = zend_gdbjit_strz(ctx, "JIT_FUNC"); ctx->p--; + sym->sectidx = GDBJIT_SECT_text; + sym->value = 0; + sym->size = ctx->szmcode; + sym->info = ELFSYM_TYPE_FUNC|ELFSYM_BIND_GLOBAL; +} + + +#define SECTALIGN(p, a) \ + ((p) = (uint8_t *)(((uintptr_t)(p) + ((a)-1)) & ~(uintptr_t)((a)-1))) + + +/* Shortcuts to generate DWARF structures. */ +#define DB(x) (*p++ = (x)) +#define DI8(x) (*(int8_t *)p = (x), p++) +#define DU16(x) (*(uint16_t *)p = (x), p += 2) +#define DU32(x) (*(uint32_t *)p = (x), p += 4) +#define DADDR(x) (*(uintptr_t *)p = (x), p += sizeof(uintptr_t)) +#define DUV(x) (ctx->p = p, zend_gdbjit_uleb128(ctx, (x)), p = ctx->p) +#define DSV(x) (ctx->p = p, zend_gdbjit_sleb128(ctx, (x)), p = ctx->p) +#define DSTR(str) (ctx->p = p, zend_gdbjit_strz(ctx, (str)), p = ctx->p) +#define DALIGNNOP(s) while ((uintptr_t)p & ((s)-1)) *p++ = DW_CFA_nop +#define DSECT(name, stmt) \ + { uint32_t *szp_##name = (uint32_t *)p; p += 4; stmt \ + *szp_##name = (uint32_t)((p-(uint8_t *)szp_##name)-4); } + +static void zend_gdbjit_ehframe(zend_gdbjit_ctx *ctx) +{ + uint8_t *p = ctx->p; + uint8_t *framep = p; + + /* Emit DWARF EH CIE. */ + DSECT(CIE, + DU32(0); /* Offset to CIE itself. */ + DB(DW_CIE_VERSION); + DSTR("zR"); /* Augmentation. */ + DUV(1); /* Code alignment factor. */ + DSV(-(int32_t)sizeof(uintptr_t)); /* Data alignment factor. */ + DB(DW_REG_RA); /* Return address register. */ + DB(1); DB(DW_EH_PE_textrel|DW_EH_PE_udata4); /* Augmentation data. */ + DB(DW_CFA_def_cfa); DUV(DW_REG_SP); DUV(sizeof(uintptr_t)); + DB(DW_CFA_offset|DW_REG_RA); DUV(1); + DALIGNNOP(sizeof(uintptr_t)); + ) + + /* Emit DWARF EH FDE. */ + DSECT(FDE, + DU32((uint32_t)(p-framep)); /* Offset to CIE. */ + DU32(0); /* Machine code offset relative to .text. */ + DU32(ctx->szmcode); /* Machine code length. */ + DB(0); /* Augmentation data. */ + /* Registers saved in CFRAME. */ +#if defined(__i386__) + DB(DW_CFA_offset|DW_REG_BP); DUV(2); + DB(DW_CFA_offset|DW_REG_DI); DUV(3); + DB(DW_CFA_offset|DW_REG_SI); DUV(4); + DB(DW_CFA_offset|DW_REG_BX); DUV(5); +#elif defined(__x86_64__) + DB(DW_CFA_offset|DW_REG_BP); DUV(2); + DB(DW_CFA_offset|DW_REG_BX); DUV(3); + DB(DW_CFA_offset|DW_REG_15); DUV(4); + DB(DW_CFA_offset|DW_REG_14); DUV(5); + /* Extra registers saved for JIT-compiled code. */ + DB(DW_CFA_offset|DW_REG_13); DUV(9); + DB(DW_CFA_offset|DW_REG_12); DUV(10); +#else +#error "Unsupported target architecture" +#endif + if (ctx->spadjp != ctx->spadj) /* Parent/interpreter stack frame size. */ + (DB(DW_CFA_def_cfa_offset), DUV(ctx->spadjp), DB(DW_CFA_advance_loc|1)); /* Only an approximation. */ + DB(DW_CFA_def_cfa_offset); DUV(ctx->spadj); /* Trace stack frame size. */ + DALIGNNOP(sizeof(uintptr_t)); + ) + + ctx->p = p; +} + +static void zend_gdbjit_debuginfo(zend_gdbjit_ctx *ctx) +{ + uint8_t *p = ctx->p; + + DSECT(info, + DU16(2); /* DWARF version. */ + DU32(0); /* Abbrev offset. */ + DB(sizeof(uintptr_t)); /* Pointer size. */ + + DUV(1); /* Abbrev #1: DW_TAG_compile_unit. */ + DSTR(ctx->filename); /* DW_AT_name. */ + DADDR(ctx->mcaddr); /* DW_AT_low_pc. */ + DADDR(ctx->mcaddr + ctx->szmcode); /* DW_AT_high_pc. */ + DU32(0); /* DW_AT_stmt_list. */ + ); + + ctx->p = p; +} + +static void zend_gdbjit_debugabbrev(zend_gdbjit_ctx *ctx) +{ + uint8_t *p = ctx->p; + + /* Abbrev #1: DW_TAG_compile_unit. */ + DUV(1); DUV(DW_TAG_compile_unit); + DB(DW_children_no); + DUV(DW_AT_name); DUV(DW_FORM_string); + DUV(DW_AT_low_pc); DUV(DW_FORM_addr); + DUV(DW_AT_high_pc); DUV(DW_FORM_addr); + DUV(DW_AT_stmt_list); DUV(DW_FORM_data4); + DB(0); DB(0); + + ctx->p = p; +} + + +#define DLNE(op, s) (DB(DW_LNS_extended_op), DUV(1+(s)), DB((op))) + +static void zend_gdbjit_debugline(zend_gdbjit_ctx *ctx) +{ + uint8_t *p = ctx->p; + + DSECT(line, + DU16(2); /* DWARF version. */ + DSECT(header, + DB(1); /* Minimum instruction length. */ + DB(1); /* is_stmt. */ + DI8(0); /* Line base for special opcodes. */ + DB(2); /* Line range for special opcodes. */ + DB(3+1); /* Opcode base at DW_LNS_advance_line+1. */ + DB(0); DB(1); DB(1); /* Standard opcode lengths. */ + /* Directory table. */ + DB(0); + /* File name table. */ + DSTR(ctx->filename); DUV(0); DUV(0); DUV(0); + DB(0); + ); + DLNE(DW_LNE_set_address, sizeof(uintptr_t)); + DADDR(ctx->mcaddr); + if (ctx->lineno) (DB(DW_LNS_advance_line), DSV(ctx->lineno-1)); + DB(DW_LNS_copy); + DB(DW_LNS_advance_pc); DUV(ctx->szmcode); + DLNE(DW_LNE_end_sequence, 0); + ); + + ctx->p = p; +} + + +#undef DLNE + +/* Undef shortcuts. */ +#undef DB +#undef DI8 +#undef DU16 +#undef DU32 +#undef DADDR +#undef DUV +#undef DSV +#undef DSTR +#undef DALIGNNOP +#undef DSECT + +typedef void (*zend_gdbjit_initf) (zend_gdbjit_ctx *ctx); + +static void zend_gdbjit_initsect(zend_gdbjit_ctx *ctx, int sect, zend_gdbjit_initf initf) { + ctx->startp = ctx->p; + ctx->obj.sect[sect].ofs = (uintptr_t)((char *)ctx->p - (char *)&ctx->obj); + initf(ctx); + ctx->obj.sect[sect].size = (uintptr_t)(ctx->p - ctx->startp); +} + +static void zend_gdbjit_buildobj(zend_gdbjit_ctx *ctx) { + zend_gdbjit_obj *obj = &ctx->obj; + + /* Fill in ELF header and clear structures. */ + memcpy(&obj->hdr, &zend_elfhdr_template, sizeof(zend_elf_header)); + memset(&obj->sect, 0, sizeof(zend_elf_sectheader) * GDBJIT_SECT__MAX); + memset(&obj->sym, 0, sizeof(zend_elf_symbol) * GDBJIT_SYM__MAX); + + /* Initialize sections. */ + ctx->p = obj->space; + zend_gdbjit_initsect(ctx, GDBJIT_SECT_shstrtab, zend_gdbjit_secthdr); + zend_gdbjit_initsect(ctx, GDBJIT_SECT_strtab, zend_gdbjit_symtab); + zend_gdbjit_initsect(ctx, GDBJIT_SECT_debug_info, zend_gdbjit_debuginfo); + zend_gdbjit_initsect(ctx, GDBJIT_SECT_debug_abbrev, zend_gdbjit_debugabbrev); + zend_gdbjit_initsect(ctx, GDBJIT_SECT_debug_line, zend_gdbjit_debugline); + SECTALIGN(ctx->p, sizeof(uintptr_t)); + zend_gdbjit_initsect(ctx, GDBJIT_SECT_eh_frame, zend_gdbjit_ehframe); + ctx->objsize = (size_t)((char *)ctx->p - (char *)obj); + + ZEND_ASSERT(ctx->objsize < sizeof(zend_gdbjit_obj)); +} + static int zend_jit_gdb_register(const char *name, const void *start, size_t size) { - struct _gdb_jit_code_entry *entry; - struct _gdb_jit_obj obj; - size_t obj_size; + zend_gdbjit_ctx ctx; + zend_gdbjit_code_entry *entry; - // TODO: create in-memory ELF object file ??? - return 0; + ctx.mcaddr = (uintptr_t)start; + ctx.szmcode = (uint32_t)size; + ctx.filename = name; - entry = malloc(sizeof(struct _gdb_jit_code_entry) + size); - if (!entry) { - return 0; - } + zend_gdbjit_buildobj(&ctx); - entry->symfile_addr = ((char*)entry) + sizeof(struct _gdb_jit_code_entry); - entry->symfile_size = obj_size; - memcpy(entry->symfile_addr, &obj, obj_size); + entry = emalloc(sizeof(zend_gdbjit_code_entry) + ctx.objsize); + entry->symfile_addr = ((char*)entry) + sizeof(zend_gdbjit_code_entry); + entry->symfile_size = ctx.objsize; + + memcpy((char *)entry->symfile_addr, &ctx.obj, ctx.objsize); entry->prev_entry = NULL; entry->next_entry = __jit_debug_descriptor.first_entry; @@ -211,12 +587,19 @@ static int zend_jit_gdb_register(const char *name, __jit_debug_descriptor.first_entry = entry; __jit_debug_descriptor.relevant_entry = entry; - __jit_debug_descriptor.action_flag = GDB_JIT_REGISTER; + __jit_debug_descriptor.action_flag = GDBJIT_REGISTER; __jit_debug_register_code(); return 1; } +static int zend_jit_gdb_unregister(const char *name, + const void *start, + size_t size) +{ + return 1; +} + /* * Local variables: * tab-width: 4 diff --git a/ext/opcache/jit/zend_jit_perf_dump.c b/ext/opcache/jit/zend_jit_perf_dump.c index 65b149d86d412..63ee96265ecc3 100644 --- a/ext/opcache/jit/zend_jit_perf_dump.c +++ b/ext/opcache/jit/zend_jit_perf_dump.c @@ -35,7 +35,7 @@ static void zend_jit_perf_dump(const char *name, void *start, size_t size) } setlinebuf(fp); } - fprintf(fp, "%lx %x %s\n", start, size, name); + fprintf(fp, "%lx %lx %s\n", (size_t)start, size, name); } /* From a5d5b6f62cc5a692bea3ae1c91a6fd72967d32b6 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 22 Aug 2016 12:31:13 +0300 Subject: [PATCH 014/569] typo --- ext/opcache/jit/zend_jit.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 3bc9e125eeae4..9e3d0ccbb7143 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -388,8 +388,7 @@ ZEND_API int zend_jit(zend_op_array *op_array, zend_script *script) zend_string *name = zend_jit_func_name(op_array); if (name) { - /* TODO: register gdb as a whole file? */ - zend_jit_perf_dump( + zend_jit_gdb_register( ZSTR_VAL(name), handler, (char*)dasm_ptr - (char*)handler); From f4a26d30b61d98ee50ff9262907279815829fea7 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 22 Aug 2016 17:29:29 +0300 Subject: [PATCH 015/569] Fixed swithing into interpretation mode --- ext/opcache/jit/zend_jit.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 9e3d0ccbb7143..619f6cd8d5bc8 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -184,7 +184,7 @@ ZEND_API int zend_jit(zend_op_array *op_array, zend_script *script) /* Build SSA */ memset(&ssa, 0, sizeof(zend_ssa)); - if (zend_build_cfg(&CG(arena), op_array, ZEND_RT_CONSTANTS | ZEND_SSA_RC_INFERENCE, &ssa.cfg, &flags) != SUCCESS) { + if (zend_build_cfg(&CG(arena), op_array, ZEND_CFG_STACKLESS | ZEND_RT_CONSTANTS | ZEND_SSA_RC_INFERENCE, &ssa.cfg, &flags) != SUCCESS) { goto jit_failure; } @@ -214,6 +214,9 @@ ZEND_API int zend_jit(zend_op_array *op_array, zend_script *script) continue; } if (ssa.cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_ENTRY)) { + if (ssa.cfg.blocks[b].flags & ZEND_BB_ENTRY) { + zend_jit_jmp(&dasm_state, b); + } zend_jit_label(&dasm_state, ssa.cfg.blocks_count + b); zend_jit_prologue(&dasm_state); } @@ -231,12 +234,12 @@ ZEND_API int zend_jit(zend_op_array *op_array, zend_script *script) opline = op_array->opcodes + i; switch (opline->opcode) { case ZEND_RECV: + case ZEND_RECV_INIT: /* RECV may be skipped */ if (!zend_jit_recv(&dasm_state, opline)) { goto jit_failure; } break; - case ZEND_RECV_INIT: case ZEND_BIND_GLOBAL: if (opline->opcode != op_array->opcodes[i+1].opcode) { /* repeatable opcodes */ @@ -365,6 +368,12 @@ ZEND_API int zend_jit(zend_op_array *op_array, zend_script *script) opline = op_array->opcodes + ssa.cfg.blocks[b].start; opline->handler = (void*)(((char*)handler) + dasm_getpclabel(&dasm_state, ssa.cfg.blocks_count + b)); + /* RECV may be skipped */ + while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) { + opline++; + opline->handler = (void*)(((char*)handler) + + dasm_getpclabel(&dasm_state, ssa.cfg.blocks_count + b)); + } } } dasm_free(&dasm_state); From e5a39927d45a2e449dca8ff7f7b38b0ae3bfaf7d Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 22 Aug 2016 17:48:04 +0300 Subject: [PATCH 016/569] Prepare for smarter JIT --- ext/opcache/jit/zend_jit.c | 29 +++++++++++++++++++++++++++++ ext/opcache/jit/zend_jit.h | 1 + 2 files changed, 30 insertions(+) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 619f6cd8d5bc8..eb2c3fab8f8f7 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -24,6 +24,13 @@ #ifdef HAVE_JIT #include "Optimizer/zend_ssa.h" +#include "Optimizer/zend_inference.h" +#include "Optimizer/zend_dump.h" + +#define ZEND_JIT_LEVEL 3 // 0 - no JIT + // 1 - minimal JIT (subroutine threading) + // 2 - selective inline threading + // 3 - optimized JIT based on Type-Inference // TODO: define DASM_M_GROW and DASM_M_FREE to use CG(arena) ??? @@ -202,6 +209,28 @@ ZEND_API int zend_jit(zend_op_array *op_array, zend_script *script) goto jit_failure; } + if (ZEND_JIT_LEVEL >= 3) { + if (zend_build_ssa(&CG(arena), script, op_array, ZEND_RT_CONSTANTS | ZEND_SSA_RC_INFERENCE, &ssa, &flags) != SUCCESS) { + goto jit_failure; + } + + if (zend_ssa_compute_use_def_chains(&CG(arena), op_array, &ssa) != SUCCESS) { + goto jit_failure; + } + + if (zend_ssa_find_false_dependencies(op_array, &ssa) != SUCCESS) { + goto jit_failure; + } + + if (zend_ssa_inference(&CG(arena), op_array, script, &ssa) != SUCCESS) { + goto jit_failure; + } + + if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_SSA) { + zend_dump_op_array(op_array, ZEND_DUMP_HIDE_UNREACHABLE|ZEND_DUMP_RC_INFERENCE|ZEND_DUMP_SSA|ZEND_DUMP_RT_CONSTANTS, "JIT", &ssa); + } + } + dasm_init(&dasm_state, DASM_MAXSECTION); dasm_setupglobal(&dasm_state, dasm_labels, lbl__MAX); dasm_setup(&dasm_state, dasm_actions); diff --git a/ext/opcache/jit/zend_jit.h b/ext/opcache/jit/zend_jit.h index c51c8e0ee753b..1369211da5c7d 100644 --- a/ext/opcache/jit/zend_jit.h +++ b/ext/opcache/jit/zend_jit.h @@ -22,6 +22,7 @@ #define HAVE_JIT_H #define ZEND_JIT_DEBUG_ASM (1<<0) +#define ZEND_JIT_DEBUG_SSA (1<<1) #define ZEND_JIT_DEBUG_GDB (1<<4) #define ZEND_JIT_DEBUG_PERF (1<<5) From 2ed2aa09a49d8bf16c750116e6092eccb377fc39 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Tue, 23 Aug 2016 11:37:44 +0800 Subject: [PATCH 017/569] Use more meaningful macro --- ext/opcache/jit/zend_jit.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index eb2c3fab8f8f7..e0af62bd2ea31 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -27,10 +27,13 @@ #include "Optimizer/zend_inference.h" #include "Optimizer/zend_dump.h" -#define ZEND_JIT_LEVEL 3 // 0 - no JIT - // 1 - minimal JIT (subroutine threading) - // 2 - selective inline threading - // 3 - optimized JIT based on Type-Inference + +#define ZEND_JIT_LEVEL_NONE 0 /* no JIT */ +#define ZEND_JIT_LEVEL_MINI 1 /* minimal JIT (subroutine threading) */ +#define ZEND_JIT_LEVEL_INLINE 2 /* selective inline threading */ +#define ZEND_JIT_LEVEL_FULL 3 /* optimized JIT based on Type-Inference */ + +#define ZEND_JIT_LEVEL ZEND_JIT_LEVEL_FULL // TODO: define DASM_M_GROW and DASM_M_FREE to use CG(arena) ??? @@ -209,7 +212,7 @@ ZEND_API int zend_jit(zend_op_array *op_array, zend_script *script) goto jit_failure; } - if (ZEND_JIT_LEVEL >= 3) { + if (ZEND_JIT_LEVEL >= ZEND_JIT_LEVEL_FULL) { if (zend_build_ssa(&CG(arena), script, op_array, ZEND_RT_CONSTANTS | ZEND_SSA_RC_INFERENCE, &ssa, &flags) != SUCCESS) { goto jit_failure; } From 382bbac2e77f33ff8ef5f3d533db03cb7f65fccc Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Tue, 23 Aug 2016 11:44:20 +0800 Subject: [PATCH 018/569] Fixed typo? --- ext/opcache/jit/zend_jit_x86.dasc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 507dcf3074800..ece98b207bb37 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -113,7 +113,7 @@ static int zend_jit_handler(dasm_State **Dst, zend_op *opline) |.else | call aword &handler |.endif - zend_jit_check_exception(Dst); + zend_jit_check_timeout(Dst); return 1; } From 20eafd64d23c238dc3bb5cf9860d05a77dead007 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Tue, 23 Aug 2016 11:46:28 +0800 Subject: [PATCH 019/569] Revert "Fixed typo?" This reverts commit 382bbac2e77f33ff8ef5f3d533db03cb7f65fccc. --- ext/opcache/jit/zend_jit_x86.dasc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index ece98b207bb37..507dcf3074800 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -113,7 +113,7 @@ static int zend_jit_handler(dasm_State **Dst, zend_op *opline) |.else | call aword &handler |.endif - zend_jit_check_timeout(Dst); + zend_jit_check_exception(Dst); return 1; } From 9cf2b1df7309265a673b162cf74833b430ec2a10 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Tue, 23 Aug 2016 14:02:47 +0800 Subject: [PATCH 020/569] Fixed segfault of null pointer def in type inference(script) besides, it's better to JIT a whole script (good for gdb regsiter) --- ext/opcache/jit/zend_jit.c | 42 +++++++++++++++++++++++++++++++++++--- ext/opcache/jit/zend_jit.h | 4 ++-- ext/opcache/zend_persist.c | 15 +++----------- 3 files changed, 44 insertions(+), 17 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index e0af62bd2ea31..a1b2ed92d3ac0 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -19,6 +19,7 @@ #include #include "Zend/zend_execute.h" #include "zend_smart_str.h" +#include "zend_shared_alloc.h" #include "jit/zend_jit.h" #ifdef HAVE_JIT @@ -175,7 +176,7 @@ static void jit_free(void *p, size_t size) #endif } -ZEND_API int zend_jit(zend_op_array *op_array, zend_script *script) +static int zend_jit_op_array(zend_op_array *op_array, zend_script *script) { uint32_t flags; zend_ssa ssa; @@ -185,10 +186,12 @@ ZEND_API int zend_jit(zend_op_array *op_array, zend_script *script) dasm_State* dasm_state = NULL; void *handler; - if (!dasm_buf) { - return FAILURE; + if (zend_shared_alloc_get_xlat_entry(op_array->opcodes)) { + return SUCCESS; } + zend_shared_alloc_register_xlat_entry(op_array->opcodes, op_array->opcodes); + checkpoint = zend_arena_checkpoint(CG(arena)); /* Build SSA */ @@ -477,6 +480,39 @@ ZEND_API int zend_jit(zend_op_array *op_array, zend_script *script) return FAILURE; } +static int zend_jit_func_table(zval *zv, void *arg) { + zend_op_array *op_array = (zend_op_array*)Z_PTR_P(zv); + zend_script *script = (zend_script *)arg; + + zend_jit_op_array(op_array, script); + + return ZEND_HASH_APPLY_KEEP; +} + +static int zend_jit_class_table(zval *zv, void *arg) { + zend_class_entry *ce = (zend_class_entry *)Z_PTR_P(zv); + zend_script *script = (zend_script *)arg; + + zend_hash_apply_with_argument(&ce->function_table, zend_jit_func_table, script); + + return ZEND_HASH_APPLY_KEEP; +} + +ZEND_API int zend_jit(zend_script *script) { + if (!dasm_buf) { + return FAILURE; + } + + /* xlat table is initialized already */ + zend_shared_alloc_clear_xlat_table(); + + zend_hash_apply_with_argument(&script->class_table, zend_jit_class_table, script); + zend_hash_apply_with_argument(&script->function_table, zend_jit_func_table, script); + zend_jit_op_array(&script->main_op_array, script); + + return SUCCESS; +} + ZEND_API void zend_jit_unprotect(void) { #ifdef HAVE_MPROTECT diff --git a/ext/opcache/jit/zend_jit.h b/ext/opcache/jit/zend_jit.h index 1369211da5c7d..34492e7fffb09 100644 --- a/ext/opcache/jit/zend_jit.h +++ b/ext/opcache/jit/zend_jit.h @@ -29,7 +29,7 @@ #define ZEND_JIT_DEBUG_OPROFILE (1<<6) -ZEND_API int zend_jit(zend_op_array *op_array, zend_script *script); +ZEND_API int zend_jit(zend_script *script); ZEND_API void zend_jit_unprotect(void); ZEND_API void zend_jit_protect(void); ZEND_API int zend_jit_startup(size_t size); @@ -43,4 +43,4 @@ ZEND_API void zend_jit_shutdown(void); * c-basic-offset: 4 * indent-tabs-mode: t * End: - */ \ No newline at end of file + */ diff --git a/ext/opcache/zend_persist.c b/ext/opcache/zend_persist.c index f47cabaf1336f..3dad282edc0e5 100644 --- a/ext/opcache/zend_persist.c +++ b/ext/opcache/zend_persist.c @@ -336,9 +336,6 @@ static void zend_persist_op_array_ex(zend_op_array *op_array, zend_persistent_sc int already_stored = 0; zend_op *persist_ptr; zval *orig_literals = NULL; -#ifdef HAVE_JIT - int do_jit = 1; -#endif if (op_array->type != ZEND_USER_FUNCTION) { return; @@ -405,9 +402,6 @@ static void zend_persist_op_array_ex(zend_op_array *op_array, zend_persistent_sc persist_ptr = zend_shared_alloc_get_xlat_entry(op_array->opcodes); ZEND_ASSERT(persist_ptr != NULL); op_array->opcodes = persist_ptr; -#ifdef HAVE_JIT - do_jit = 0; -#endif } else { zend_op *new_opcodes = zend_accel_memdup(op_array->opcodes, sizeof(zend_op) * op_array->last); #if ZEND_USE_ABS_CONST_ADDR || ZEND_USE_ABS_JMP_ADDR @@ -568,12 +562,6 @@ static void zend_persist_op_array_ex(zend_op_array *op_array, zend_persistent_sc } ZCG(mem) = (void*)((char*)ZCG(mem) + ZEND_ALIGNED_SIZE(zend_extensions_op_array_persist(op_array, ZCG(mem)))); - -#ifdef HAVE_JIT - if (do_jit && ZCG(accel_directives).jit_buffer_size) { - zend_jit(op_array, &main_persistent_script->script); - } -#endif } static void zend_persist_op_array(zval *zv) @@ -865,6 +853,9 @@ zend_persistent_script *zend_accel_script_persist(zend_persistent_script *script zend_persist_op_array_ex(&script->script.main_op_array, script); #ifdef HAVE_JIT + if (ZCG(accel_directives).jit_buffer_size) { + zend_jit(&script->script); + } zend_jit_protect(); #endif From 378dc4c4321c5b53cccb0149cec4fe9a89c235f2 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Tue, 23 Aug 2016 14:07:21 +0800 Subject: [PATCH 021/569] Skip nops (like Zend/tests/ns_026.phpt) --- ext/opcache/jit/zend_jit.c | 13 ++++++++++--- ext/opcache/jit/zend_jit_x86.dasc | 4 ++-- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index a1b2ed92d3ac0..e5f6db443ccc1 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -284,8 +284,15 @@ static int zend_jit_op_array(zend_op_array *op_array, zend_script *script) } break; case ZEND_NOP: - if (!zend_jit_skip_handler(&dasm_state)) { - goto jit_failure; + { + uint32_t skip = 1; + while (i < end && (opline + skip)->opcode == ZEND_NOP) { + i++; + skip++; + } + if (!zend_jit_skip_handler(&dasm_state, skip)) { + goto jit_failure; + } } break; case ZEND_OP_DATA: @@ -322,7 +329,7 @@ static int zend_jit_op_array(zend_op_array *op_array, zend_script *script) break; case ZEND_JMPZ: case ZEND_JMPNZ: - if (opline != 0) { + if (i != 0) { if ((opline-1)->opcode == ZEND_IS_EQUAL || (opline-1)->opcode == ZEND_IS_NOT_EQUAL || (opline-1)->opcode == ZEND_IS_SMALLER || diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 507dcf3074800..efd2c6d13f4de 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -132,9 +132,9 @@ static int zend_jit_tail_handler(dasm_State **Dst, zend_op *opline) return 1; } -static int zend_jit_skip_handler(dasm_State **Dst) +static int zend_jit_skip_handler(dasm_State **Dst, uint32_t skip) { - | add IP, sizeof(zend_op) + | add IP, sizeof(zend_op) * skip return 1; } From f38a831590728b73208a63ad8ca4dc6c92b929fb Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Tue, 23 Aug 2016 16:01:00 +0800 Subject: [PATCH 022/569] UNREGISTER entry in shutdown --- ext/opcache/jit/zend_jit.c | 6 ++++ ext/opcache/jit/zend_jit_gdb.c | 53 ++++++++++++++++++++++------------ 2 files changed, 40 insertions(+), 19 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index e5f6db443ccc1..7fd6f1ac60e33 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -617,6 +617,12 @@ ZEND_API void zend_jit_shutdown(void) if (dasm_buf) { jit_free(dasm_buf, ((char*)dasm_end) - ((char*)dasm_buf)); } + +#ifdef HAVE_GDB + if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_GDB) { + zend_jit_gdb_unregister(); + } +#endif } #else /* HAVE_JIT */ diff --git a/ext/opcache/jit/zend_jit_gdb.c b/ext/opcache/jit/zend_jit_gdb.c index f221244977eda..f0341b945a611 100644 --- a/ext/opcache/jit/zend_jit_gdb.c +++ b/ext/opcache/jit/zend_jit_gdb.c @@ -171,7 +171,7 @@ enum { enum { GDBJIT_NOACTION, GDBJIT_REGISTER, - GDBJIT_UNREGISTYER + GDBJIT_UNREGISTER }; enum { @@ -402,13 +402,13 @@ static void zend_gdbjit_ehframe(zend_gdbjit_ctx *ctx) /* Emit DWARF EH CIE. */ DSECT(CIE, - DU32(0); /* Offset to CIE itself. */ + DU32(0); /* Offset to CIE itself. */ DB(DW_CIE_VERSION); - DSTR("zR"); /* Augmentation. */ - DUV(1); /* Code alignment factor. */ - DSV(-(int32_t)sizeof(uintptr_t)); /* Data alignment factor. */ - DB(DW_REG_RA); /* Return address register. */ - DB(1); DB(DW_EH_PE_textrel|DW_EH_PE_udata4); /* Augmentation data. */ + DSTR("zR"); /* Augmentation. */ + DUV(1); /* Code alignment factor. */ + DSV(-(int32_t)sizeof(uintptr_t)); /* Data alignment factor. */ + DB(DW_REG_RA); /* Return address register. */ + DB(1); DB(DW_EH_PE_textrel|DW_EH_PE_udata4); /* Augmentation data. */ DB(DW_CFA_def_cfa); DUV(DW_REG_SP); DUV(sizeof(uintptr_t)); DB(DW_CFA_offset|DW_REG_RA); DUV(1); DALIGNNOP(sizeof(uintptr_t)); @@ -417,9 +417,9 @@ static void zend_gdbjit_ehframe(zend_gdbjit_ctx *ctx) /* Emit DWARF EH FDE. */ DSECT(FDE, DU32((uint32_t)(p-framep)); /* Offset to CIE. */ - DU32(0); /* Machine code offset relative to .text. */ - DU32(ctx->szmcode); /* Machine code length. */ - DB(0); /* Augmentation data. */ + DU32(0); /* Machine code offset relative to .text. */ + DU32(ctx->szmcode); /* Machine code length. */ + DB(0); /* Augmentation data. */ /* Registers saved in CFRAME. */ #if defined(__i386__) DB(DW_CFA_offset|DW_REG_BP); DUV(2); @@ -470,13 +470,19 @@ static void zend_gdbjit_debugabbrev(zend_gdbjit_ctx *ctx) uint8_t *p = ctx->p; /* Abbrev #1: DW_TAG_compile_unit. */ - DUV(1); DUV(DW_TAG_compile_unit); + DUV(1); + DUV(DW_TAG_compile_unit); DB(DW_children_no); - DUV(DW_AT_name); DUV(DW_FORM_string); - DUV(DW_AT_low_pc); DUV(DW_FORM_addr); - DUV(DW_AT_high_pc); DUV(DW_FORM_addr); - DUV(DW_AT_stmt_list); DUV(DW_FORM_data4); - DB(0); DB(0); + DUV(DW_AT_name); + DUV(DW_FORM_string); + DUV(DW_AT_low_pc); + DUV(DW_FORM_addr); + DUV(DW_AT_high_pc); + DUV(DW_FORM_addr); + DUV(DW_AT_stmt_list); + DUV(DW_FORM_data4); + DB(0); + DB(0); ctx->p = p; } @@ -593,10 +599,19 @@ static int zend_jit_gdb_register(const char *name, return 1; } -static int zend_jit_gdb_unregister(const char *name, - const void *start, - size_t size) +static int zend_jit_gdb_unregister() { + zend_gdbjit_code_entry *entry, *next; + + /* TODO: release entry earlier */ + for (entry = __jit_debug_descriptor.first_entry; entry;) { + next = entry->next_entry; + __jit_debug_descriptor.relevant_entry = entry; + __jit_debug_descriptor.action_flag = GDBJIT_UNREGISTER; + __jit_debug_register_code(); + efree(entry); + entry = next; + } return 1; } From 563fdf23893e7f12ac2d60f768caef4ce98cbbce Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Tue, 23 Aug 2016 17:32:10 +0800 Subject: [PATCH 023/569] make it unique --- ext/opcache/jit/zend_jit_gdb.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ext/opcache/jit/zend_jit_gdb.c b/ext/opcache/jit/zend_jit_gdb.c index f0341b945a611..e9a76843e4d84 100644 --- a/ext/opcache/jit/zend_jit_gdb.c +++ b/ext/opcache/jit/zend_jit_gdb.c @@ -364,12 +364,14 @@ static void zend_gdbjit_symtab(zend_gdbjit_ctx *ctx) *ctx->p++ = '\0'; sym = &ctx->obj.sym[GDBJIT_SYM_FILE]; - sym->name = zend_gdbjit_strz(ctx, "JIT mcode"); + sym->name = zend_gdbjit_strz(ctx, "JIT code"); sym->sectidx = ELFSECT_IDX_ABS; sym->info = ELFSYM_TYPE_FILE|ELFSYM_BIND_LOCAL; sym = &ctx->obj.sym[GDBJIT_SYM_FUNC]; - sym->name = zend_gdbjit_strz(ctx, "JIT_FUNC"); ctx->p--; + sym->name = zend_gdbjit_strz(ctx, "JIT_FUNC_"); ctx->p--; + /* filename actually is function name */ + zend_gdbjit_strz(ctx, ctx->filename); sym->sectidx = GDBJIT_SECT_text; sym->value = 0; sym->size = ctx->szmcode; From 538489331b49c2f0b0e2ce07034525f21da5e909 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Tue, 23 Aug 2016 17:44:00 +0800 Subject: [PATCH 024/569] Seems gdb will take care of releasing --- ext/opcache/jit/zend_jit.c | 3 ++- ext/opcache/jit/zend_jit_gdb.c | 9 +++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 7fd6f1ac60e33..25307478fff92 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -440,6 +440,7 @@ static int zend_jit_op_array(zend_op_array *op_array, zend_script *script) if (name) { zend_jit_gdb_register( + op_array, ZSTR_VAL(name), handler, (char*)dasm_ptr - (char*)handler); @@ -620,7 +621,7 @@ ZEND_API void zend_jit_shutdown(void) #ifdef HAVE_GDB if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_GDB) { - zend_jit_gdb_unregister(); + zend_jit_gdb_unregister(); } #endif } diff --git a/ext/opcache/jit/zend_jit_gdb.c b/ext/opcache/jit/zend_jit_gdb.c index e9a76843e4d84..c1580bb5e2fdb 100644 --- a/ext/opcache/jit/zend_jit_gdb.c +++ b/ext/opcache/jit/zend_jit_gdb.c @@ -568,20 +568,22 @@ static void zend_gdbjit_buildobj(zend_gdbjit_ctx *ctx) { ZEND_ASSERT(ctx->objsize < sizeof(zend_gdbjit_obj)); } -static int zend_jit_gdb_register(const char *name, +static int zend_jit_gdb_register(zend_op_array *op_array, + const char *name, const void *start, size_t size) { - zend_gdbjit_ctx ctx; + zend_gdbjit_ctx ctx = {0}; zend_gdbjit_code_entry *entry; ctx.mcaddr = (uintptr_t)start; ctx.szmcode = (uint32_t)size; ctx.filename = name; + ctx.lineno = op_array->line_start; zend_gdbjit_buildobj(&ctx); - entry = emalloc(sizeof(zend_gdbjit_code_entry) + ctx.objsize); + entry = malloc(sizeof(zend_gdbjit_code_entry) + ctx.objsize); entry->symfile_addr = ((char*)entry) + sizeof(zend_gdbjit_code_entry); entry->symfile_size = ctx.objsize; @@ -611,7 +613,6 @@ static int zend_jit_gdb_unregister() __jit_debug_descriptor.relevant_entry = entry; __jit_debug_descriptor.action_flag = GDBJIT_UNREGISTER; __jit_debug_register_code(); - efree(entry); entry = next; } return 1; From 646f9cb3413a24078579c0c26c9c71cc555da058 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 23 Aug 2016 13:30:46 +0300 Subject: [PATCH 025/569] Don't build SSA and perform type inference for functions with indirect varaiable access. --- ext/opcache/jit/zend_jit.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 25307478fff92..a826c30e77102 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -24,6 +24,7 @@ #ifdef HAVE_JIT +#include "Optimizer/zend_func_info.h" #include "Optimizer/zend_ssa.h" #include "Optimizer/zend_inference.h" #include "Optimizer/zend_dump.h" @@ -215,7 +216,8 @@ static int zend_jit_op_array(zend_op_array *op_array, zend_script *script) goto jit_failure; } - if (ZEND_JIT_LEVEL >= ZEND_JIT_LEVEL_FULL) { + if ((ZEND_JIT_LEVEL >= ZEND_JIT_LEVEL_FULL) + && !(flags & ZEND_FUNC_INDIRECT_VAR_ACCESS)) { if (zend_build_ssa(&CG(arena), script, op_array, ZEND_RT_CONSTANTS | ZEND_SSA_RC_INFERENCE, &ssa, &flags) != SUCCESS) { goto jit_failure; } From be1699c3194f2bff22d80143c00c91ad30a81543 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Tue, 23 Aug 2016 18:41:14 +0800 Subject: [PATCH 026/569] gdb support (incomplete) --- ext/opcache/jit/zend_jit.c | 8 +------- ext/opcache/jit/zend_jit_gdb.c | 34 ++++++++++++++++++---------------- 2 files changed, 19 insertions(+), 23 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 25307478fff92..851e7eb67e2aa 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -436,16 +436,10 @@ static int zend_jit_op_array(zend_op_array *op_array, zend_script *script) #ifdef HAVE_GDB if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_GDB) { - zend_string *name = zend_jit_func_name(op_array); - - if (name) { - zend_jit_gdb_register( + zend_jit_gdb_register( op_array, - ZSTR_VAL(name), handler, (char*)dasm_ptr - (char*)handler); - zend_string_release(name); - } } #endif diff --git a/ext/opcache/jit/zend_jit_gdb.c b/ext/opcache/jit/zend_jit_gdb.c index c1580bb5e2fdb..bfa21e6bc50f5 100644 --- a/ext/opcache/jit/zend_jit_gdb.c +++ b/ext/opcache/jit/zend_jit_gdb.c @@ -266,17 +266,18 @@ typedef struct _zend_gdbjit_descriptor { /* Context for generating the ELF object for the GDB JIT API. */ typedef struct _zend_gdbjit_ctx { - uint8_t *p; /* Pointer to next address in obj.space. */ - uint8_t *startp; /* Pointer to start address in obj.space. */ - void *T; /* Generate symbols for this trace. */ - uintptr_t mcaddr; /* Machine code address. */ - uint32_t szmcode; /* Size of machine code. */ - uint32_t spadjp; /* Stack adjustment for parent trace or interpreter. */ - uint32_t spadj; /* Stack adjustment for trace itself. */ - int32_t lineno; /* Starting line number. */ - const char *filename; /* Starting file name. */ - size_t objsize; /* Final size of ELF object. */ - zend_gdbjit_obj obj; /* In-memory ELF object. */ + zend_op_array *op_array; /* Pointer to op_array */ + uint8_t *p; /* Pointer to next address in obj.space. */ + uint8_t *startp; /* Pointer to start address in obj.space. */ + void *T; /* Generate symbols for this trace. */ + uintptr_t mcaddr; /* Machine code address. */ + uint32_t szmcode; /* Size of machine code. */ + uint32_t spadjp; /* Stack adjustment for parent trace or interpreter. */ + uint32_t spadj; /* Stack adjustment for trace itself. */ + int32_t lineno; /* Starting line number. */ + const char *filename; /* Starting file name. */ + size_t objsize; /* Final size of ELF object. */ + zend_gdbjit_obj obj; /* In-memory ELF object. */ } zend_gdbjit_ctx; zend_gdbjit_descriptor __jit_debug_descriptor = { @@ -369,9 +370,10 @@ static void zend_gdbjit_symtab(zend_gdbjit_ctx *ctx) sym->info = ELFSYM_TYPE_FILE|ELFSYM_BIND_LOCAL; sym = &ctx->obj.sym[GDBJIT_SYM_FUNC]; - sym->name = zend_gdbjit_strz(ctx, "JIT_FUNC_"); ctx->p--; - /* filename actually is function name */ - zend_gdbjit_strz(ctx, ctx->filename); + sym->name = zend_gdbjit_strz(ctx, "JIT_FUNC_"); + ctx->p--; + zend_gdbjit_strz(ctx, + ctx->op_array->function_name? ZSTR_VAL(ctx->op_array->function_name) : "main"); sym->sectidx = GDBJIT_SECT_text; sym->value = 0; sym->size = ctx->szmcode; @@ -569,16 +571,16 @@ static void zend_gdbjit_buildobj(zend_gdbjit_ctx *ctx) { } static int zend_jit_gdb_register(zend_op_array *op_array, - const char *name, const void *start, size_t size) { zend_gdbjit_ctx ctx = {0}; zend_gdbjit_code_entry *entry; + ctx.op_array = op_array; ctx.mcaddr = (uintptr_t)start; ctx.szmcode = (uint32_t)size; - ctx.filename = name; + ctx.filename = ZSTR_VAL(op_array->filename); ctx.lineno = op_array->line_start; zend_gdbjit_buildobj(&ctx); From e5bddbce1da3728c98542b6ea65a8bba1fdf0a4f Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 23 Aug 2016 14:07:13 +0300 Subject: [PATCH 027/569] Revert "Fixed segfault of null pointer def in type inference(script)" This reverts commit 9cf2b1df7309265a673b162cf74833b430ec2a10. --- ext/opcache/jit/zend_jit.c | 42 +++----------------------------------- ext/opcache/jit/zend_jit.h | 4 ++-- ext/opcache/zend_persist.c | 15 +++++++++++--- 3 files changed, 17 insertions(+), 44 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 6f737b29cc229..b2d5a4729c02c 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -19,7 +19,6 @@ #include #include "Zend/zend_execute.h" #include "zend_smart_str.h" -#include "zend_shared_alloc.h" #include "jit/zend_jit.h" #ifdef HAVE_JIT @@ -177,7 +176,7 @@ static void jit_free(void *p, size_t size) #endif } -static int zend_jit_op_array(zend_op_array *op_array, zend_script *script) +ZEND_API int zend_jit(zend_op_array *op_array, zend_script *script) { uint32_t flags; zend_ssa ssa; @@ -187,12 +186,10 @@ static int zend_jit_op_array(zend_op_array *op_array, zend_script *script) dasm_State* dasm_state = NULL; void *handler; - if (zend_shared_alloc_get_xlat_entry(op_array->opcodes)) { - return SUCCESS; + if (!dasm_buf) { + return FAILURE; } - zend_shared_alloc_register_xlat_entry(op_array->opcodes, op_array->opcodes); - checkpoint = zend_arena_checkpoint(CG(arena)); /* Build SSA */ @@ -484,39 +481,6 @@ static int zend_jit_op_array(zend_op_array *op_array, zend_script *script) return FAILURE; } -static int zend_jit_func_table(zval *zv, void *arg) { - zend_op_array *op_array = (zend_op_array*)Z_PTR_P(zv); - zend_script *script = (zend_script *)arg; - - zend_jit_op_array(op_array, script); - - return ZEND_HASH_APPLY_KEEP; -} - -static int zend_jit_class_table(zval *zv, void *arg) { - zend_class_entry *ce = (zend_class_entry *)Z_PTR_P(zv); - zend_script *script = (zend_script *)arg; - - zend_hash_apply_with_argument(&ce->function_table, zend_jit_func_table, script); - - return ZEND_HASH_APPLY_KEEP; -} - -ZEND_API int zend_jit(zend_script *script) { - if (!dasm_buf) { - return FAILURE; - } - - /* xlat table is initialized already */ - zend_shared_alloc_clear_xlat_table(); - - zend_hash_apply_with_argument(&script->class_table, zend_jit_class_table, script); - zend_hash_apply_with_argument(&script->function_table, zend_jit_func_table, script); - zend_jit_op_array(&script->main_op_array, script); - - return SUCCESS; -} - ZEND_API void zend_jit_unprotect(void) { #ifdef HAVE_MPROTECT diff --git a/ext/opcache/jit/zend_jit.h b/ext/opcache/jit/zend_jit.h index 34492e7fffb09..1369211da5c7d 100644 --- a/ext/opcache/jit/zend_jit.h +++ b/ext/opcache/jit/zend_jit.h @@ -29,7 +29,7 @@ #define ZEND_JIT_DEBUG_OPROFILE (1<<6) -ZEND_API int zend_jit(zend_script *script); +ZEND_API int zend_jit(zend_op_array *op_array, zend_script *script); ZEND_API void zend_jit_unprotect(void); ZEND_API void zend_jit_protect(void); ZEND_API int zend_jit_startup(size_t size); @@ -43,4 +43,4 @@ ZEND_API void zend_jit_shutdown(void); * c-basic-offset: 4 * indent-tabs-mode: t * End: - */ + */ \ No newline at end of file diff --git a/ext/opcache/zend_persist.c b/ext/opcache/zend_persist.c index 3dad282edc0e5..f47cabaf1336f 100644 --- a/ext/opcache/zend_persist.c +++ b/ext/opcache/zend_persist.c @@ -336,6 +336,9 @@ static void zend_persist_op_array_ex(zend_op_array *op_array, zend_persistent_sc int already_stored = 0; zend_op *persist_ptr; zval *orig_literals = NULL; +#ifdef HAVE_JIT + int do_jit = 1; +#endif if (op_array->type != ZEND_USER_FUNCTION) { return; @@ -402,6 +405,9 @@ static void zend_persist_op_array_ex(zend_op_array *op_array, zend_persistent_sc persist_ptr = zend_shared_alloc_get_xlat_entry(op_array->opcodes); ZEND_ASSERT(persist_ptr != NULL); op_array->opcodes = persist_ptr; +#ifdef HAVE_JIT + do_jit = 0; +#endif } else { zend_op *new_opcodes = zend_accel_memdup(op_array->opcodes, sizeof(zend_op) * op_array->last); #if ZEND_USE_ABS_CONST_ADDR || ZEND_USE_ABS_JMP_ADDR @@ -562,6 +568,12 @@ static void zend_persist_op_array_ex(zend_op_array *op_array, zend_persistent_sc } ZCG(mem) = (void*)((char*)ZCG(mem) + ZEND_ALIGNED_SIZE(zend_extensions_op_array_persist(op_array, ZCG(mem)))); + +#ifdef HAVE_JIT + if (do_jit && ZCG(accel_directives).jit_buffer_size) { + zend_jit(op_array, &main_persistent_script->script); + } +#endif } static void zend_persist_op_array(zval *zv) @@ -853,9 +865,6 @@ zend_persistent_script *zend_accel_script_persist(zend_persistent_script *script zend_persist_op_array_ex(&script->script.main_op_array, script); #ifdef HAVE_JIT - if (ZCG(accel_directives).jit_buffer_size) { - zend_jit(&script->script); - } zend_jit_protect(); #endif From 1b4e0210d58615fee960264565597cdf9c5e3521 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 23 Aug 2016 16:22:26 +0300 Subject: [PATCH 028/569] Fixd uninitialized variable access --- ext/opcache/jit/zend_jit.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index b2d5a4729c02c..be83bf8e2f61b 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -178,7 +178,7 @@ static void jit_free(void *p, size_t size) ZEND_API int zend_jit(zend_op_array *op_array, zend_script *script) { - uint32_t flags; + uint32_t flags = 0; zend_ssa ssa; void *checkpoint; int b, i, end; From b71eaf48f11f2514c4ea79ca5c4bcc2d9acb5a2e Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Wed, 24 Aug 2016 00:09:10 +0800 Subject: [PATCH 029/569] Seems works now --- ext/opcache/jit/zend_jit_gdb.c | 51 +++++++++++++++++++++++++++++++--- 1 file changed, 47 insertions(+), 4 deletions(-) diff --git a/ext/opcache/jit/zend_jit_gdb.c b/ext/opcache/jit/zend_jit_gdb.c index bfa21e6bc50f5..0ed2eb856f350 100644 --- a/ext/opcache/jit/zend_jit_gdb.c +++ b/ext/opcache/jit/zend_jit_gdb.c @@ -24,6 +24,29 @@ # undef ELF64 #endif +#if defined(__x86_64) +#define CFRAME_OFS_ERRF (15*4) +#define CFRAME_OFS_NRES (14*4) +#define CFRAME_OFS_PREV (13*4) +#define CFRAME_OFS_L (12*4) +#define CFRAME_OFS_PC (6*4) +#define CFRAME_OFS_MULTRES (5*4) +#define CFRAME_SIZE (12*4) +#define CFRAME_SHIFT_MULTRES 0 +#elif defined(__i386__) +#define CFRAME_OFS_PREV (4*8) +#define CFRAME_OFS_PC (7*4) +#define CFRAME_OFS_L (6*4) +#define CFRAME_OFS_ERRF (5*4) +#define CFRAME_OFS_NRES (4*4) +#define CFRAME_OFS_MULTRES (1*4) +#define CFRAME_SIZE (10*8) +#define CFRAME_SIZE_JIT (CFRAME_SIZE + 16) +#define CFRAME_SHIFT_MULTRES 0 +#endif + +#define CFRAME_SIZE_JIT CFRAME_SIZE + typedef struct _zend_elf_header { uint8_t emagic[4]; uint8_t eclass; @@ -372,6 +395,12 @@ static void zend_gdbjit_symtab(zend_gdbjit_ctx *ctx) sym = &ctx->obj.sym[GDBJIT_SYM_FUNC]; sym->name = zend_gdbjit_strz(ctx, "JIT_FUNC_"); ctx->p--; + if (ctx->op_array->scope) { + zend_gdbjit_strz(ctx, ZSTR_VAL(ctx->op_array->scope->name)); + ctx->p--; + zend_gdbjit_strz(ctx, "::"); + ctx->p--; + } zend_gdbjit_strz(ctx, ctx->op_array->function_name? ZSTR_VAL(ctx->op_array->function_name) : "main"); sym->sectidx = GDBJIT_SECT_text; @@ -580,6 +609,12 @@ static int zend_jit_gdb_register(zend_op_array *op_array, ctx.op_array = op_array; ctx.mcaddr = (uintptr_t)start; ctx.szmcode = (uint32_t)size; +#if defined(__x86_64__) + ctx.spadjp = CFRAME_SIZE_JIT + 8; +#elif defined(__i386__) + ctx.spadjp = CFRAME_SIZE_JIT + 12; +#endif + ctx.spadj = CFRAME_SIZE_JIT; ctx.filename = ZSTR_VAL(op_array->filename); ctx.lineno = op_array->line_start; @@ -607,16 +642,24 @@ static int zend_jit_gdb_register(zend_op_array *op_array, static int zend_jit_gdb_unregister() { - zend_gdbjit_code_entry *entry, *next; + zend_gdbjit_code_entry *entry; /* TODO: release entry earlier */ - for (entry = __jit_debug_descriptor.first_entry; entry;) { - next = entry->next_entry; + while ((entry = __jit_debug_descriptor.first_entry)) { + if (entry->prev_entry) { + entry->prev_entry->next_entry = entry->next_entry; + } else { + __jit_debug_descriptor.first_entry = entry->next_entry; + } + if (entry->next_entry) { + entry->next_entry->prev_entry = entry->prev_entry; + } __jit_debug_descriptor.relevant_entry = entry; __jit_debug_descriptor.action_flag = GDBJIT_UNREGISTER; __jit_debug_register_code(); - entry = next; + free(entry); } + return 1; } From 82a452caaf4175ac64ff4c77b27767eaf0aa046f Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Wed, 24 Aug 2016 11:00:41 +0800 Subject: [PATCH 030/569] Only keep necessary macros --- ext/opcache/jit/zend_jit_gdb.c | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/ext/opcache/jit/zend_jit_gdb.c b/ext/opcache/jit/zend_jit_gdb.c index 0ed2eb856f350..ebf9970fdf941 100644 --- a/ext/opcache/jit/zend_jit_gdb.c +++ b/ext/opcache/jit/zend_jit_gdb.c @@ -13,6 +13,7 @@ | license@php.net so we can mail you a copy immediately. | +----------------------------------------------------------------------+ | Authors: Dmitry Stogov | + | Xinchen Hui | +----------------------------------------------------------------------+ */ @@ -25,28 +26,15 @@ #endif #if defined(__x86_64) -#define CFRAME_OFS_ERRF (15*4) -#define CFRAME_OFS_NRES (14*4) -#define CFRAME_OFS_PREV (13*4) -#define CFRAME_OFS_L (12*4) -#define CFRAME_OFS_PC (6*4) -#define CFRAME_OFS_MULTRES (5*4) #define CFRAME_SIZE (12*4) -#define CFRAME_SHIFT_MULTRES 0 +#define CFRAME_SIZE_JIT CFRAME_SIZE #elif defined(__i386__) -#define CFRAME_OFS_PREV (4*8) -#define CFRAME_OFS_PC (7*4) -#define CFRAME_OFS_L (6*4) -#define CFRAME_OFS_ERRF (5*4) -#define CFRAME_OFS_NRES (4*4) -#define CFRAME_OFS_MULTRES (1*4) #define CFRAME_SIZE (10*8) #define CFRAME_SIZE_JIT (CFRAME_SIZE + 16) -#define CFRAME_SHIFT_MULTRES 0 +#else +#error "Unsupported target architecture" #endif -#define CFRAME_SIZE_JIT CFRAME_SIZE - typedef struct _zend_elf_header { uint8_t emagic[4]; uint8_t eclass; @@ -644,7 +632,6 @@ static int zend_jit_gdb_unregister() { zend_gdbjit_code_entry *entry; - /* TODO: release entry earlier */ while ((entry = __jit_debug_descriptor.first_entry)) { if (entry->prev_entry) { entry->prev_entry->next_entry = entry->next_entry; From 6f7f97dbb07ad0935c9093dd9e1825f6f029c89a Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Wed, 24 Aug 2016 11:03:50 +0800 Subject: [PATCH 031/569] Free buf later --- ext/opcache/jit/zend_jit.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index be83bf8e2f61b..e233d89cfc400 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -575,15 +575,16 @@ ZEND_API void zend_jit_shutdown(void) zend_jit_oprofile_shutdown(); } #endif - if (dasm_buf) { - jit_free(dasm_buf, ((char*)dasm_end) - ((char*)dasm_buf)); - } #ifdef HAVE_GDB if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_GDB) { zend_jit_gdb_unregister(); } #endif + + if (dasm_buf) { + jit_free(dasm_buf, ((char*)dasm_end) - ((char*)dasm_buf)); + } } #else /* HAVE_JIT */ From aa695c7ec1e4620a59e0b1a72be4de5dfa580234 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Wed, 24 Aug 2016 12:19:42 +0800 Subject: [PATCH 032/569] Keep the order (FIFO) --- ext/opcache/jit/zend_jit_gdb.c | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/ext/opcache/jit/zend_jit_gdb.c b/ext/opcache/jit/zend_jit_gdb.c index ebf9970fdf941..04212444c53ea 100644 --- a/ext/opcache/jit/zend_jit_gdb.c +++ b/ext/opcache/jit/zend_jit_gdb.c @@ -591,7 +591,7 @@ static int zend_jit_gdb_register(zend_op_array *op_array, const void *start, size_t size) { - zend_gdbjit_ctx ctx = {0}; + zend_gdbjit_ctx ctx; zend_gdbjit_code_entry *entry; ctx.op_array = op_array; @@ -609,17 +609,23 @@ static int zend_jit_gdb_register(zend_op_array *op_array, zend_gdbjit_buildobj(&ctx); entry = malloc(sizeof(zend_gdbjit_code_entry) + ctx.objsize); + if (entry == NULL) { + return 0; + } + entry->symfile_addr = ((char*)entry) + sizeof(zend_gdbjit_code_entry); entry->symfile_size = ctx.objsize; memcpy((char *)entry->symfile_addr, &ctx.obj, ctx.objsize); - entry->prev_entry = NULL; - entry->next_entry = __jit_debug_descriptor.first_entry; - if (__jit_debug_descriptor.first_entry) { - __jit_debug_descriptor.first_entry->prev_entry = entry; + entry->next_entry = NULL; + entry->prev_entry = __jit_debug_descriptor.relevant_entry; + + if (entry->prev_entry) { + entry->prev_entry->next_entry = entry; + } else { + __jit_debug_descriptor.first_entry = entry; } - __jit_debug_descriptor.first_entry = entry; __jit_debug_descriptor.relevant_entry = entry; __jit_debug_descriptor.action_flag = GDBJIT_REGISTER; @@ -632,18 +638,18 @@ static int zend_jit_gdb_unregister() { zend_gdbjit_code_entry *entry; - while ((entry = __jit_debug_descriptor.first_entry)) { + while ((entry = __jit_debug_descriptor.relevant_entry)) { if (entry->prev_entry) { - entry->prev_entry->next_entry = entry->next_entry; + entry->prev_entry->next_entry = NULL; } else { - __jit_debug_descriptor.first_entry = entry->next_entry; - } - if (entry->next_entry) { - entry->next_entry->prev_entry = entry->prev_entry; + __jit_debug_descriptor.first_entry = NULL; } - __jit_debug_descriptor.relevant_entry = entry; + __jit_debug_descriptor.action_flag = GDBJIT_UNREGISTER; __jit_debug_register_code(); + + __jit_debug_descriptor.relevant_entry = entry->prev_entry; + free(entry); } From c5b3b0d26c06681a5c07b83be6964c56eedef9ef Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Wed, 24 Aug 2016 14:20:25 +0800 Subject: [PATCH 033/569] cleanup --- ext/opcache/jit/zend_jit.c | 6 ++++-- ext/opcache/jit/zend_jit_gdb.c | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index e233d89cfc400..9cf165eb18b52 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -511,10 +511,12 @@ ZEND_API int zend_jit_startup(size_t size) /* Round up to the page size, which should be a power of two. */ page_size = jit_page_size(); + if (!page_size || (page_size & (page_size - 1))) { abort(); } - size = (size + page_size - 1) & ~(page_size - 1); + + size = ZEND_MM_ALIGNED_SIZE_EX(size, page_size); buf = jit_alloc(size, shared); @@ -523,7 +525,7 @@ ZEND_API int zend_jit_startup(size_t size) } dasm_buf = dasm_ptr = buf; - dasm_end = (void*)(((char*)dasm_buf)+size); + dasm_end = (void*)(((char*)dasm_buf) + size); #ifdef HAVE_DISASM if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_ASM) { diff --git a/ext/opcache/jit/zend_jit_gdb.c b/ext/opcache/jit/zend_jit_gdb.c index 04212444c53ea..f189a90ff632e 100644 --- a/ext/opcache/jit/zend_jit_gdb.c +++ b/ext/opcache/jit/zend_jit_gdb.c @@ -638,6 +638,7 @@ static int zend_jit_gdb_unregister() { zend_gdbjit_code_entry *entry; + __jit_debug_descriptor.action_flag = GDBJIT_UNREGISTER; while ((entry = __jit_debug_descriptor.relevant_entry)) { if (entry->prev_entry) { entry->prev_entry->next_entry = NULL; @@ -645,7 +646,6 @@ static int zend_jit_gdb_unregister() __jit_debug_descriptor.first_entry = NULL; } - __jit_debug_descriptor.action_flag = GDBJIT_UNREGISTER; __jit_debug_register_code(); __jit_debug_descriptor.relevant_entry = entry->prev_entry; From 3709a5dc3df890493ce2fb4744c2aa0df20c2d20 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 24 Aug 2016 11:01:08 +0300 Subject: [PATCH 034/569] Disable JIT memory protection if running with GDB (opcache.jit_debug=0x10) --- ext/opcache/jit/zend_jit.c | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 9cf165eb18b52..603d1018aaefd 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -139,25 +139,26 @@ static void *jit_alloc(size_t size, int shared) return VirtualAlloc(0, size, MEM_COMMIT | MEM_RESERVE, PAGE_EXECUTE_READWRITE); #else void *p; + int prot; + +# ifdef HAVE_MPROTECT + if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_GDB) { + prot = PROT_EXEC | PROT_READ | PROT_WRITE; + } else { + prot = PROT_NONE; + } +# else + prot = PROT_EXEC | PROT_READ | PROT_WRITE; +# endif # ifdef MAP_HUGETLB - p = mmap(NULL, size, -# ifdef HAVE_MPROTECT - PROT_NONE, -# else - PROT_EXEC | PROT_READ | PROT_WRITE, -# endif + p = mmap(NULL, size, prot, (shared ? MAP_SHARED : MAP_PRIVATE) | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0); if (p != MAP_FAILED) { return (void*)p; } # endif - p = mmap(NULL, size, -# ifdef HAVE_MPROTECT - PROT_NONE, -# else - PROT_EXEC | PROT_READ | PROT_WRITE, -# endif + p = mmap(NULL, size, prot, (shared ? MAP_SHARED : MAP_PRIVATE) | MAP_ANONYMOUS, -1, 0); if (p == MAP_FAILED) { return NULL; @@ -484,14 +485,18 @@ ZEND_API int zend_jit(zend_op_array *op_array, zend_script *script) ZEND_API void zend_jit_unprotect(void) { #ifdef HAVE_MPROTECT - mprotect(dasm_buf, ((char*)dasm_end) - ((char*)dasm_buf), PROT_READ | PROT_WRITE); + if (!(ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_GDB)) { + mprotect(dasm_buf, ((char*)dasm_end) - ((char*)dasm_buf), PROT_READ | PROT_WRITE); + } #endif } ZEND_API void zend_jit_protect(void) { #ifdef HAVE_MPROTECT - mprotect(dasm_buf, ((char*)dasm_end) - ((char*)dasm_buf), PROT_READ | PROT_EXEC); + if (!(ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_GDB)) { + mprotect(dasm_buf, ((char*)dasm_end) - ((char*)dasm_buf), PROT_READ | PROT_EXEC); + } #endif } From dc0b393b198b827c1aca647db2da4dbe14270305 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Wed, 24 Aug 2016 16:16:56 +0800 Subject: [PATCH 035/569] Make names consistent --- ext/opcache/jit/zend_jit.c | 17 ++++++++++------- ext/opcache/jit/zend_jit_gdb.c | 17 +++++------------ 2 files changed, 15 insertions(+), 19 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 603d1018aaefd..a531ee1bfa591 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -82,9 +82,7 @@ static zend_string *zend_jit_func_name(zend_op_array *op_array) return buf.s; } } else if (op_array->filename) { - smart_str_appends(&buf, "JIT$"); - smart_str_appendl(&buf, ZSTR_VAL(op_array->filename), ZSTR_LEN(op_array->filename)); - smart_str_0(&buf); + smart_str_appends(&buf, "JIT$(main)"); return buf.s; } else { return NULL; @@ -436,10 +434,15 @@ ZEND_API int zend_jit(zend_op_array *op_array, zend_script *script) #ifdef HAVE_GDB if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_GDB) { - zend_jit_gdb_register( - op_array, - handler, - (char*)dasm_ptr - (char*)handler); + zend_string *name = zend_jit_func_name(op_array); + if (name) { + zend_jit_gdb_register( + ZSTR_VAL(name), + op_array, + handler, + (char*)dasm_ptr - (char*)handler); + zend_string_release(name); + } } #endif diff --git a/ext/opcache/jit/zend_jit_gdb.c b/ext/opcache/jit/zend_jit_gdb.c index f189a90ff632e..e9043c26c975c 100644 --- a/ext/opcache/jit/zend_jit_gdb.c +++ b/ext/opcache/jit/zend_jit_gdb.c @@ -280,12 +280,12 @@ typedef struct _zend_gdbjit_ctx { zend_op_array *op_array; /* Pointer to op_array */ uint8_t *p; /* Pointer to next address in obj.space. */ uint8_t *startp; /* Pointer to start address in obj.space. */ - void *T; /* Generate symbols for this trace. */ uintptr_t mcaddr; /* Machine code address. */ uint32_t szmcode; /* Size of machine code. */ uint32_t spadjp; /* Stack adjustment for parent trace or interpreter. */ uint32_t spadj; /* Stack adjustment for trace itself. */ int32_t lineno; /* Starting line number. */ + const char *name; /* JIT function name */ const char *filename; /* Starting file name. */ size_t objsize; /* Final size of ELF object. */ zend_gdbjit_obj obj; /* In-memory ELF object. */ @@ -381,16 +381,7 @@ static void zend_gdbjit_symtab(zend_gdbjit_ctx *ctx) sym->info = ELFSYM_TYPE_FILE|ELFSYM_BIND_LOCAL; sym = &ctx->obj.sym[GDBJIT_SYM_FUNC]; - sym->name = zend_gdbjit_strz(ctx, "JIT_FUNC_"); - ctx->p--; - if (ctx->op_array->scope) { - zend_gdbjit_strz(ctx, ZSTR_VAL(ctx->op_array->scope->name)); - ctx->p--; - zend_gdbjit_strz(ctx, "::"); - ctx->p--; - } - zend_gdbjit_strz(ctx, - ctx->op_array->function_name? ZSTR_VAL(ctx->op_array->function_name) : "main"); + sym->name = zend_gdbjit_strz(ctx, ctx->name); sym->sectidx = GDBJIT_SECT_text; sym->value = 0; sym->size = ctx->szmcode; @@ -587,7 +578,8 @@ static void zend_gdbjit_buildobj(zend_gdbjit_ctx *ctx) { ZEND_ASSERT(ctx->objsize < sizeof(zend_gdbjit_obj)); } -static int zend_jit_gdb_register(zend_op_array *op_array, +static int zend_jit_gdb_register(const char *name, + zend_op_array *op_array, const void *start, size_t size) { @@ -603,6 +595,7 @@ static int zend_jit_gdb_register(zend_op_array *op_array, ctx.spadjp = CFRAME_SIZE_JIT + 12; #endif ctx.spadj = CFRAME_SIZE_JIT; + ctx.name = name; ctx.filename = ZSTR_VAL(op_array->filename); ctx.lineno = op_array->line_start; From 3f49defc4bc1ee7c6410e220d1da374c536666da Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Wed, 24 Aug 2016 17:03:30 +0800 Subject: [PATCH 036/569] Added zend_gdb.(c|h) to make gdb works with JIT debug info --- configure.in | 2 +- ext/opcache/jit/zend_jit_gdb.c | 35 ++++------------------------------ 2 files changed, 5 insertions(+), 32 deletions(-) diff --git a/configure.in b/configure.in index de68932ca1fa3..3de605b7648aa 100644 --- a/configure.in +++ b/configure.in @@ -1513,7 +1513,7 @@ PHP_ADD_SOURCES(Zend, \ zend_iterators.c zend_interfaces.c zend_exceptions.c zend_strtod.c zend_gc.c \ zend_closures.c zend_float.c zend_string.c zend_signal.c zend_generators.c \ zend_virtual_cwd.c zend_ast.c zend_objects.c zend_object_handlers.c zend_objects_API.c \ - zend_default_classes.c zend_inheritance.c zend_smart_str.c, \ + zend_default_classes.c zend_inheritance.c zend_smart_str.c zend_gdb.c, \ -DZEND_ENABLE_STATIC_TSRMLS_CACHE=1) dnl Selectively disable optimization due to high RAM usage during diff --git a/ext/opcache/jit/zend_jit_gdb.c b/ext/opcache/jit/zend_jit_gdb.c index e9043c26c975c..cd0587fe1afd7 100644 --- a/ext/opcache/jit/zend_jit_gdb.c +++ b/ext/opcache/jit/zend_jit_gdb.c @@ -19,6 +19,8 @@ #define HAVE_GDB +#include "zend_gdb.h" + #if SIZEOF_SIZE_T == 8 # define ELF64 #else @@ -179,12 +181,6 @@ enum { #endif }; -enum { - GDBJIT_NOACTION, - GDBJIT_REGISTER, - GDBJIT_UNREGISTER -}; - enum { GDBJIT_SECT_NULL, GDBJIT_SECT_text, @@ -261,20 +257,6 @@ static const zend_elf_header zend_elfhdr_template = { .shstridx = GDBJIT_SECT_shstrtab }; -typedef struct _zend_gdbjit_code_entry { - struct _zend_gdbjit_code_entry *next_entry; - struct _zend_gdbjit_code_entry *prev_entry; - const char *symfile_addr; - uint64_t symfile_size; -} zend_gdbjit_code_entry; - -typedef struct _zend_gdbjit_descriptor { - uint32_t version; - uint32_t action_flag; - struct _zend_gdbjit_code_entry *relevant_entry; - struct _zend_gdbjit_code_entry *first_entry; -} zend_gdbjit_descriptor; - /* Context for generating the ELF object for the GDB JIT API. */ typedef struct _zend_gdbjit_ctx { zend_op_array *op_array; /* Pointer to op_array */ @@ -291,15 +273,6 @@ typedef struct _zend_gdbjit_ctx { zend_gdbjit_obj obj; /* In-memory ELF object. */ } zend_gdbjit_ctx; -zend_gdbjit_descriptor __jit_debug_descriptor = { - 1, GDBJIT_NOACTION, NULL, NULL -}; - -zend_never_inline void __jit_debug_register_code() -{ - __asm__ __volatile__(""); -} - /* Add a zero-terminated string */ static uint32_t zend_gdbjit_strz(zend_gdbjit_ctx *ctx, const char *str) { @@ -621,7 +594,7 @@ static int zend_jit_gdb_register(const char *name, } __jit_debug_descriptor.relevant_entry = entry; - __jit_debug_descriptor.action_flag = GDBJIT_REGISTER; + __jit_debug_descriptor.action_flag = ZEND_GDBJIT_REGISTER; __jit_debug_register_code(); return 1; @@ -631,7 +604,7 @@ static int zend_jit_gdb_unregister() { zend_gdbjit_code_entry *entry; - __jit_debug_descriptor.action_flag = GDBJIT_UNREGISTER; + __jit_debug_descriptor.action_flag = ZEND_GDBJIT_UNREGISTER; while ((entry = __jit_debug_descriptor.relevant_entry)) { if (entry->prev_entry) { entry->prev_entry->next_entry = NULL; From 9e9d0414d34eb0eb71363d32bdefb84c357b59c9 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Wed, 24 Aug 2016 17:08:01 +0800 Subject: [PATCH 037/569] Revert "Added zend_gdb.(c|h) to make gdb works with JIT debug info" This reverts commit 3f49defc4bc1ee7c6410e220d1da374c536666da. --- configure.in | 2 +- ext/opcache/jit/zend_jit_gdb.c | 35 ++++++++++++++++++++++++++++++---- 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/configure.in b/configure.in index 3de605b7648aa..de68932ca1fa3 100644 --- a/configure.in +++ b/configure.in @@ -1513,7 +1513,7 @@ PHP_ADD_SOURCES(Zend, \ zend_iterators.c zend_interfaces.c zend_exceptions.c zend_strtod.c zend_gc.c \ zend_closures.c zend_float.c zend_string.c zend_signal.c zend_generators.c \ zend_virtual_cwd.c zend_ast.c zend_objects.c zend_object_handlers.c zend_objects_API.c \ - zend_default_classes.c zend_inheritance.c zend_smart_str.c zend_gdb.c, \ + zend_default_classes.c zend_inheritance.c zend_smart_str.c, \ -DZEND_ENABLE_STATIC_TSRMLS_CACHE=1) dnl Selectively disable optimization due to high RAM usage during diff --git a/ext/opcache/jit/zend_jit_gdb.c b/ext/opcache/jit/zend_jit_gdb.c index cd0587fe1afd7..e9043c26c975c 100644 --- a/ext/opcache/jit/zend_jit_gdb.c +++ b/ext/opcache/jit/zend_jit_gdb.c @@ -19,8 +19,6 @@ #define HAVE_GDB -#include "zend_gdb.h" - #if SIZEOF_SIZE_T == 8 # define ELF64 #else @@ -181,6 +179,12 @@ enum { #endif }; +enum { + GDBJIT_NOACTION, + GDBJIT_REGISTER, + GDBJIT_UNREGISTER +}; + enum { GDBJIT_SECT_NULL, GDBJIT_SECT_text, @@ -257,6 +261,20 @@ static const zend_elf_header zend_elfhdr_template = { .shstridx = GDBJIT_SECT_shstrtab }; +typedef struct _zend_gdbjit_code_entry { + struct _zend_gdbjit_code_entry *next_entry; + struct _zend_gdbjit_code_entry *prev_entry; + const char *symfile_addr; + uint64_t symfile_size; +} zend_gdbjit_code_entry; + +typedef struct _zend_gdbjit_descriptor { + uint32_t version; + uint32_t action_flag; + struct _zend_gdbjit_code_entry *relevant_entry; + struct _zend_gdbjit_code_entry *first_entry; +} zend_gdbjit_descriptor; + /* Context for generating the ELF object for the GDB JIT API. */ typedef struct _zend_gdbjit_ctx { zend_op_array *op_array; /* Pointer to op_array */ @@ -273,6 +291,15 @@ typedef struct _zend_gdbjit_ctx { zend_gdbjit_obj obj; /* In-memory ELF object. */ } zend_gdbjit_ctx; +zend_gdbjit_descriptor __jit_debug_descriptor = { + 1, GDBJIT_NOACTION, NULL, NULL +}; + +zend_never_inline void __jit_debug_register_code() +{ + __asm__ __volatile__(""); +} + /* Add a zero-terminated string */ static uint32_t zend_gdbjit_strz(zend_gdbjit_ctx *ctx, const char *str) { @@ -594,7 +621,7 @@ static int zend_jit_gdb_register(const char *name, } __jit_debug_descriptor.relevant_entry = entry; - __jit_debug_descriptor.action_flag = ZEND_GDBJIT_REGISTER; + __jit_debug_descriptor.action_flag = GDBJIT_REGISTER; __jit_debug_register_code(); return 1; @@ -604,7 +631,7 @@ static int zend_jit_gdb_unregister() { zend_gdbjit_code_entry *entry; - __jit_debug_descriptor.action_flag = ZEND_GDBJIT_UNREGISTER; + __jit_debug_descriptor.action_flag = GDBJIT_UNREGISTER; while ((entry = __jit_debug_descriptor.relevant_entry)) { if (entry->prev_entry) { entry->prev_entry->next_entry = NULL; From ddbbcb59a3635bfb2ced31febf0592e468fe1fb9 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Wed, 24 Aug 2016 17:08:26 +0800 Subject: [PATCH 038/569] Added zend_gdb.(c|h) to make gdb works with JIT debug info --- Zend/zend_gdb.c | 38 ++++++++++++++++++++++++ Zend/zend_gdb.h | 54 ++++++++++++++++++++++++++++++++++ configure.in | 2 +- ext/opcache/jit/zend_jit_gdb.c | 35 +++------------------- 4 files changed, 97 insertions(+), 32 deletions(-) create mode 100644 Zend/zend_gdb.c create mode 100644 Zend/zend_gdb.h diff --git a/Zend/zend_gdb.c b/Zend/zend_gdb.c new file mode 100644 index 0000000000000..b630c1bd8bc4e --- /dev/null +++ b/Zend/zend_gdb.c @@ -0,0 +1,38 @@ +/* + +----------------------------------------------------------------------+ + | Zend JIT | + +----------------------------------------------------------------------+ + | Copyright (c) 1998-2016 The PHP Group | + +----------------------------------------------------------------------+ + | This source file is subject to version 3.01 of the PHP license, | + | that is bundled with this package in the file LICENSE, and is | + | available through the world-wide-web at the following url: | + | http://www.php.net/license/3_01.txt | + | If you did not receive a copy of the PHP license and are unable to | + | obtain it through the world-wide-web, please send a note to | + | license@php.net so we can mail you a copy immediately. | + +----------------------------------------------------------------------+ + | Authors: Dmitry Stogov | + | Xinchen Hui | + +----------------------------------------------------------------------+ +*/ + +#include "zend.h" +#include "zend_gdb.h" + +ZEND_API zend_gdbjit_descriptor __jit_debug_descriptor = { + 1, ZEND_GDBJIT_NOACTION, NULL, NULL +}; + +ZEND_API zend_never_inline void __jit_debug_register_code() +{ + __asm__ __volatile__(""); +} + +/* + * Local variables: + * tab-width: 4 + * c-basic-offset: 4 + * indent-tabs-mode: t + * End: + */ diff --git a/Zend/zend_gdb.h b/Zend/zend_gdb.h new file mode 100644 index 0000000000000..7eab5ec59b7b4 --- /dev/null +++ b/Zend/zend_gdb.h @@ -0,0 +1,54 @@ +/* + +----------------------------------------------------------------------+ + | Zend JIT | + +----------------------------------------------------------------------+ + | Copyright (c) 1998-2016 The PHP Group | + +----------------------------------------------------------------------+ + | This source file is subject to version 3.01 of the PHP license, | + | that is bundled with this package in the file LICENSE, and is | + | available through the world-wide-web at the following url: | + | http://www.php.net/license/3_01.txt | + | If you did not receive a copy of the PHP license and are unable to | + | obtain it through the world-wide-web, please send a note to | + | license@php.net so we can mail you a copy immediately. | + +----------------------------------------------------------------------+ + | Authors: Dmitry Stogov | + | Xinchen Hui | + +----------------------------------------------------------------------+ +*/ + +#ifndef ZEND_GDB + +enum { + ZEND_GDBJIT_NOACTION, + ZEND_GDBJIT_REGISTER, + ZEND_GDBJIT_UNREGISTER +}; + +typedef struct _zend_gdbjit_code_entry { + struct _zend_gdbjit_code_entry *next_entry; + struct _zend_gdbjit_code_entry *prev_entry; + const char *symfile_addr; + uint64_t symfile_size; +} zend_gdbjit_code_entry; + +typedef struct _zend_gdbjit_descriptor { + uint32_t version; + uint32_t action_flag; + struct _zend_gdbjit_code_entry *relevant_entry; + struct _zend_gdbjit_code_entry *first_entry; +} zend_gdbjit_descriptor; + +extern ZEND_API zend_gdbjit_descriptor __jit_debug_descriptor; + +ZEND_API zend_never_inline void __jit_debug_register_code(); + +#endif + +/* + * Local variables: + * tab-width: 4 + * c-basic-offset: 4 + * indent-tabs-mode: t + * End: + */ diff --git a/configure.in b/configure.in index de68932ca1fa3..3de605b7648aa 100644 --- a/configure.in +++ b/configure.in @@ -1513,7 +1513,7 @@ PHP_ADD_SOURCES(Zend, \ zend_iterators.c zend_interfaces.c zend_exceptions.c zend_strtod.c zend_gc.c \ zend_closures.c zend_float.c zend_string.c zend_signal.c zend_generators.c \ zend_virtual_cwd.c zend_ast.c zend_objects.c zend_object_handlers.c zend_objects_API.c \ - zend_default_classes.c zend_inheritance.c zend_smart_str.c, \ + zend_default_classes.c zend_inheritance.c zend_smart_str.c zend_gdb.c, \ -DZEND_ENABLE_STATIC_TSRMLS_CACHE=1) dnl Selectively disable optimization due to high RAM usage during diff --git a/ext/opcache/jit/zend_jit_gdb.c b/ext/opcache/jit/zend_jit_gdb.c index e9043c26c975c..cd0587fe1afd7 100644 --- a/ext/opcache/jit/zend_jit_gdb.c +++ b/ext/opcache/jit/zend_jit_gdb.c @@ -19,6 +19,8 @@ #define HAVE_GDB +#include "zend_gdb.h" + #if SIZEOF_SIZE_T == 8 # define ELF64 #else @@ -179,12 +181,6 @@ enum { #endif }; -enum { - GDBJIT_NOACTION, - GDBJIT_REGISTER, - GDBJIT_UNREGISTER -}; - enum { GDBJIT_SECT_NULL, GDBJIT_SECT_text, @@ -261,20 +257,6 @@ static const zend_elf_header zend_elfhdr_template = { .shstridx = GDBJIT_SECT_shstrtab }; -typedef struct _zend_gdbjit_code_entry { - struct _zend_gdbjit_code_entry *next_entry; - struct _zend_gdbjit_code_entry *prev_entry; - const char *symfile_addr; - uint64_t symfile_size; -} zend_gdbjit_code_entry; - -typedef struct _zend_gdbjit_descriptor { - uint32_t version; - uint32_t action_flag; - struct _zend_gdbjit_code_entry *relevant_entry; - struct _zend_gdbjit_code_entry *first_entry; -} zend_gdbjit_descriptor; - /* Context for generating the ELF object for the GDB JIT API. */ typedef struct _zend_gdbjit_ctx { zend_op_array *op_array; /* Pointer to op_array */ @@ -291,15 +273,6 @@ typedef struct _zend_gdbjit_ctx { zend_gdbjit_obj obj; /* In-memory ELF object. */ } zend_gdbjit_ctx; -zend_gdbjit_descriptor __jit_debug_descriptor = { - 1, GDBJIT_NOACTION, NULL, NULL -}; - -zend_never_inline void __jit_debug_register_code() -{ - __asm__ __volatile__(""); -} - /* Add a zero-terminated string */ static uint32_t zend_gdbjit_strz(zend_gdbjit_ctx *ctx, const char *str) { @@ -621,7 +594,7 @@ static int zend_jit_gdb_register(const char *name, } __jit_debug_descriptor.relevant_entry = entry; - __jit_debug_descriptor.action_flag = GDBJIT_REGISTER; + __jit_debug_descriptor.action_flag = ZEND_GDBJIT_REGISTER; __jit_debug_register_code(); return 1; @@ -631,7 +604,7 @@ static int zend_jit_gdb_unregister() { zend_gdbjit_code_entry *entry; - __jit_debug_descriptor.action_flag = GDBJIT_UNREGISTER; + __jit_debug_descriptor.action_flag = ZEND_GDBJIT_UNREGISTER; while ((entry = __jit_debug_descriptor.relevant_entry)) { if (entry->prev_entry) { entry->prev_entry->next_entry = NULL; From d2ed375c4ebe9c61a564cb631d76d5db2deb7f8c Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 24 Aug 2016 12:25:32 +0300 Subject: [PATCH 039/569] Encapsulate GDB interface details into Zend/zend_gdb.c --- Zend/zend_gdb.c | 70 ++++++++++++++++++++++++++++++++++ Zend/zend_gdb.h | 25 +----------- ext/opcache/jit/zend_jit_gdb.c | 46 ++-------------------- 3 files changed, 75 insertions(+), 66 deletions(-) diff --git a/Zend/zend_gdb.c b/Zend/zend_gdb.c index b630c1bd8bc4e..7564476b85494 100644 --- a/Zend/zend_gdb.c +++ b/Zend/zend_gdb.c @@ -20,6 +20,26 @@ #include "zend.h" #include "zend_gdb.h" +enum { + ZEND_GDBJIT_NOACTION, + ZEND_GDBJIT_REGISTER, + ZEND_GDBJIT_UNREGISTER +}; + +typedef struct _zend_gdbjit_code_entry { + struct _zend_gdbjit_code_entry *next_entry; + struct _zend_gdbjit_code_entry *prev_entry; + const char *symfile_addr; + uint64_t symfile_size; +} zend_gdbjit_code_entry; + +typedef struct _zend_gdbjit_descriptor { + uint32_t version; + uint32_t action_flag; + struct _zend_gdbjit_code_entry *relevant_entry; + struct _zend_gdbjit_code_entry *first_entry; +} zend_gdbjit_descriptor; + ZEND_API zend_gdbjit_descriptor __jit_debug_descriptor = { 1, ZEND_GDBJIT_NOACTION, NULL, NULL }; @@ -29,6 +49,56 @@ ZEND_API zend_never_inline void __jit_debug_register_code() __asm__ __volatile__(""); } +ZEND_API int zend_gdb_register_code(const void *object, size_t size) +{ + zend_gdbjit_code_entry *entry; + + entry = malloc(sizeof(zend_gdbjit_code_entry) + size); + if (entry == NULL) { + return 0; + } + + entry->symfile_addr = ((char*)entry) + sizeof(zend_gdbjit_code_entry); + entry->symfile_size = size; + + memcpy((char *)entry->symfile_addr, object, size); + + entry->next_entry = NULL; + entry->prev_entry = __jit_debug_descriptor.relevant_entry; + + if (entry->prev_entry) { + entry->prev_entry->next_entry = entry; + } else { + __jit_debug_descriptor.first_entry = entry; + } + __jit_debug_descriptor.relevant_entry = entry; + + __jit_debug_descriptor.action_flag = ZEND_GDBJIT_REGISTER; + __jit_debug_register_code(); + + return 1; +} + +ZEND_API void zend_gdb_unregister_all(void) +{ + zend_gdbjit_code_entry *entry; + + __jit_debug_descriptor.action_flag = ZEND_GDBJIT_UNREGISTER; + while ((entry = __jit_debug_descriptor.relevant_entry)) { + if (entry->prev_entry) { + entry->prev_entry->next_entry = NULL; + } else { + __jit_debug_descriptor.first_entry = NULL; + } + + __jit_debug_register_code(); + + __jit_debug_descriptor.relevant_entry = entry->prev_entry; + + free(entry); + } +} + /* * Local variables: * tab-width: 4 diff --git a/Zend/zend_gdb.h b/Zend/zend_gdb.h index 7eab5ec59b7b4..a2560d8772c6e 100644 --- a/Zend/zend_gdb.h +++ b/Zend/zend_gdb.h @@ -19,29 +19,8 @@ #ifndef ZEND_GDB -enum { - ZEND_GDBJIT_NOACTION, - ZEND_GDBJIT_REGISTER, - ZEND_GDBJIT_UNREGISTER -}; - -typedef struct _zend_gdbjit_code_entry { - struct _zend_gdbjit_code_entry *next_entry; - struct _zend_gdbjit_code_entry *prev_entry; - const char *symfile_addr; - uint64_t symfile_size; -} zend_gdbjit_code_entry; - -typedef struct _zend_gdbjit_descriptor { - uint32_t version; - uint32_t action_flag; - struct _zend_gdbjit_code_entry *relevant_entry; - struct _zend_gdbjit_code_entry *first_entry; -} zend_gdbjit_descriptor; - -extern ZEND_API zend_gdbjit_descriptor __jit_debug_descriptor; - -ZEND_API zend_never_inline void __jit_debug_register_code(); +ZEND_API int zend_gdb_register_code(const void *object, size_t size); +ZEND_API void zend_gdb_unregister_all(void); #endif diff --git a/ext/opcache/jit/zend_jit_gdb.c b/ext/opcache/jit/zend_jit_gdb.c index cd0587fe1afd7..74559525ad2a9 100644 --- a/ext/opcache/jit/zend_jit_gdb.c +++ b/ext/opcache/jit/zend_jit_gdb.c @@ -557,7 +557,6 @@ static int zend_jit_gdb_register(const char *name, size_t size) { zend_gdbjit_ctx ctx; - zend_gdbjit_code_entry *entry; ctx.op_array = op_array; ctx.mcaddr = (uintptr_t)start; @@ -574,51 +573,12 @@ static int zend_jit_gdb_register(const char *name, zend_gdbjit_buildobj(&ctx); - entry = malloc(sizeof(zend_gdbjit_code_entry) + ctx.objsize); - if (entry == NULL) { - return 0; - } - - entry->symfile_addr = ((char*)entry) + sizeof(zend_gdbjit_code_entry); - entry->symfile_size = ctx.objsize; - - memcpy((char *)entry->symfile_addr, &ctx.obj, ctx.objsize); - - entry->next_entry = NULL; - entry->prev_entry = __jit_debug_descriptor.relevant_entry; - - if (entry->prev_entry) { - entry->prev_entry->next_entry = entry; - } else { - __jit_debug_descriptor.first_entry = entry; - } - __jit_debug_descriptor.relevant_entry = entry; - - __jit_debug_descriptor.action_flag = ZEND_GDBJIT_REGISTER; - __jit_debug_register_code(); - - return 1; + return zend_gdb_register_code(&ctx.obj, ctx.objsize); } -static int zend_jit_gdb_unregister() +static int zend_jit_gdb_unregister(void) { - zend_gdbjit_code_entry *entry; - - __jit_debug_descriptor.action_flag = ZEND_GDBJIT_UNREGISTER; - while ((entry = __jit_debug_descriptor.relevant_entry)) { - if (entry->prev_entry) { - entry->prev_entry->next_entry = NULL; - } else { - __jit_debug_descriptor.first_entry = NULL; - } - - __jit_debug_register_code(); - - __jit_debug_descriptor.relevant_entry = entry->prev_entry; - - free(entry); - } - + zend_gdb_unregister_all(); return 1; } From 63bbd0ce21796be74a614d6ed016e20ead292433 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Wed, 24 Aug 2016 17:55:28 +0800 Subject: [PATCH 040/569] Fixed padding issue while disassmbling --- ext/opcache/jit/zend_jit.c | 23 +++++++++++++---------- ext/opcache/jit/zend_jit_disasm_x86.c | 3 ++- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index a531ee1bfa591..80ae13422f0db 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -89,18 +89,18 @@ static zend_string *zend_jit_func_name(zend_op_array *op_array) } } -static void *dasm_link_and_encode(dasm_State **dasm_state) +static void *dasm_link_and_encode(dasm_State **dasm_state, size_t *len) { size_t size; int ret; void *entry; - if (dasm_link(dasm_state, &size) != DASM_S_OK) { + if (dasm_link(dasm_state, len) != DASM_S_OK) { // TODO: dasm_link() failed ??? return NULL; } - size = ZEND_MM_ALIGNED_SIZE_EX(size, DASM_ALIGNMENT); + size = ZEND_MM_ALIGNED_SIZE_EX(*len, DASM_ALIGNMENT); if ((void*)((char*)dasm_ptr + size) > dasm_end) { // TODO: jit_buffer_size overflow ??? @@ -184,6 +184,7 @@ ZEND_API int zend_jit(zend_op_array *op_array, zend_script *script) zend_op *opline; dasm_State* dasm_state = NULL; void *handler; + size_t size; if (!dasm_buf) { return FAILURE; @@ -399,7 +400,7 @@ ZEND_API int zend_jit(zend_op_array *op_array, zend_script *script) } } - handler = dasm_link_and_encode(&dasm_state); + handler = dasm_link_and_encode(&dasm_state, &size); if (!handler) { goto jit_failure; } @@ -425,7 +426,8 @@ ZEND_API int zend_jit(zend_op_array *op_array, zend_script *script) zend_jit_disasm( name ? ZSTR_VAL(name) : NULL, op_array->filename ? ZSTR_VAL(op_array->filename) : NULL, - handler, dasm_ptr); + handler, + size); if (name) { zend_string_release(name); } @@ -440,7 +442,7 @@ ZEND_API int zend_jit(zend_op_array *op_array, zend_script *script) ZSTR_VAL(name), op_array, handler, - (char*)dasm_ptr - (char*)handler); + size); zend_string_release(name); } } @@ -453,7 +455,7 @@ ZEND_API int zend_jit(zend_op_array *op_array, zend_script *script) zend_jit_oprofile_register( name ? ZSTR_VAL(name) : NULL, handler, - (char*)dasm_ptr - (char*)handler); + size); if (name) { zend_string_release(name); } @@ -468,7 +470,7 @@ ZEND_API int zend_jit(zend_op_array *op_array, zend_script *script) zend_jit_perf_dump( ZSTR_VAL(name), handler, - (char*)dasm_ptr - (char*)handler); + size); zend_string_release(name); } } @@ -508,6 +510,7 @@ ZEND_API int zend_jit_startup(size_t size) size_t page_size = jit_page_size(); int shared = 1; void *buf; + size_t buf_len; dasm_State* dasm_state = NULL; int ret; @@ -557,7 +560,7 @@ ZEND_API int zend_jit_startup(size_t size) dasm_init(&dasm_state, DASM_MAXSECTION); dasm_setupglobal(&dasm_state, dasm_labels, lbl__MAX); dasm_setup(&dasm_state, dasm_actions); - ret = zend_jit_stubs(&dasm_state) && dasm_link_and_encode(&dasm_state); + ret = zend_jit_stubs(&dasm_state) && dasm_link_and_encode(&dasm_state, &buf_len); dasm_free(&dasm_state); zend_jit_protect(); @@ -565,7 +568,7 @@ ZEND_API int zend_jit_startup(size_t size) #ifdef HAVE_DISASM if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_ASM) { if (dasm_buf != dasm_ptr) { - zend_jit_disasm(NULL, NULL, dasm_buf, dasm_ptr); + zend_jit_disasm(NULL, NULL, dasm_buf, buf_len); } } #endif diff --git a/ext/opcache/jit/zend_jit_disasm_x86.c b/ext/opcache/jit/zend_jit_disasm_x86.c index 0c5ac88445482..6aafa062d5b0d 100644 --- a/ext/opcache/jit/zend_jit_disasm_x86.c +++ b/ext/opcache/jit/zend_jit_disasm_x86.c @@ -56,8 +56,9 @@ static const char* zend_jit_disasm_resolver(struct ud *ud, static int zend_jit_disasm(const char *name, const char *filename, const void *start, - const void *end) + size_t size) { + const void *end = (void *)((char *)start + size); if (name) { fprintf(stderr, "%s: ; (%s)\n", name, filename ? filename : "unknown"); } From 6482c06264e98bc8afb7db4331bdba044301b172 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 24 Aug 2016 13:03:13 +0300 Subject: [PATCH 041/569] Implemented GDB autodetection (may be disabled in the future) --- Zend/zend_gdb.c | 42 ++++++++++++++++++++++++++++++++++ Zend/zend_gdb.h | 1 + ext/opcache/jit/zend_jit.c | 4 ++++ ext/opcache/jit/zend_jit_gdb.c | 9 ++++++++ 4 files changed, 56 insertions(+) diff --git a/Zend/zend_gdb.c b/Zend/zend_gdb.c index 7564476b85494..9a8b93871fe2f 100644 --- a/Zend/zend_gdb.c +++ b/Zend/zend_gdb.c @@ -20,6 +20,11 @@ #include "zend.h" #include "zend_gdb.h" +#include +#include +#include +#include + enum { ZEND_GDBJIT_NOACTION, ZEND_GDBJIT_REGISTER, @@ -99,6 +104,43 @@ ZEND_API void zend_gdb_unregister_all(void) } } +ZEND_API int zend_gdb_present(void) +{ + int ret = 0; + int fd = open("/proc/self/status", O_RDONLY); + + if (fd > 0) { + char buf[1024]; + ssize_t n = read(fd, buf, sizeof(buf) - 1); + char *s; + pid_t pid; + + if (n > 0) { + buf[n] = 0; + s = strstr(buf, "TracerPid:"); + if (s) { + s += sizeof("TracerPid:") - 1; + while (*s == ' ' || *s == '\t') { + s++; + } + pid = atoi(s); + if (pid) { + sprintf(buf, "/proc/%d/exe", (int)pid); + if (readlink(buf, buf, sizeof(buf) - 1) > 0) { + if (strstr(buf, "gdb")) { + ret = 1; + } + } + } + } + } + + close(fd); + } + + return ret; +} + /* * Local variables: * tab-width: 4 diff --git a/Zend/zend_gdb.h b/Zend/zend_gdb.h index a2560d8772c6e..616e36b971877 100644 --- a/Zend/zend_gdb.h +++ b/Zend/zend_gdb.h @@ -21,6 +21,7 @@ ZEND_API int zend_gdb_register_code(const void *object, size_t size); ZEND_API void zend_gdb_unregister_all(void); +ZEND_API int zend_gdb_present(void); #endif diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 80ae13422f0db..1dddffb08e059 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -514,6 +514,10 @@ ZEND_API int zend_jit_startup(size_t size) dasm_State* dasm_state = NULL; int ret; +#ifdef HAVE_GDB + zend_jit_gdb_init(); +#endif + #ifdef HAVE_OPROFILE if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_OPROFILE) { shared = 0; diff --git a/ext/opcache/jit/zend_jit_gdb.c b/ext/opcache/jit/zend_jit_gdb.c index 74559525ad2a9..02b0b16c1983a 100644 --- a/ext/opcache/jit/zend_jit_gdb.c +++ b/ext/opcache/jit/zend_jit_gdb.c @@ -582,6 +582,15 @@ static int zend_jit_gdb_unregister(void) return 1; } +static void zend_jit_gdb_init(void) +{ +#if 1 + if (zend_gdb_present()) { + ZCG(accel_directives).jit_debug |= ZEND_JIT_DEBUG_GDB; + } +#endif +} + /* * Local variables: * tab-width: 4 From 6cf4430aef867b7cac530e22e6e1fb6a084b47b9 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 24 Aug 2016 14:00:45 +0300 Subject: [PATCH 042/569] Fixed printf() format specifiers --- ext/opcache/jit/zend_jit_perf_dump.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_perf_dump.c b/ext/opcache/jit/zend_jit_perf_dump.c index 63ee96265ecc3..5feb93dc662d1 100644 --- a/ext/opcache/jit/zend_jit_perf_dump.c +++ b/ext/opcache/jit/zend_jit_perf_dump.c @@ -35,7 +35,7 @@ static void zend_jit_perf_dump(const char *name, void *start, size_t size) } setlinebuf(fp); } - fprintf(fp, "%lx %lx %s\n", (size_t)start, size, name); + fprintf(fp, "%zx %zx %s\n", (size_t)(uintptr_t)start, size, name); } /* From 5b69dc2553b6b255047168553bccb7243f0cfa7d Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 24 Aug 2016 14:50:19 +0300 Subject: [PATCH 043/569] Include filename into a debug symbol name (break 'JIT$/../file.php') --- ext/opcache/jit/zend_jit.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 1dddffb08e059..949a5d2c65e23 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -82,7 +82,9 @@ static zend_string *zend_jit_func_name(zend_op_array *op_array) return buf.s; } } else if (op_array->filename) { - smart_str_appends(&buf, "JIT$(main)"); + smart_str_appends(&buf, "JIT$"); + smart_str_appendl(&buf, ZSTR_VAL(op_array->filename), ZSTR_LEN(op_array->filename)); + smart_str_0(&buf); return buf.s; } else { return NULL; From 19324da69c4dc34d3756ab7499659d9501b8d49f Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 24 Aug 2016 15:57:59 +0300 Subject: [PATCH 044/569] Generate debug info for JIT stubs --- ext/opcache/jit/zend_jit.c | 201 ++++++++++++++++-------------- ext/opcache/jit/zend_jit_gdb.c | 4 +- ext/opcache/jit/zend_jit_x86.dasc | 16 ++- 3 files changed, 123 insertions(+), 98 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 949a5d2c65e23..f14e3f9ce2634 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -36,9 +36,21 @@ #define ZEND_JIT_LEVEL ZEND_JIT_LEVEL_FULL +#define JIT_PREFIX "JIT$" +#define JIT_STUB_PREFIX "JIT$$" + // TODO: define DASM_M_GROW and DASM_M_FREE to use CG(arena) ??? #include "dynasm/dasm_proto.h" + +typedef struct _zend_jit_stub { + const char *name; + int (*stub)(dasm_State **Dst); +} zend_jit_stub; + +#define JIT_STUB(name) \ + {JIT_STUB_PREFIX #name, zend_jit_ ## name ## _stub} + #include "dynasm/dasm_x86.h" #include "jit/zend_jit_x86.c" #include "jit/zend_jit_disasm_x86.c" @@ -69,20 +81,20 @@ static zend_string *zend_jit_func_name(zend_op_array *op_array) if (op_array->function_name) { if (op_array->scope) { - smart_str_appends(&buf, "JIT$"); + smart_str_appends(&buf, JIT_PREFIX); smart_str_appendl(&buf, ZSTR_VAL(op_array->scope->name), ZSTR_LEN(op_array->scope->name)); smart_str_appends(&buf, "::"); smart_str_appendl(&buf, ZSTR_VAL(op_array->function_name), ZSTR_LEN(op_array->function_name)); smart_str_0(&buf); return buf.s; } else { - smart_str_appends(&buf, "JIT$"); + smart_str_appends(&buf, JIT_PREFIX); smart_str_appendl(&buf, ZSTR_VAL(op_array->function_name), ZSTR_LEN(op_array->function_name)); smart_str_0(&buf); return buf.s; } } else if (op_array->filename) { - smart_str_appends(&buf, "JIT$"); + smart_str_appends(&buf, JIT_PREFIX); smart_str_appendl(&buf, ZSTR_VAL(op_array->filename), ZSTR_LEN(op_array->filename)); smart_str_0(&buf); return buf.s; @@ -91,19 +103,22 @@ static zend_string *zend_jit_func_name(zend_op_array *op_array) } } -static void *dasm_link_and_encode(dasm_State **dasm_state, size_t *len) +static void *dasm_link_and_encode(dasm_State **dasm_state, + zend_op_array *op_array, + const char *name) { size_t size; int ret; void *entry; +#if defined(HAVE_DISASM) || defined(HAVE_GDB) || defined(HAVE_OPROFILE) || defined(HAVE_PERFTOOLS) + zend_string *str = NULL; +#endif - if (dasm_link(dasm_state, len) != DASM_S_OK) { + if (dasm_link(dasm_state, &size) != DASM_S_OK) { // TODO: dasm_link() failed ??? return NULL; } - size = ZEND_MM_ALIGNED_SIZE_EX(*len, DASM_ALIGNMENT); - if ((void*)((char*)dasm_ptr + size) > dasm_end) { // TODO: jit_buffer_size overflow ??? return NULL; @@ -117,7 +132,66 @@ static void *dasm_link_and_encode(dasm_State **dasm_state, size_t *len) } entry = dasm_ptr; - dasm_ptr = (void*)((char*)dasm_ptr + size); + dasm_ptr = (void*)((char*)dasm_ptr + ZEND_MM_ALIGNED_SIZE_EX(size, DASM_ALIGNMENT)); + +#if defined(HAVE_DISASM) || defined(HAVE_GDB) || defined(HAVE_OPROFILE) || defined(HAVE_PERFTOOLS) + if (!name) { + if (ZCG(accel_directives).jit_debug & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_OPROFILE|ZEND_JIT_DEBUG_PERF)) { + str = zend_jit_func_name(op_array); + if (str) { + name = ZSTR_VAL(str); + } + } + } +#endif + +#ifdef HAVE_DISASM + if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_ASM) { + zend_jit_disasm( + name, + (op_array && op_array->filename) ? ZSTR_VAL(op_array->filename) : NULL, + entry, + size); + } +#endif + +#ifdef HAVE_GDB + if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_GDB) { + if (name) { + zend_jit_gdb_register( + name, + op_array, + entry, + size); + } + } +#endif + +#ifdef HAVE_OPROFILE + if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_OPROFILE) { + zend_jit_oprofile_register( + name, + entry, + size); + } +#endif + +#ifdef HAVE_PERFTOOLS + if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_PERF) { + if (name) { + zend_jit_perf_dump( + name, + entry, + size); + } + } +#endif + +#if defined(HAVE_DISASM) || defined(HAVE_GDB) || defined(HAVE_OPROFILE) || defined(HAVE_PERFTOOLS) + if (str) { + zend_string_release(str); + } +#endif return entry; } @@ -186,7 +260,6 @@ ZEND_API int zend_jit(zend_op_array *op_array, zend_script *script) zend_op *opline; dasm_State* dasm_state = NULL; void *handler; - size_t size; if (!dasm_buf) { return FAILURE; @@ -402,7 +475,7 @@ ZEND_API int zend_jit(zend_op_array *op_array, zend_script *script) } } - handler = dasm_link_and_encode(&dasm_state, &size); + handler = dasm_link_and_encode(&dasm_state, op_array, NULL); if (!handler) { goto jit_failure; } @@ -421,63 +494,6 @@ ZEND_API int zend_jit(zend_op_array *op_array, zend_script *script) } dasm_free(&dasm_state); -#ifdef HAVE_DISASM - if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_ASM) { - zend_string *name = zend_jit_func_name(op_array); - - zend_jit_disasm( - name ? ZSTR_VAL(name) : NULL, - op_array->filename ? ZSTR_VAL(op_array->filename) : NULL, - handler, - size); - if (name) { - zend_string_release(name); - } - } -#endif - -#ifdef HAVE_GDB - if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_GDB) { - zend_string *name = zend_jit_func_name(op_array); - if (name) { - zend_jit_gdb_register( - ZSTR_VAL(name), - op_array, - handler, - size); - zend_string_release(name); - } - } -#endif - -#ifdef HAVE_OPROFILE - if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_OPROFILE) { - zend_string *name = zend_jit_func_name(op_array); - - zend_jit_oprofile_register( - name ? ZSTR_VAL(name) : NULL, - handler, - size); - if (name) { - zend_string_release(name); - } - } -#endif - -#ifdef HAVE_PERFTOOLS - if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_PERF) { - zend_string *name = zend_jit_func_name(op_array); - - if (name) { - zend_jit_perf_dump( - ZSTR_VAL(name), - handler, - size); - zend_string_release(name); - } - } -#endif - zend_arena_release(&CG(arena), checkpoint); return SUCCESS; @@ -507,13 +523,32 @@ ZEND_API void zend_jit_protect(void) #endif } +static int zend_jit_make_stubs(void) +{ + dasm_State* dasm_state = NULL; + uint32_t i; + + dasm_init(&dasm_state, DASM_MAXSECTION); + dasm_setupglobal(&dasm_state, dasm_labels, lbl__MAX); + + for (i = 0; i < sizeof(zend_jit_stubs)/sizeof(zend_jit_stubs[0]); i++) { + dasm_setup(&dasm_state, dasm_actions); + if (!zend_jit_stubs[i].stub(&dasm_state)) { + return 0; + } + if (!dasm_link_and_encode(&dasm_state, NULL, zend_jit_stubs[i].name)) { + return 0; + } + } + dasm_free(&dasm_state); + return 1; +} + ZEND_API int zend_jit_startup(size_t size) { size_t page_size = jit_page_size(); int shared = 1; void *buf; - size_t buf_len; - dasm_State* dasm_state = NULL; int ret; #ifdef HAVE_GDB @@ -523,6 +558,10 @@ ZEND_API int zend_jit_startup(size_t size) #ifdef HAVE_OPROFILE if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_OPROFILE) { shared = 0; + if (!zend_jit_oprofile_startup()) { + // TODO: error reporting and cleanup ??? + return FAILURE; + } } #endif @@ -552,33 +591,11 @@ ZEND_API int zend_jit_startup(size_t size) } } #endif -#ifdef HAVE_OPROFILE - if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_OPROFILE) { - if (!zend_jit_oprofile_startup()) { - // TODO: error reporting and cleanup ??? - return FAILURE; - } - } -#endif zend_jit_unprotect(); - - dasm_init(&dasm_state, DASM_MAXSECTION); - dasm_setupglobal(&dasm_state, dasm_labels, lbl__MAX); - dasm_setup(&dasm_state, dasm_actions); - ret = zend_jit_stubs(&dasm_state) && dasm_link_and_encode(&dasm_state, &buf_len); - dasm_free(&dasm_state); - + ret = zend_jit_make_stubs(); zend_jit_protect(); -#ifdef HAVE_DISASM - if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_ASM) { - if (dasm_buf != dasm_ptr) { - zend_jit_disasm(NULL, NULL, dasm_buf, buf_len); - } - } -#endif - if (!ret) { // TODO: error reporting and cleanup ??? return FAILURE; diff --git a/ext/opcache/jit/zend_jit_gdb.c b/ext/opcache/jit/zend_jit_gdb.c index 02b0b16c1983a..8b1168e31f722 100644 --- a/ext/opcache/jit/zend_jit_gdb.c +++ b/ext/opcache/jit/zend_jit_gdb.c @@ -568,8 +568,8 @@ static int zend_jit_gdb_register(const char *name, #endif ctx.spadj = CFRAME_SIZE_JIT; ctx.name = name; - ctx.filename = ZSTR_VAL(op_array->filename); - ctx.lineno = op_array->line_start; + ctx.filename = op_array ? ZSTR_VAL(op_array->filename) : "unknown"; + ctx.lineno = op_array ? op_array->line_start : 0; zend_gdbjit_buildobj(&ctx); diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index efd2c6d13f4de..3597b9e80f5ff 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -39,10 +39,9 @@ static void* dasm_labels[lbl__MAX]; |.section code -static int zend_jit_stubs(dasm_State **Dst) +static int zend_jit_interrupt_handler_stub(dasm_State **Dst) { // TODO: check interrupt_handler ??? - |.align 16 |->interrupt_handler: |.if X64 | add rsp, 8 // stack alignment @@ -52,8 +51,12 @@ static int zend_jit_stubs(dasm_State **Dst) | add esp, 12 // stack alignment | jmp extern zend_timeout |.endif - | - |.align 16 + + return 1; +} + +static int zend_jit_exception_handler_stub(dasm_State **Dst) +{ |->exception_handler: |.if X64 | add rsp, 8 // stack alignment @@ -67,6 +70,11 @@ static int zend_jit_stubs(dasm_State **Dst) return 1; } +static const zend_jit_stub zend_jit_stubs[] = { + JIT_STUB(interrupt_handler), + JIT_STUB(exception_handler), +}; + static int zend_jit_align_func(dasm_State **Dst) { |.align 16 From 1db70bf826ca2a857c4a29987448f65dde6c21ea Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Wed, 24 Aug 2016 23:41:05 +0800 Subject: [PATCH 045/569] Prefer FILO, besides relevant_entry not always means last one in list --- Zend/zend_gdb.c | 28 +++++++++++++--------------- ext/opcache/jit/zend_jit_gdb.c | 2 -- 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/Zend/zend_gdb.c b/Zend/zend_gdb.c index 9a8b93871fe2f..dec0e6289f1f5 100644 --- a/Zend/zend_gdb.c +++ b/Zend/zend_gdb.c @@ -68,16 +68,16 @@ ZEND_API int zend_gdb_register_code(const void *object, size_t size) memcpy((char *)entry->symfile_addr, object, size); - entry->next_entry = NULL; - entry->prev_entry = __jit_debug_descriptor.relevant_entry; + entry->prev_entry = NULL; + entry->next_entry = __jit_debug_descriptor.first_entry; - if (entry->prev_entry) { - entry->prev_entry->next_entry = entry; - } else { - __jit_debug_descriptor.first_entry = entry; + if (entry->next_entry) { + entry->next_entry->prev_entry = entry; } - __jit_debug_descriptor.relevant_entry = entry; + __jit_debug_descriptor.first_entry = entry; + /* Notify GDB */ + __jit_debug_descriptor.relevant_entry = entry; __jit_debug_descriptor.action_flag = ZEND_GDBJIT_REGISTER; __jit_debug_register_code(); @@ -89,17 +89,15 @@ ZEND_API void zend_gdb_unregister_all(void) zend_gdbjit_code_entry *entry; __jit_debug_descriptor.action_flag = ZEND_GDBJIT_UNREGISTER; - while ((entry = __jit_debug_descriptor.relevant_entry)) { - if (entry->prev_entry) { - entry->prev_entry->next_entry = NULL; - } else { - __jit_debug_descriptor.first_entry = NULL; + while ((entry = __jit_debug_descriptor.first_entry)) { + __jit_debug_descriptor.first_entry = entry->next_entry; + if (entry->next_entry) { + entry->next_entry->prev_entry = NULL; } - + /* Notify GDB */ + __jit_debug_descriptor.relevant_entry = entry; __jit_debug_register_code(); - __jit_debug_descriptor.relevant_entry = entry->prev_entry; - free(entry); } } diff --git a/ext/opcache/jit/zend_jit_gdb.c b/ext/opcache/jit/zend_jit_gdb.c index 8b1168e31f722..0814f85a86494 100644 --- a/ext/opcache/jit/zend_jit_gdb.c +++ b/ext/opcache/jit/zend_jit_gdb.c @@ -361,11 +361,9 @@ static void zend_gdbjit_symtab(zend_gdbjit_ctx *ctx) sym->info = ELFSYM_TYPE_FUNC|ELFSYM_BIND_GLOBAL; } - #define SECTALIGN(p, a) \ ((p) = (uint8_t *)(((uintptr_t)(p) + ((a)-1)) & ~(uintptr_t)((a)-1))) - /* Shortcuts to generate DWARF structures. */ #define DB(x) (*p++ = (x)) #define DI8(x) (*(int8_t *)p = (x), p++) From 7b87d58fa97cea62e5472d6ef6cf36280c6b0348 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 25 Aug 2016 11:19:56 +0300 Subject: [PATCH 046/569] Use more compact instruction to load immediate operand into 64-bit registers, if the value fits into 32-bit --- ext/opcache/jit/zend_jit_x86.dasc | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 3597b9e80f5ff..603d2c3c2b7d5 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -39,13 +39,22 @@ static void* dasm_labels[lbl__MAX]; |.section code +|.macro load_addr, reg, addr +||if (((ptrdiff_t)addr) <= 0x7fffffff) { +| mov reg, ((ptrdiff_t)addr) // 0x48 0xc7 0xc0 +|| } else { +| mov64 reg, ((ptrdiff_t)addr) // 0x48 0xb8 +||} +|.endmacro + + static int zend_jit_interrupt_handler_stub(dasm_State **Dst) { // TODO: check interrupt_handler ??? |->interrupt_handler: |.if X64 | add rsp, 8 // stack alignment - | mov64 rax, ((ptrdiff_t)zend_timeout) + | load_addr rax, zend_timeout | jmp rax |.else | add esp, 12 // stack alignment @@ -60,7 +69,7 @@ static int zend_jit_exception_handler_stub(dasm_State **Dst) |->exception_handler: |.if X64 | add rsp, 8 // stack alignment - | mov64 rax, ((ptrdiff_t)EG(exception_op)->handler) + | load_addr rax, EG(exception_op)->handler | jmp rax |.else | add esp, 12 // stack alignment @@ -116,7 +125,7 @@ static int zend_jit_handler(dasm_State **Dst, zend_op *opline) const void *handler = opline->handler; |.if X64 - | mov64 rax, ((ptrdiff_t)handler) + | load_addr rax, handler | call rax |.else | call aword &handler @@ -131,7 +140,7 @@ static int zend_jit_tail_handler(dasm_State **Dst, zend_op *opline) |.if X64 | add rsp, 8 // stack alignment - | mov64 rax, ((ptrdiff_t)handler) + | load_addr rax, handler | jmp rax |.else | add esp, 12 // stack alignment @@ -149,7 +158,7 @@ static int zend_jit_skip_handler(dasm_State **Dst, uint32_t skip) static int zend_jit_set_opline(dasm_State **Dst, zend_op *target_opline) { |.if X64 - | mov64 IP, ((ptrdiff_t)target_opline) + | load_addr IP, target_opline |.else | mov IP, target_opline |.endif From 12f4497b6fcd53a9c6ff32c7932234b6e35f6855 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 25 Aug 2016 12:48:15 +0300 Subject: [PATCH 047/569] Fixed interrupt handling --- ext/opcache/jit/zend_jit_x86.dasc | 74 ++++++++++++++++++++++++++----- 1 file changed, 62 insertions(+), 12 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 603d2c3c2b7d5..d003bbaea88ea 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -23,15 +23,36 @@ |.endif |.if X64 - |.define FP, r14 - |.define IP, r15 - |.define IPl, r15d + |.define FP, r14 + |.define IP, r15 + |.define IPl, r15d + |.define CARG1, rdi // x64/POSIX C call arguments. + |.define CARG2, rsi + |.define CARG3, rdx + |.define CARG4, rcx + |.define CARG5, r8 + |.define CARG6, r9 + |.define CARG1d, edi + |.define CARG2d, esi + |.define CARG3d, edx + |.define CARG4d, ecx + |.define CARG5d, r8d + |.define CARG6d, r9d + |.define FCARG1, CARG1d // Simulate x86 fastcall. + |.define FCARG2, CARG2d + |.define SPAD, 8 |.else - |.define FP, esi - |.define IP, edi - |.define IPl, edi + |.define FP, esi + |.define IP, edi + |.define IPl, edi + |.define FCARG1, ecx // x86 fastcall arguments. + |.define FCARG2, edx + |.define SPAD, 12 |.endif +|.type EX, zend_execute_data, FP +|.type OP, zend_opline, IP + |.actionlist dasm_actions |.globals lbl_ @@ -50,16 +71,45 @@ static void* dasm_labels[lbl__MAX]; static int zend_jit_interrupt_handler_stub(dasm_State **Dst) { - // TODO: check interrupt_handler ??? |->interrupt_handler: + | //EG(vm_interrupt) = 0; + | mov byte [&EG(vm_interrupt)], 0 + | //if (EG(timed_out)) { + | cmp byte [&EG(timed_out)], 0 + | je >1 + | //zend_timeout(0); |.if X64 - | add rsp, 8 // stack alignment - | load_addr rax, zend_timeout - | jmp rax + | mov CARG1d, 0 + | load_addr rax, zend_timeout + | call rax |.else - | add esp, 12 // stack alignment - | jmp extern zend_timeout + | push 0 + | call &zend_timeout + |.endif + |1: + | //} else if (zend_interrupt_function) { + | mov r0, aword [zend_interrupt_function] + | test r0, r0 + | je >2 + | //SAVE_OPLINE(); + | mov EX->opline, IP + | //zend_interrupt_function(execute_data); + |.if X64 + | mov CARG1, FP + |.else + | push FP |.endif + | call r0 + | //ZEND_VM_ENTER(); + | //execute_data = EG(current_execute_data); + | mov FP, aword [&EG(current_execute_data)] + | // LOAD_OPLINE(); + | mov IP, EX->opline + | //} + |2: + | //ZEND_VM_CONTINUE() + | add r4, SPAD // stack alignment + | jmp aword [IP] return 1; } From 0cd2a27f7d05a23a6b51f630c3f289de76365756 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 25 Aug 2016 12:51:15 +0300 Subject: [PATCH 048/569] Disable SSA construction and type inference for functions with exception handlers and generators. --- ext/opcache/jit/zend_jit.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index f14e3f9ce2634..64de470331015 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -289,6 +289,9 @@ ZEND_API int zend_jit(zend_op_array *op_array, zend_script *script) } if ((ZEND_JIT_LEVEL >= ZEND_JIT_LEVEL_FULL) + && ssa.cfg.blocks + && op_array->last_try_catch == 0 + && !(op_array->fn_flags & ZEND_ACC_GENERATOR) && !(flags & ZEND_FUNC_INDIRECT_VAR_ACCESS)) { if (zend_build_ssa(&CG(arena), script, op_array, ZEND_RT_CONSTANTS | ZEND_SSA_RC_INFERENCE, &ssa, &flags) != SUCCESS) { goto jit_failure; From e9492d32eea5c1d8dec1a1bc29a4f318731aa600 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Thu, 25 Aug 2016 19:20:15 +0800 Subject: [PATCH 049/569] Supportted disasm static symbols (x86_64 doesn't work now) --- ext/opcache/jit/Makefile.frag | 3 +- ext/opcache/jit/zend_elf.c | 139 ++++++++++++++++++++++++++ ext/opcache/jit/zend_elf.h | 116 +++++++++++++++++++++ ext/opcache/jit/zend_jit.c | 8 ++ ext/opcache/jit/zend_jit_disasm_x86.c | 4 +- ext/opcache/jit/zend_jit_gdb.c | 81 +-------------- 6 files changed, 269 insertions(+), 82 deletions(-) create mode 100644 ext/opcache/jit/zend_elf.c create mode 100644 ext/opcache/jit/zend_elf.h diff --git a/ext/opcache/jit/Makefile.frag b/ext/opcache/jit/Makefile.frag index dcacd3ad9041d..7f66dee0dcce4 100644 --- a/ext/opcache/jit/Makefile.frag +++ b/ext/opcache/jit/Makefile.frag @@ -10,5 +10,6 @@ $(builddir)/jit/zend_jit.lo: \ $(srcdir)/jit/zend_jit_disasm_x86.c \ $(srcdir)/jit/zend_jit_gdb.c \ $(srcdir)/jit/zend_jit_perf_dump.c \ - $(srcdir)/jit/zend_jit_oprofile.c + $(srcdir)/jit/zend_jit_oprofile.c \ + $(srcdir)/jit/zend_elf.c diff --git a/ext/opcache/jit/zend_elf.c b/ext/opcache/jit/zend_elf.c new file mode 100644 index 0000000000000..1ccc1e27835a0 --- /dev/null +++ b/ext/opcache/jit/zend_elf.c @@ -0,0 +1,139 @@ +/* + +----------------------------------------------------------------------+ + | Zend JIT | + +----------------------------------------------------------------------+ + | Copyright (c) 1998-2016 The PHP Group | + +----------------------------------------------------------------------+ + | This source file is subject to version 3.01 of the PHP license, | + | that is bundled with this package in the file LICENSE, and is | + | available through the world-wide-web at the following url: | + | http://www.php.net/license/3_01.txt | + | If you did not receive a copy of the PHP license and are unable to | + | obtain it through the world-wide-web, please send a note to | + | license@php.net so we can mail you a copy immediately. | + +----------------------------------------------------------------------+ + | Authors: Dmitry Stogov | + | Xinchen Hui | + +----------------------------------------------------------------------+ +*/ + +#include +#include +#include +#include + +#include "zend_API.h" +#include "zend_elf.h" + +static zend_array *symbols_table; + +static zend_elf_header *zend_elf_read_elfhdr(int fd, void *buf) { + if (read(fd, buf, sizeof(zend_elf_header)) == sizeof(zend_elf_header)) { + return (zend_elf_header*)buf; + } + return NULL; +} + +static int zend_elf_parse_sym_entry(zend_array *sym_table, char *str_tbl, zend_elf_symbol *sym) { + if (sym->name) { + zval zv; + ZVAL_STRING(&zv, (str_tbl + sym->name)); + zend_hash_index_update(sym_table, sym->value, &zv); + } + return 1; +} + +static void* zend_elf_read_sect(int fd, zend_elf_sectheader *sect) { + void *s = emalloc(sect->size); + + lseek(fd, sect->ofs, SEEK_SET); + if (read(fd, s, sect->size) != sect->size) { + efree(s); + return NULL; + } + + return s; +} + +static zend_elf_sectheader* zend_elf_load_sects(int fd, zend_elf_header *hdr) { + uint32_t i; + zend_elf_sectheader *sects = (zend_elf_sectheader *)emalloc(hdr->shentsize * hdr->shnum); + + lseek(fd, hdr->shofs, SEEK_SET); + for (i = 0; i < hdr->shnum; i++) { + read(fd, §s[i], hdr->shentsize); + } + + return sects; +} + +static zend_array* zend_elf_load_symbols(int fd, zend_elf_header *hdr) { + uint32_t i, ofs; + zend_array *symbols; + zend_elf_sectheader *sects; + + symbols = emalloc(sizeof(zend_array)); + zend_hash_init(symbols, 8, NULL, ZVAL_PTR_DTOR, 0); + + sects = zend_elf_load_sects(fd, hdr); + for (i = 0; i < hdr->shnum; i++) { + zend_elf_symbol *syms; + if (sects[i].type != ELFSECT_TYPE_SYMTAB) { + continue; + } + if ((syms = (zend_elf_symbol*)zend_elf_read_sect(fd, §s[i]))) { + uint32_t n, count = (sects[i].size) / sizeof(zend_elf_symbol); + char *str_tbl = (char*)zend_elf_read_sect(fd, §s[sects[i].link]); + ZEND_ASSERT(sects[i].entsize == sizeof(zend_elf_symbol)); + for (n = 0; n < count; n++) { + if ((syms[n].info & ELFSYM_TYPE_FUNC) && + !(syms[n].info & ELFSYM_BIND_GLOBAL)) { + zend_elf_parse_sym_entry(symbols, str_tbl, &syms[n]); + } + } + efree(str_tbl); + efree(syms); + } + } + + efree(sects); + + return symbols; +} + +void zend_elf_init(void) { + int fd = open("/proc/self/exe", O_RDONLY); + + if (fd) { + char buf[sizeof(zend_elf_header)]; + zend_elf_header *elfhdr = zend_elf_read_elfhdr(fd, buf); + + if (elfhdr) { + ZEND_ASSERT(strncmp((char*)elfhdr->emagic, "\177ELF", 4) == 0); + symbols_table = zend_elf_load_symbols(fd, elfhdr); + } + close(fd); + } +} + +void zend_elf_shutdown(void) { + zend_array_destroy(symbols_table); +} + +const char* zend_elf_resolve_sym(void *addr) { + zval *zv; + if (!symbols_table) { + return NULL; + } + if ((zv = zend_hash_index_find(symbols_table, (size_t)addr))) { + return Z_STRVAL_P(zv); + } +} + +/* + * Local variables: + * tab-width: 4 + * c-basic-offset: 4 + * indent-tabs-mode: t + * End: + */ diff --git a/ext/opcache/jit/zend_elf.h b/ext/opcache/jit/zend_elf.h new file mode 100644 index 0000000000000..1a0dfe6611ff7 --- /dev/null +++ b/ext/opcache/jit/zend_elf.h @@ -0,0 +1,116 @@ +/* + +----------------------------------------------------------------------+ + | Zend JIT | + +----------------------------------------------------------------------+ + | Copyright (c) 1998-2016 The PHP Group | + +----------------------------------------------------------------------+ + | This source file is subject to version 3.01 of the PHP license, | + | that is bundled with this package in the file LICENSE, and is | + | available through the world-wide-web at the following url: | + | http://www.php.net/license/3_01.txt | + | If you did not receive a copy of the PHP license and are unable to | + | obtain it through the world-wide-web, please send a note to | + | license@php.net so we can mail you a copy immediately. | + +----------------------------------------------------------------------+ + | Authors: Dmitry Stogov | + | Xinchen Hui | + +----------------------------------------------------------------------+ +*/ + +#ifndef ZEND_ELF +#define ZEND_ELF + +#if SIZEOF_SIZE_T == 8 +# define ELF64 +#else +# undef ELF64 +#endif + +typedef struct _zend_elf_header { + uint8_t emagic[4]; + uint8_t eclass; + uint8_t eendian; + uint8_t eversion; + uint8_t eosabi; + uint8_t eabiversion; + uint8_t epad[7]; + uint16_t type; + uint16_t machine; + uint32_t version; + uintptr_t entry; + uintptr_t phofs; + uintptr_t shofs; + uint32_t flags; + uint16_t ehsize; + uint16_t phentsize; + uint16_t phnum; + uint16_t shentsize; + uint16_t shnum; + uint16_t shstridx; +} zend_elf_header; + +typedef struct zend_elf_sectheader { + uint32_t name; + uint32_t type; + uintptr_t flags; + uintptr_t addr; + uintptr_t ofs; + uintptr_t size; + uint32_t link; + uint32_t info; + uintptr_t align; + uintptr_t entsize; +} zend_elf_sectheader; + +#define ELFSECT_IDX_ABS 0xfff1 + +enum { + ELFSECT_TYPE_PROGBITS = 1, + ELFSECT_TYPE_SYMTAB = 2, + ELFSECT_TYPE_STRTAB = 3, + ELFSECT_TYPE_NOBITS = 8, + ELFSECT_TYPE_DYNSYM = 11, +}; + +#define ELFSECT_FLAGS_WRITE 1 +#define ELFSECT_FLAGS_ALLOC 2 +#define ELFSECT_FLAGS_EXEC 4 + +typedef struct zend_elf_symbol { +#ifdef ELF64 + uint32_t name; + uint8_t info; + uint8_t other; + uint16_t sectidx; + uintptr_t value; + uint64_t size; +#else + uint32_t name; + uintptr_t value; + uint32_t size; + uint8_t info; + uint8_t other; + uint16_t sectidx; +#endif +} zend_elf_symbol; + +enum { + ELFSYM_TYPE_FUNC = 2, + ELFSYM_TYPE_FILE = 4, + ELFSYM_BIND_LOCAL = 0 << 4, + ELFSYM_BIND_GLOBAL = 1 << 4, +}; + +void zend_elf_init(void); +void zend_elf_shutdown(void); +const char* zend_elf_resolve_sym(void *addr); + +#endif + +/* + * Local variables: + * tab-width: 4 + * c-basic-offset: 4 + * indent-tabs-mode: t + * End: + */ diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 64de470331015..9793c9f1f7cee 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -59,6 +59,7 @@ typedef struct _zend_jit_stub { #ifdef HAVE_OPROFILE # include "jit/zend_jit_oprofile.c" #endif +#include "jit/zend_elf.c" #if _WIN32 # include @@ -588,6 +589,7 @@ ZEND_API int zend_jit_startup(size_t size) #ifdef HAVE_DISASM if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_ASM) { + zend_elf_init(); if (!zend_jit_disasm_init()) { // TODO: error reporting and cleanup ??? return FAILURE; @@ -621,6 +623,12 @@ ZEND_API void zend_jit_shutdown(void) } #endif +#ifdef HAVE_DISASM + if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_ASM) { + zend_elf_shutdown(); + } +#endif + if (dasm_buf) { jit_free(dasm_buf, ((char*)dasm_end) - ((char*)dasm_buf)); } diff --git a/ext/opcache/jit/zend_jit_disasm_x86.c b/ext/opcache/jit/zend_jit_disasm_x86.c index 6aafa062d5b0d..61c8fd50da0bb 100644 --- a/ext/opcache/jit/zend_jit_disasm_x86.c +++ b/ext/opcache/jit/zend_jit_disasm_x86.c @@ -19,6 +19,8 @@ #define HAVE_DISASM 1 #define DISASM_INTEL_SYNTAX 0 +#include "zend_elf.h" + #include "jit/libudis86/itab.c" #include "jit/libudis86/decode.c" #include "jit/libudis86/syn.c" @@ -50,7 +52,7 @@ static const char* zend_jit_disasm_resolver(struct ud *ud, return info.dli_sname; } - return NULL; + return zend_elf_resolve_sym((void*)addr); } static int zend_jit_disasm(const char *name, diff --git a/ext/opcache/jit/zend_jit_gdb.c b/ext/opcache/jit/zend_jit_gdb.c index 0814f85a86494..b3087e1aaa84e 100644 --- a/ext/opcache/jit/zend_jit_gdb.c +++ b/ext/opcache/jit/zend_jit_gdb.c @@ -19,14 +19,9 @@ #define HAVE_GDB +#include "zend_elf.h" #include "zend_gdb.h" -#if SIZEOF_SIZE_T == 8 -# define ELF64 -#else -# undef ELF64 -#endif - #if defined(__x86_64) #define CFRAME_SIZE (12*4) #define CFRAME_SIZE_JIT CFRAME_SIZE @@ -37,80 +32,6 @@ #error "Unsupported target architecture" #endif -typedef struct _zend_elf_header { - uint8_t emagic[4]; - uint8_t eclass; - uint8_t eendian; - uint8_t eversion; - uint8_t eosabi; - uint8_t eabiversion; - uint8_t epad[7]; - uint16_t type; - uint16_t machine; - uint32_t version; - uintptr_t entry; - uintptr_t phofs; - uintptr_t shofs; - uint32_t flags; - uint16_t ehsize; - uint16_t phentsize; - uint16_t phnum; - uint16_t shentsize; - uint16_t shnum; - uint16_t shstridx; -} zend_elf_header; - -typedef struct zend_elf_sectheader { - uint32_t name; - uint32_t type; - uintptr_t flags; - uintptr_t addr; - uintptr_t ofs; - uintptr_t size; - uint32_t link; - uint32_t info; - uintptr_t align; - uintptr_t entsize; -} zend_elf_sectheader; - -#define ELFSECT_IDX_ABS 0xfff1 - -enum { - ELFSECT_TYPE_PROGBITS = 1, - ELFSECT_TYPE_SYMTAB = 2, - ELFSECT_TYPE_STRTAB = 3, - ELFSECT_TYPE_NOBITS = 8 -}; - -#define ELFSECT_FLAGS_WRITE 1 -#define ELFSECT_FLAGS_ALLOC 2 -#define ELFSECT_FLAGS_EXEC 4 - -typedef struct zend_elf_symbol { -#ifdef ELF64 - uint32_t name; - uint8_t info; - uint8_t other; - uint16_t sectidx; - uintptr_t value; - uint64_t size; -#else - uint32_t name; - uintptr_t value; - uint32_t size; - uint8_t info; - uint8_t other; - uint16_t sectidx; -#endif -} zend_elf_symbol; - -enum { - ELFSYM_TYPE_FUNC = 2, - ELFSYM_TYPE_FILE = 4, - ELFSYM_BIND_LOCAL = 0 << 4, - ELFSYM_BIND_GLOBAL = 1 << 4, -}; - /* DWARF definitions. */ #define DW_CIE_VERSION 1 From f57ca3a56b528cf9a511926fc4361408d7c5e6dc Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 25 Aug 2016 14:28:01 +0300 Subject: [PATCH 050/569] Symbols should be in permanent memory --- ext/opcache/jit/zend_elf.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ext/opcache/jit/zend_elf.c b/ext/opcache/jit/zend_elf.c index 1ccc1e27835a0..6968826016cad 100644 --- a/ext/opcache/jit/zend_elf.c +++ b/ext/opcache/jit/zend_elf.c @@ -25,7 +25,7 @@ #include "zend_API.h" #include "zend_elf.h" -static zend_array *symbols_table; +static zend_array *symbols_table = NULL; static zend_elf_header *zend_elf_read_elfhdr(int fd, void *buf) { if (read(fd, buf, sizeof(zend_elf_header)) == sizeof(zend_elf_header)) { @@ -72,8 +72,8 @@ static zend_array* zend_elf_load_symbols(int fd, zend_elf_header *hdr) { zend_array *symbols; zend_elf_sectheader *sects; - symbols = emalloc(sizeof(zend_array)); - zend_hash_init(symbols, 8, NULL, ZVAL_PTR_DTOR, 0); + symbols = malloc(sizeof(zend_array)); + zend_hash_init(symbols, 8, NULL, ZVAL_PTR_DTOR, 1); sects = zend_elf_load_sects(fd, hdr); for (i = 0; i < hdr->shnum; i++) { From 9ad63573de5d9c36ef1b29f62539a5b770fd520a Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 25 Aug 2016 15:36:51 +0300 Subject: [PATCH 051/569] Disassembler fixes --- ext/opcache/jit/zend_elf.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/ext/opcache/jit/zend_elf.c b/ext/opcache/jit/zend_elf.c index 6968826016cad..469a12183fe69 100644 --- a/ext/opcache/jit/zend_elf.c +++ b/ext/opcache/jit/zend_elf.c @@ -37,6 +37,9 @@ static zend_elf_header *zend_elf_read_elfhdr(int fd, void *buf) { static int zend_elf_parse_sym_entry(zend_array *sym_table, char *str_tbl, zend_elf_symbol *sym) { if (sym->name) { zval zv; + zend_string *str = zend_string_init(str_tbl + sym->name, strlen(str_tbl + sym->name), 1); + + ZVAL_STR(&zv, str); ZVAL_STRING(&zv, (str_tbl + sym->name)); zend_hash_index_update(sym_table, sym->value, &zv); } @@ -68,7 +71,7 @@ static zend_elf_sectheader* zend_elf_load_sects(int fd, zend_elf_header *hdr) { } static zend_array* zend_elf_load_symbols(int fd, zend_elf_header *hdr) { - uint32_t i, ofs; + uint32_t i; zend_array *symbols; zend_elf_sectheader *sects; @@ -125,9 +128,8 @@ const char* zend_elf_resolve_sym(void *addr) { if (!symbols_table) { return NULL; } - if ((zv = zend_hash_index_find(symbols_table, (size_t)addr))) { - return Z_STRVAL_P(zv); - } + zv = zend_hash_index_find(symbols_table, (size_t)addr); + return zv ? Z_STRVAL_P(zv) : NULL; } /* From b274de8b4b7438b37ab9415f7a513b69a356acd4 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 25 Aug 2016 15:38:44 +0300 Subject: [PATCH 052/569] Allow resolving compiler_globals and executor_globals --- ext/opcache/jit/zend_jit_disasm_x86.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/ext/opcache/jit/zend_jit_disasm_x86.c b/ext/opcache/jit/zend_jit_disasm_x86.c index 61c8fd50da0bb..1d3ace8caeb98 100644 --- a/ext/opcache/jit/zend_jit_disasm_x86.c +++ b/ext/opcache/jit/zend_jit_disasm_x86.c @@ -43,9 +43,21 @@ static const char* zend_jit_disasm_resolver(struct ud *ud, uint64_t addr, int64_t *offset) { + ((void)ud); void *a = (void*)(zend_uintptr_t)(addr); Dl_info info; +#ifndef ZTS + if (a > (void*)&executor_globals && a < (void*)((char*)&executor_globals + sizeof(executor_globals))) { + *offset = (int64_t)((char*)a - (char*)&executor_globals); + return "executor_globals"; + } + if (a > (void*)&compiler_globals && a < (void*)((char*)&compiler_globals + sizeof(compiler_globals))) { + *offset = (int64_t)((char*)a - (char*)&compiler_globals); + return "compiler_globals"; + } +#endif + if (dladdr(a, &info) && info.dli_sname != NULL && info.dli_saddr == a) { From 4994d4482715810bb73eee4a921fbf38e7eb4579 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 25 Aug 2016 15:46:18 +0300 Subject: [PATCH 053/569] removed deprecated line --- ext/opcache/jit/zend_elf.c | 1 - 1 file changed, 1 deletion(-) diff --git a/ext/opcache/jit/zend_elf.c b/ext/opcache/jit/zend_elf.c index 469a12183fe69..1c2423a041354 100644 --- a/ext/opcache/jit/zend_elf.c +++ b/ext/opcache/jit/zend_elf.c @@ -40,7 +40,6 @@ static int zend_elf_parse_sym_entry(zend_array *sym_table, char *str_tbl, zend_e zend_string *str = zend_string_init(str_tbl + sym->name, strlen(str_tbl + sym->name), 1); ZVAL_STR(&zv, str); - ZVAL_STRING(&zv, (str_tbl + sym->name)); zend_hash_index_update(sym_table, sym->value, &zv); } return 1; From 31bc0878fa6018858e6a885d0bea4f296fb76fdd Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 25 Aug 2016 15:46:45 +0300 Subject: [PATCH 054/569] Call resolver for immediate and data address operands --- ext/opcache/jit/libudis86/syn.c | 42 +++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/ext/opcache/jit/libudis86/syn.c b/ext/opcache/jit/libudis86/syn.c index 1b9e1d42a5961..2e73362ee3933 100644 --- a/ext/opcache/jit/libudis86/syn.c +++ b/ext/opcache/jit/libudis86/syn.c @@ -171,6 +171,20 @@ ud_syn_print_imm(struct ud* u, const struct ud_operand *op) default: UD_ASSERT(!"invalid offset"); v = 0; /* keep cc happy */ } } +#if 1 + if (u->sym_resolver) { + int64_t offset = 0; + const char *name = u->sym_resolver(u, v, &offset); + if (name) { + if (offset) { + ud_asmprintf(u, "%s%+" FMT64 "d", name, offset); + } else { + ud_asmprintf(u, "%s", name); + } + return; + } + } +#endif ud_asmprintf(u, "0x%" FMT64 "x", v); } @@ -189,6 +203,20 @@ ud_syn_print_mem_disp(struct ud* u, const struct ud_operand *op, int sign) case 64: v = op->lval.uqword; break; default: UD_ASSERT(!"invalid offset"); v = 0; /* keep cc happy */ } +#if 1 + if (u->sym_resolver) { + int64_t offset = 0; + const char *name = u->sym_resolver(u, v, &offset); + if (name) { + if (offset) { + ud_asmprintf(u, "%s%+" FMT64 "d", name, offset); + } else { + ud_asmprintf(u, "%s", name); + } + return; + } + } +#endif ud_asmprintf(u, "0x%" FMT64 "x", v); } else { int64_t v; @@ -199,6 +227,20 @@ ud_syn_print_mem_disp(struct ud* u, const struct ud_operand *op, int sign) case 32: v = op->lval.sdword; break; default: UD_ASSERT(!"invalid offset"); v = 0; /* keep cc happy */ } +#if 1 + if (u->sym_resolver) { + int64_t offset = 0; + const char *name = u->sym_resolver(u, v, &offset); + if (name) { + if (offset) { + ud_asmprintf(u, "%s%+" FMT64 "d", name, offset); + } else { + ud_asmprintf(u, "%s", name); + } + return; + } + } +#endif if (v < 0) { ud_asmprintf(u, "-0x%" FMT64 "x", -v); } else if (v > 0) { From feaf65c593614428cd594768854e9ebe5316ed14 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 25 Aug 2016 17:00:57 +0300 Subject: [PATCH 055/569] Refactored symbol resolver --- ext/opcache/jit/zend_elf.c | 136 +++++++++----------------- ext/opcache/jit/zend_elf.h | 4 +- ext/opcache/jit/zend_jit.c | 5 +- ext/opcache/jit/zend_jit_disasm_x86.c | 32 +++++- 4 files changed, 80 insertions(+), 97 deletions(-) diff --git a/ext/opcache/jit/zend_elf.c b/ext/opcache/jit/zend_elf.c index 1c2423a041354..30ed92029c408 100644 --- a/ext/opcache/jit/zend_elf.c +++ b/ext/opcache/jit/zend_elf.c @@ -25,112 +25,68 @@ #include "zend_API.h" #include "zend_elf.h" -static zend_array *symbols_table = NULL; - -static zend_elf_header *zend_elf_read_elfhdr(int fd, void *buf) { - if (read(fd, buf, sizeof(zend_elf_header)) == sizeof(zend_elf_header)) { - return (zend_elf_header*)buf; - } - return NULL; -} - -static int zend_elf_parse_sym_entry(zend_array *sym_table, char *str_tbl, zend_elf_symbol *sym) { - if (sym->name) { - zval zv; - zend_string *str = zend_string_init(str_tbl + sym->name, strlen(str_tbl + sym->name), 1); - - ZVAL_STR(&zv, str); - zend_hash_index_update(sym_table, sym->value, &zv); - } - return 1; -} - -static void* zend_elf_read_sect(int fd, zend_elf_sectheader *sect) { +static void* zend_elf_read_sect(int fd, zend_elf_sectheader *sect) +{ void *s = emalloc(sect->size); - lseek(fd, sect->ofs, SEEK_SET); - if (read(fd, s, sect->size) != sect->size) { + if (lseek(fd, sect->ofs, SEEK_SET) < 0) { efree(s); return NULL; } - - return s; -} - -static zend_elf_sectheader* zend_elf_load_sects(int fd, zend_elf_header *hdr) { - uint32_t i; - zend_elf_sectheader *sects = (zend_elf_sectheader *)emalloc(hdr->shentsize * hdr->shnum); - - lseek(fd, hdr->shofs, SEEK_SET); - for (i = 0; i < hdr->shnum; i++) { - read(fd, §s[i], hdr->shentsize); - } - - return sects; -} - -static zend_array* zend_elf_load_symbols(int fd, zend_elf_header *hdr) { - uint32_t i; - zend_array *symbols; - zend_elf_sectheader *sects; - - symbols = malloc(sizeof(zend_array)); - zend_hash_init(symbols, 8, NULL, ZVAL_PTR_DTOR, 1); - - sects = zend_elf_load_sects(fd, hdr); - for (i = 0; i < hdr->shnum; i++) { - zend_elf_symbol *syms; - if (sects[i].type != ELFSECT_TYPE_SYMTAB) { - continue; - } - if ((syms = (zend_elf_symbol*)zend_elf_read_sect(fd, §s[i]))) { - uint32_t n, count = (sects[i].size) / sizeof(zend_elf_symbol); - char *str_tbl = (char*)zend_elf_read_sect(fd, §s[sects[i].link]); - ZEND_ASSERT(sects[i].entsize == sizeof(zend_elf_symbol)); - for (n = 0; n < count; n++) { - if ((syms[n].info & ELFSYM_TYPE_FUNC) && - !(syms[n].info & ELFSYM_BIND_GLOBAL)) { - zend_elf_parse_sym_entry(symbols, str_tbl, &syms[n]); - } - } - efree(str_tbl); - efree(syms); - } + if (read(fd, s, sect->size) != (ssize_t)sect->size) { + efree(s); + return NULL; } - efree(sects); - - return symbols; + return s; } -void zend_elf_init(void) { +void zend_elf_load_symbols(void) +{ + zend_elf_header hdr; + zend_elf_sectheader sect; + int i; int fd = open("/proc/self/exe", O_RDONLY); - if (fd) { - char buf[sizeof(zend_elf_header)]; - zend_elf_header *elfhdr = zend_elf_read_elfhdr(fd, buf); - - if (elfhdr) { - ZEND_ASSERT(strncmp((char*)elfhdr->emagic, "\177ELF", 4) == 0); - symbols_table = zend_elf_load_symbols(fd, elfhdr); + if (fd >= 0) { + if (read(fd, &hdr, sizeof(hdr)) == sizeof(hdr) + && hdr.emagic[0] == '\177' + && hdr.emagic[1] == 'E' + && hdr.emagic[2] == 'L' + && hdr.emagic[3] == 'F' + && lseek(fd, hdr.shofs, SEEK_SET) >= 0) { + for (i = 0; i < hdr.shnum; i++) { + if (read(fd, §, sizeof(sect)) == sizeof(sect) + && sect.type == ELFSECT_TYPE_SYMTAB) { + uint32_t n, count = sect.size / sizeof(zend_elf_symbol); + zend_elf_symbol *syms = zend_elf_read_sect(fd, §); + char *str_tbl; + + if (syms) { + if (lseek(fd, hdr.shofs + sect.link * sizeof(sect), SEEK_SET) >= 0 + && read(fd, §, sizeof(sect)) == sizeof(sect) + && (str_tbl = (char*)zend_elf_read_sect(fd, §)) != NULL) { + for (n = 0; n < count; n++) { + if (syms[n].name + && (syms[n].info & ELFSYM_TYPE_FUNC) + && !(syms[n].info & ELFSYM_BIND_GLOBAL)) { + zend_jit_disasm_add_symbol(str_tbl + syms[n].name, syms[n].value); + } + } + efree(str_tbl); + } + efree(syms); + } + if (lseek(fd, hdr.shofs + (i + 1) * sizeof(sect), SEEK_SET) < 0) { + break; + } + } + } } close(fd); } } -void zend_elf_shutdown(void) { - zend_array_destroy(symbols_table); -} - -const char* zend_elf_resolve_sym(void *addr) { - zval *zv; - if (!symbols_table) { - return NULL; - } - zv = zend_hash_index_find(symbols_table, (size_t)addr); - return zv ? Z_STRVAL_P(zv) : NULL; -} - /* * Local variables: * tab-width: 4 diff --git a/ext/opcache/jit/zend_elf.h b/ext/opcache/jit/zend_elf.h index 1a0dfe6611ff7..87d9cc9fe5148 100644 --- a/ext/opcache/jit/zend_elf.h +++ b/ext/opcache/jit/zend_elf.h @@ -101,9 +101,7 @@ enum { ELFSYM_BIND_GLOBAL = 1 << 4, }; -void zend_elf_init(void); -void zend_elf_shutdown(void); -const char* zend_elf_resolve_sym(void *addr); +void zend_elf_load_symbols(void); #endif diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 9793c9f1f7cee..1876a6302b167 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -59,7 +59,6 @@ typedef struct _zend_jit_stub { #ifdef HAVE_OPROFILE # include "jit/zend_jit_oprofile.c" #endif -#include "jit/zend_elf.c" #if _WIN32 # include @@ -148,6 +147,7 @@ static void *dasm_link_and_encode(dasm_State **dasm_state, #ifdef HAVE_DISASM if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_ASM) { + zend_jit_disasm_add_symbol(name, (uintptr_t)entry); zend_jit_disasm( name, (op_array && op_array->filename) ? ZSTR_VAL(op_array->filename) : NULL, @@ -589,7 +589,6 @@ ZEND_API int zend_jit_startup(size_t size) #ifdef HAVE_DISASM if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_ASM) { - zend_elf_init(); if (!zend_jit_disasm_init()) { // TODO: error reporting and cleanup ??? return FAILURE; @@ -625,7 +624,7 @@ ZEND_API void zend_jit_shutdown(void) #ifdef HAVE_DISASM if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_ASM) { - zend_elf_shutdown(); + zend_jit_disasm_shutdown(); } #endif diff --git a/ext/opcache/jit/zend_jit_disasm_x86.c b/ext/opcache/jit/zend_jit_disasm_x86.c index 1d3ace8caeb98..3f450e7cee45f 100644 --- a/ext/opcache/jit/zend_jit_disasm_x86.c +++ b/ext/opcache/jit/zend_jit_disasm_x86.c @@ -31,6 +31,11 @@ #endif #include "jit/libudis86/udis86.c" +static void zend_jit_disasm_add_symbol(const char *name, + uint64_t addr); + +#include "jit/zend_elf.c" + #ifndef _GNU_SOURCE # define _GNU_SOURCE #endif @@ -38,6 +43,17 @@ #include static struct ud ud; +static HashTable disasm_symbols; + +static void zend_jit_disasm_add_symbol(const char *name, + uint64_t addr) +{ + zval zv; + zend_string *str = zend_string_init(name, strlen(name), 1); + + ZVAL_STR(&zv, str); + zend_hash_index_add(&disasm_symbols, addr, &zv); +} static const char* zend_jit_disasm_resolver(struct ud *ud, uint64_t addr, @@ -45,6 +61,7 @@ static const char* zend_jit_disasm_resolver(struct ud *ud, { ((void)ud); void *a = (void*)(zend_uintptr_t)(addr); + zval *zv; Dl_info info; #ifndef ZTS @@ -58,13 +75,18 @@ static const char* zend_jit_disasm_resolver(struct ud *ud, } #endif + zv = zend_hash_index_find(&disasm_symbols, addr); + if (zv) { + return Z_STRVAL_P(zv); + } + if (dladdr(a, &info) && info.dli_sname != NULL && info.dli_saddr == a) { return info.dli_sname; } - return zend_elf_resolve_sym((void*)addr); + return NULL; } static int zend_jit_disasm(const char *name, @@ -103,9 +125,17 @@ static int zend_jit_disasm_init(void) #endif ud_set_sym_resolver(&ud, zend_jit_disasm_resolver); + zend_hash_init(&disasm_symbols, 8, NULL, ZVAL_PTR_DTOR, 1); + zend_elf_load_symbols(); + return 1; } +static void zend_jit_disasm_shutdown(void) +{ + zend_hash_destroy(&disasm_symbols); +} + /* * Local variables: * tab-width: 4 From b61b227171efc9a1b5e9c1cc2768dd5ad1cc5800 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 25 Aug 2016 17:27:42 +0300 Subject: [PATCH 056/569] cleanup & white spaces --- ext/opcache/jit/zend_jit_x86.dasc | 82 ++++++++++++++----------------- 1 file changed, 38 insertions(+), 44 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index d003bbaea88ea..1bd0d1a1d4d51 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -117,13 +117,12 @@ static int zend_jit_interrupt_handler_stub(dasm_State **Dst) static int zend_jit_exception_handler_stub(dasm_State **Dst) { |->exception_handler: + | add r4, SPAD // stack alignment |.if X64 - | add rsp, 8 // stack alignment - | load_addr rax, EG(exception_op)->handler - | jmp rax + | load_addr rax, EG(exception_op)->handler + | jmp rax |.else - | add esp, 12 // stack alignment - | jmp &(EG(exception_op)->handler) + | jmp &(EG(exception_op)->handler) |.endif return 1; @@ -142,11 +141,7 @@ static int zend_jit_align_func(dasm_State **Dst) static int zend_jit_prologue(dasm_State **Dst) { - |.if X64 - | sub rsp, 8 // stack alignment - |.else - | sub esp, 12 // stack alignment - |.endif + | sub r4, SPAD // stack alignment return 1; } @@ -158,15 +153,15 @@ static int zend_jit_label(dasm_State **Dst, unsigned int label) static int zend_jit_check_timeout(dasm_State **Dst) { - | cmp byte [&EG(vm_interrupt)], 0 - | jne ->interrupt_handler + | cmp byte [&EG(vm_interrupt)], 0 + | jne ->interrupt_handler return 1; } static int zend_jit_check_exception(dasm_State **Dst) { - | cmp aword [&EG(exception)], 0 - | jne ->exception_handler + | cmp aword [&EG(exception)], 0 + | jne ->exception_handler return 1; } @@ -175,10 +170,10 @@ static int zend_jit_handler(dasm_State **Dst, zend_op *opline) const void *handler = opline->handler; |.if X64 - | load_addr rax, handler - | call rax + | load_addr rax, handler + | call rax |.else - | call aword &handler + | call aword &handler |.endif zend_jit_check_exception(Dst); return 1; @@ -188,62 +183,61 @@ static int zend_jit_tail_handler(dasm_State **Dst, zend_op *opline) { const void *handler = opline->handler; + | add r4, SPAD // stack alignment |.if X64 - | add rsp, 8 // stack alignment - | load_addr rax, handler - | jmp rax + | load_addr rax, handler + | jmp rax |.else - | add esp, 12 // stack alignment - | jmp aword &handler + | jmp aword &handler |.endif return 1; } static int zend_jit_skip_handler(dasm_State **Dst, uint32_t skip) { - | add IP, sizeof(zend_op) * skip + | add IP, sizeof(zend_op) * skip return 1; } static int zend_jit_set_opline(dasm_State **Dst, zend_op *target_opline) { |.if X64 - | load_addr IP, target_opline + | load_addr IP, target_opline |.else - | mov IP, target_opline + | mov IP, target_opline |.endif return 1; } static int zend_jit_jmp(dasm_State **Dst, unsigned int target_label) { - | jmp =>target_label + | jmp =>target_label return 1; } static int zend_jit_cond_jmp(dasm_State **Dst, zend_op *next_opline, unsigned int target_label) { - | cmp IPl, next_opline - | jnz =>target_label + | cmp IPl, next_opline + | jnz =>target_label return 1; } static int zend_jit_smart_branch(dasm_State **Dst, zend_op *opline, unsigned int next_label, unsigned int target_label) { - zend_op *next_opline = opline + 1; - zend_op *target_opline = OP_JMP_ADDR(opline, opline->op2); + zend_op *next_opline = opline + 1; + zend_op *target_opline = OP_JMP_ADDR(opline, opline->op2); - | cmp IPl, next_opline - | jz =>next_label - | cmp IPl, target_opline - | jz =>target_label + | cmp IPl, next_opline + | jz =>next_label + | cmp IPl, target_opline + | jz =>target_label return 1; } static int zend_jit_recv(dasm_State **Dst, zend_op *opline) { - | cmp IPl, opline - | jnz >1 + | cmp IPl, opline + | jnz >1 zend_jit_handler(Dst, opline); |1: return 1; @@ -254,13 +248,13 @@ static int zend_jit_context_threaded_call(dasm_State **Dst, zend_op *opline) { if (!zend_jit_handler(Dst, opline)) return 0; if (opline->opcode == ZEND_DO_UCALL) { - | call aword [IP] + | call aword [IP] } else { - zend_op *next_opline = opline + 1; + zend_op *next_opline = opline + 1; - | cmp IPl, next_opline - | jz >1 - | call aword [IP] + | cmp IPl, next_opline + | jz >1 + | call aword [IP] |1: } return 1; @@ -280,10 +274,10 @@ static int zend_jit_new(dasm_State **Dst, zend_op *opline) { if (!zend_jit_handler(Dst, opline)) return 0; if (EXPECTED(opline->extended_value == 0 && (opline+1)->opcode == ZEND_DO_FCALL)) { - zend_op *next_opline = opline + 1; + zend_op *next_opline = opline + 1; - | cmp IPl, next_opline - | jnz >1 + | cmp IPl, next_opline + | jnz >1 zend_jit_call(Dst, next_opline); |1: } From 0718524c02f15af7794a8764fbf4f7bbaf4bdded Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Thu, 25 Aug 2016 22:42:47 +0800 Subject: [PATCH 057/569] Check return --- ext/opcache/jit/zend_jit_disasm_x86.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_disasm_x86.c b/ext/opcache/jit/zend_jit_disasm_x86.c index 3f450e7cee45f..086b71da067eb 100644 --- a/ext/opcache/jit/zend_jit_disasm_x86.c +++ b/ext/opcache/jit/zend_jit_disasm_x86.c @@ -52,7 +52,9 @@ static void zend_jit_disasm_add_symbol(const char *name, zend_string *str = zend_string_init(name, strlen(name), 1); ZVAL_STR(&zv, str); - zend_hash_index_add(&disasm_symbols, addr, &zv); + if (zend_hash_index_add(&disasm_symbols, addr, &zv) == NULL) { + zend_string_release(str); + } } static const char* zend_jit_disasm_resolver(struct ud *ud, From 7066bf7209f057bae94bd7d77ad2c9ed40f6b72f Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Thu, 25 Aug 2016 22:46:44 +0800 Subject: [PATCH 058/569] Missed define --- Zend/zend_gdb.h | 1 + 1 file changed, 1 insertion(+) diff --git a/Zend/zend_gdb.h b/Zend/zend_gdb.h index 616e36b971877..f2dddb05d4e3a 100644 --- a/Zend/zend_gdb.h +++ b/Zend/zend_gdb.h @@ -18,6 +18,7 @@ */ #ifndef ZEND_GDB +#define ZEND_GDB ZEND_API int zend_gdb_register_code(const void *object, size_t size); ZEND_API void zend_gdb_unregister_all(void); From 5b1343108c3a177f74d98a10256f999c6b94dc02 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 25 Aug 2016 19:08:32 +0300 Subject: [PATCH 059/569] Fixed invalid address calculation in 64-bit mode --- ext/opcache/jit/libudis86/syn.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ext/opcache/jit/libudis86/syn.c b/ext/opcache/jit/libudis86/syn.c index 2e73362ee3933..c7453d63b7821 100644 --- a/ext/opcache/jit/libudis86/syn.c +++ b/ext/opcache/jit/libudis86/syn.c @@ -91,7 +91,11 @@ const char* ud_reg_tab[] = uint64_t ud_syn_rel_target(struct ud *u, struct ud_operand *opr) { +#if 1 + const uint64_t trunc_mask = 0xffffffffffffffffull >> (64 - u->adr_mode); +#else const uint64_t trunc_mask = 0xffffffffffffffffull >> (64 - u->opr_mode); +#endif switch (opr->size) { case 8 : return (u->pc + opr->lval.sbyte) & trunc_mask; case 16: return (u->pc + opr->lval.sword) & trunc_mask; From 5b4bd0363eaffb53ac9f72e7dd78d98211cda980 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 25 Aug 2016 19:10:29 +0300 Subject: [PATCH 060/569] Use local labels instead of absolute addresses in disassembler output --- ext/opcache/jit/zend_jit_disasm_x86.c | 72 +++++++++++++++++++++++++-- 1 file changed, 69 insertions(+), 3 deletions(-) diff --git a/ext/opcache/jit/zend_jit_disasm_x86.c b/ext/opcache/jit/zend_jit_disasm_x86.c index 086b71da067eb..6a129c78dc243 100644 --- a/ext/opcache/jit/zend_jit_disasm_x86.c +++ b/ext/opcache/jit/zend_jit_disasm_x86.c @@ -19,8 +19,6 @@ #define HAVE_DISASM 1 #define DISASM_INTEL_SYNTAX 0 -#include "zend_elf.h" - #include "jit/libudis86/itab.c" #include "jit/libudis86/decode.c" #include "jit/libudis86/syn.c" @@ -36,6 +34,8 @@ static void zend_jit_disasm_add_symbol(const char *name, #include "jit/zend_elf.c" +#include "zend_sort.h" + #ifndef _GNU_SOURCE # define _GNU_SOURCE #endif @@ -91,12 +91,24 @@ static const char* zend_jit_disasm_resolver(struct ud *ud, return NULL; } +static int zend_jit_cmp_labels(Bucket *b1, Bucket *b2) +{ + return ((b1->h > b2->h) > 0) ? 1 : -1; +} + + static int zend_jit_disasm(const char *name, const char *filename, const void *start, size_t size) { const void *end = (void *)((char *)start + size); + zval zv, *z; + zend_long n; + HashTable labels; + const struct ud_operand *op; + uint64_t addr; + if (name) { fprintf(stderr, "%s: ; (%s)\n", name, filename ? filename : "unknown"); } @@ -104,11 +116,65 @@ static int zend_jit_disasm(const char *name, ud_set_input_buffer(&ud, (uint8_t*)start, (uint8_t*)end - (uint8_t*)start); ud_set_pc(&ud, (uint64_t)(uintptr_t)start); + zend_hash_init(&labels, 8, NULL, NULL, 0); + ZVAL_NULL(&zv); + while (ud_disassemble(&ud)) { + op = ud_insn_opr(&ud, 0); + if (op && op->type == UD_OP_JIMM) { + addr = ud_syn_rel_target(&ud, (struct ud_operand*)op); + if (addr >= (uint64_t)(uintptr_t)start && addr < (uint64_t)(uintptr_t)end) { + zend_hash_index_add(&labels, addr, &zv); + } + } + } + + zend_hash_sort(&labels, (compare_func_t)zend_jit_cmp_labels, 0); + + /* label numbering */ + n = 0; + ZEND_HASH_FOREACH_VAL(&labels, z) { + n++; + ZVAL_LONG(z, n); + } ZEND_HASH_FOREACH_END(); + + ud_set_input_buffer(&ud, (uint8_t*)start, (uint8_t*)end - (uint8_t*)start); + ud_set_pc(&ud, (uint64_t)(uintptr_t)start); + while (ud_disassemble(&ud)) { - fprintf(stderr, "0x%" PRIx64 ":\t%s\n", ud_insn_off(&ud), ud_insn_asm(&ud)); + addr = ud_insn_off(&ud); + z = zend_hash_index_find(&labels, addr); + if (z) { + fprintf(stderr, ".L%ld:\n", Z_LVAL_P(z)); + } + op = ud_insn_opr(&ud, 0); + if (op && op->type == UD_OP_JIMM) { + addr = ud_syn_rel_target(&ud, (struct ud_operand*)op); + if (addr >= (uint64_t)(uintptr_t)start && addr < (uint64_t)(uintptr_t)end) { + z = zend_hash_index_find(&labels, addr); + if (z) { + const char *str = ud_insn_asm(&ud); + int len; + + len = 0; + while (str[len] != 0 && str[len] != ' ' && str[len] != '\t') { + len++; + } + if (str[len] != 0) { + while (str[len] == ' ' || str[len] == '\t') { + len++; + } + fprintf(stderr, "\t%.*s.L%ld\n", len, str, Z_LVAL_P(z)); + continue; + } + } + } + } + fprintf(stderr, "\t%s\n", ud_insn_asm(&ud)); } fprintf(stderr, "\n"); + zend_hash_destroy(&labels); + return 1; } From 77a925189f8f4e0c292e7e38e85d6dc01492246f Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Fri, 26 Aug 2016 13:17:52 +0800 Subject: [PATCH 061/569] Refactor zend_jit_disasm_add_symbol --- ext/opcache/jit/zend_elf.c | 2 +- ext/opcache/jit/zend_jit.c | 2 +- ext/opcache/jit/zend_jit_disasm_x86.c | 193 ++++++++++++++++++++++---- 3 files changed, 168 insertions(+), 29 deletions(-) diff --git a/ext/opcache/jit/zend_elf.c b/ext/opcache/jit/zend_elf.c index 30ed92029c408..b30b2db671336 100644 --- a/ext/opcache/jit/zend_elf.c +++ b/ext/opcache/jit/zend_elf.c @@ -70,7 +70,7 @@ void zend_elf_load_symbols(void) if (syms[n].name && (syms[n].info & ELFSYM_TYPE_FUNC) && !(syms[n].info & ELFSYM_BIND_GLOBAL)) { - zend_jit_disasm_add_symbol(str_tbl + syms[n].name, syms[n].value); + zend_jit_disasm_add_symbol(str_tbl + syms[n].name, syms[n].value, 0); } } efree(str_tbl); diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 1876a6302b167..7310afc0adb72 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -147,7 +147,7 @@ static void *dasm_link_and_encode(dasm_State **dasm_state, #ifdef HAVE_DISASM if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_ASM) { - zend_jit_disasm_add_symbol(name, (uintptr_t)entry); + zend_jit_disasm_add_symbol(name, (uintptr_t)entry, size); zend_jit_disasm( name, (op_array && op_array->filename) ? ZSTR_VAL(op_array->filename) : NULL, diff --git a/ext/opcache/jit/zend_jit_disasm_x86.c b/ext/opcache/jit/zend_jit_disasm_x86.c index 6a129c78dc243..faa393047e39c 100644 --- a/ext/opcache/jit/zend_jit_disasm_x86.c +++ b/ext/opcache/jit/zend_jit_disasm_x86.c @@ -13,6 +13,7 @@ | license@php.net so we can mail you a copy immediately. | +----------------------------------------------------------------------+ | Authors: Dmitry Stogov | + | Xinchen Hui | +----------------------------------------------------------------------+ */ @@ -30,7 +31,8 @@ #include "jit/libudis86/udis86.c" static void zend_jit_disasm_add_symbol(const char *name, - uint64_t addr); + uint64_t addr, + uint64_t size); #include "jit/zend_elf.c" @@ -43,18 +45,165 @@ static void zend_jit_disasm_add_symbol(const char *name, #include static struct ud ud; -static HashTable disasm_symbols; + +typedef struct _sym_node { + uint64_t addr; + uint64_t size; + struct _sym_node *parent; + struct _sym_node *child[2]; + unsigned char info; + unsigned char name[1]; +} zend_sym_node; + +static zend_sym_node *symbols = NULL; + +static void zend_syms_rotateleft(zend_sym_node *p) { + zend_sym_node *r = p->child[1]; + p->child[1] = r->child[0]; + if (r->child[0]) { + r->child[0]->parent = p; + } + r->parent = p->parent; + if (p->parent == NULL) { + symbols = r; + } else if (p->parent->child[0] == p) { + p->parent->child[0] = r; + } else { + p->parent->child[1] = r; + } + r->child[0] = p; + p->parent = r; +} + +static void zend_syms_rotateright(zend_sym_node *p) { + zend_sym_node *l = p->child[0]; + p->child[0] = l->child[1]; + if (l->child[1]) { + l->child[1]->parent = p; + } + l->parent = p->parent; + if (p->parent == NULL) { + symbols = l; + } else if (p->parent->child[1] == p) { + p->parent->child[1] = l; + } else { + p->parent->child[0] = l; + } + l->child[1] = p; + p->parent = l; +} static void zend_jit_disasm_add_symbol(const char *name, - uint64_t addr) + uint64_t addr, + uint64_t size) { - zval zv; - zend_string *str = zend_string_init(name, strlen(name), 1); + zend_sym_node *sym; + size_t len = strlen(name); - ZVAL_STR(&zv, str); - if (zend_hash_index_add(&disasm_symbols, addr, &zv) == NULL) { - zend_string_release(str); + sym = malloc(sizeof(zend_sym_node) + len + 1); + if (!sym) { + return; } + sym->addr = addr; + sym->size = size; + memcpy((char*)&sym->name, name, len + 1); + sym->parent = sym->child[0] = sym->child[1] = NULL; + sym->info = 1; + if (symbols) { + zend_sym_node *node = symbols; + + /* insert it into rbtree */ + do { + if (sym->addr > node->addr) { + if (node->child[1]) { + node = node->child[1]; + } else { + node->child[1] = sym; + sym->parent = node; + break; + } + } else if (sym->addr < node->addr) { + if (node->child[0]) { + node = node->child[0]; + } else { + node->child[0] = sym; + sym->parent = node; + break; + } + } else { + ZEND_ASSERT(sym->addr == node->addr); + free(sym); + return; + } + } while (1); + + /* fix rbtree after instering */ + while (sym && sym != symbols && sym->parent->info == 1) { + if (sym->parent == sym->parent->parent->child[0]) { + node = sym->parent->parent->child[1]; + if (node && node->info == 1) { + sym->parent->info = 0; + node->info = 0; + sym->parent->parent->info = 1; + sym = sym->parent->parent; + } else { + if (sym == sym->parent->child[1]) { + sym = sym->parent; + zend_syms_rotateleft(sym); + } + sym->parent->info = 0; + sym->parent->parent->info = 1; + zend_syms_rotateright(sym->parent->parent); + } + } else { + node = sym->parent->parent->child[0]; + if (node && node->info == 1) { + sym->parent->info = 0; + node->info = 0; + sym->parent->parent->info = 1; + sym = sym->parent->parent; + } else { + if (sym == sym->parent->child[0]) { + sym = sym->parent; + zend_syms_rotateright(sym); + } + sym->parent->info = 0; + sym->parent->parent->info = 1; + zend_syms_rotateleft(sym->parent->parent); + } + } + } + } else { + symbols = sym; + } + symbols->info = 0; +} + +static void zend_jit_disasm_destroy_symbols(zend_sym_node *n) { + if (n) { + if (n->child[0]) { + zend_jit_disasm_destroy_symbols(n->child[0]); + } else if (n->child[1]) { + zend_jit_disasm_destroy_symbols(n->child[1]); + } + free(n); + } +} + +static const char* zend_jit_disasm_find_symbol(uint64_t addr, + int64_t *offset) { + zend_sym_node *node = symbols; + while (node) { + if (addr < node->addr) { + node = node->child[0]; + } else if (addr > (node->addr + node->size)) { + node = node->child[1]; + } else { + *offset = addr - node->addr; + return node->name; + } + } + return NULL; } static const char* zend_jit_disasm_resolver(struct ud *ud, @@ -62,24 +211,13 @@ static const char* zend_jit_disasm_resolver(struct ud *ud, int64_t *offset) { ((void)ud); + const char *name; void *a = (void*)(zend_uintptr_t)(addr); - zval *zv; Dl_info info; -#ifndef ZTS - if (a > (void*)&executor_globals && a < (void*)((char*)&executor_globals + sizeof(executor_globals))) { - *offset = (int64_t)((char*)a - (char*)&executor_globals); - return "executor_globals"; - } - if (a > (void*)&compiler_globals && a < (void*)((char*)&compiler_globals + sizeof(compiler_globals))) { - *offset = (int64_t)((char*)a - (char*)&compiler_globals); - return "compiler_globals"; - } -#endif - - zv = zend_hash_index_find(&disasm_symbols, addr); - if (zv) { - return Z_STRVAL_P(zv); + name = zend_jit_disasm_find_symbol(addr, offset); + if (name) { + return name; } if (dladdr(a, &info) @@ -96,7 +234,6 @@ static int zend_jit_cmp_labels(Bucket *b1, Bucket *b2) return ((b1->h > b2->h) > 0) ? 1 : -1; } - static int zend_jit_disasm(const char *name, const char *filename, const void *start, @@ -192,8 +329,10 @@ static int zend_jit_disasm_init(void) ud_set_syntax(&ud, UD_SYN_ATT); #endif ud_set_sym_resolver(&ud, zend_jit_disasm_resolver); - - zend_hash_init(&disasm_symbols, 8, NULL, ZVAL_PTR_DTOR, 1); +#ifndef ZTS + zend_jit_disasm_add_symbol("executor_globals", (uint64_t)&executor_globals, sizeof(executor_globals)); + zend_jit_disasm_add_symbol("compiler_globals", (uint64_t)&compiler_globals, sizeof(compiler_globals)); +#endif zend_elf_load_symbols(); return 1; @@ -201,7 +340,7 @@ static int zend_jit_disasm_init(void) static void zend_jit_disasm_shutdown(void) { - zend_hash_destroy(&disasm_symbols); + zend_jit_disasm_destroy_symbols(symbols); } /* From 7ad935ff4d1fbe2bbe83d29836f06296d66a72f5 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 26 Aug 2016 08:34:38 +0300 Subject: [PATCH 062/569] Use Elf symbol size --- ext/opcache/jit/zend_elf.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_elf.c b/ext/opcache/jit/zend_elf.c index b30b2db671336..ac0287f341c35 100644 --- a/ext/opcache/jit/zend_elf.c +++ b/ext/opcache/jit/zend_elf.c @@ -70,7 +70,7 @@ void zend_elf_load_symbols(void) if (syms[n].name && (syms[n].info & ELFSYM_TYPE_FUNC) && !(syms[n].info & ELFSYM_BIND_GLOBAL)) { - zend_jit_disasm_add_symbol(str_tbl + syms[n].name, syms[n].value, 0); + zend_jit_disasm_add_symbol(str_tbl + syms[n].name, syms[n].value, syms[n].size); } } efree(str_tbl); From 109dabbb9639880c80a7d78a63649a1ebc32c445 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 26 Aug 2016 08:34:58 +0300 Subject: [PATCH 063/569] Fixed compilation warnings --- ext/opcache/jit/zend_jit_disasm_x86.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ext/opcache/jit/zend_jit_disasm_x86.c b/ext/opcache/jit/zend_jit_disasm_x86.c index faa393047e39c..0e25edb4e456e 100644 --- a/ext/opcache/jit/zend_jit_disasm_x86.c +++ b/ext/opcache/jit/zend_jit_disasm_x86.c @@ -52,7 +52,7 @@ typedef struct _sym_node { struct _sym_node *parent; struct _sym_node *child[2]; unsigned char info; - unsigned char name[1]; + char name[1]; } zend_sym_node; static zend_sym_node *symbols = NULL; @@ -281,7 +281,7 @@ static int zend_jit_disasm(const char *name, addr = ud_insn_off(&ud); z = zend_hash_index_find(&labels, addr); if (z) { - fprintf(stderr, ".L%ld:\n", Z_LVAL_P(z)); + fprintf(stderr, ".L" ZEND_LONG_FMT ":\n", Z_LVAL_P(z)); } op = ud_insn_opr(&ud, 0); if (op && op->type == UD_OP_JIMM) { @@ -300,7 +300,7 @@ static int zend_jit_disasm(const char *name, while (str[len] == ' ' || str[len] == '\t') { len++; } - fprintf(stderr, "\t%.*s.L%ld\n", len, str, Z_LVAL_P(z)); + fprintf(stderr, "\t%.*s.L" ZEND_LONG_FMT "\n", len, str, Z_LVAL_P(z)); continue; } } @@ -330,8 +330,8 @@ static int zend_jit_disasm_init(void) #endif ud_set_sym_resolver(&ud, zend_jit_disasm_resolver); #ifndef ZTS - zend_jit_disasm_add_symbol("executor_globals", (uint64_t)&executor_globals, sizeof(executor_globals)); - zend_jit_disasm_add_symbol("compiler_globals", (uint64_t)&compiler_globals, sizeof(compiler_globals)); + zend_jit_disasm_add_symbol("executor_globals", (uint64_t)(uintptr_t)&executor_globals, sizeof(executor_globals)); + zend_jit_disasm_add_symbol("compiler_globals", (uint64_t)(uintptr_t)&compiler_globals, sizeof(compiler_globals)); #endif zend_elf_load_symbols(); From 2817d322a00766ecd4345729d3ef31dc1216b1ae Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 26 Aug 2016 09:24:25 +0300 Subject: [PATCH 064/569] Show ENTRY labels --- ext/opcache/jit/zend_jit.c | 39 +++++++++++++-------- ext/opcache/jit/zend_jit_disasm_x86.c | 49 +++++++++++++++++++++------ 2 files changed, 62 insertions(+), 26 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 7310afc0adb72..979407bdc3dc2 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -105,6 +105,7 @@ static zend_string *zend_jit_func_name(zend_op_array *op_array) static void *dasm_link_and_encode(dasm_State **dasm_state, zend_op_array *op_array, + zend_cfg *cfg, const char *name) { size_t size; @@ -134,6 +135,25 @@ static void *dasm_link_and_encode(dasm_State **dasm_state, entry = dasm_ptr; dasm_ptr = (void*)((char*)dasm_ptr + ZEND_MM_ALIGNED_SIZE_EX(size, DASM_ALIGNMENT)); + if (op_array && cfg) { + int b; + + for (b = 0; b < cfg->blocks_count; b++) { + if (cfg->blocks[b].flags & (ZEND_BB_START|ZEND_BB_ENTRY)) { + zend_op *opline = op_array->opcodes + cfg->blocks[b].start; + + opline->handler = (void*)(((char*)entry) + + dasm_getpclabel(dasm_state, cfg->blocks_count + b)); + /* RECV may be skipped */ + while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) { + opline++; + opline->handler = (void*)(((char*)entry) + + dasm_getpclabel(dasm_state, cfg->blocks_count + b)); + } + } + } + } + #if defined(HAVE_DISASM) || defined(HAVE_GDB) || defined(HAVE_OPROFILE) || defined(HAVE_PERFTOOLS) if (!name) { if (ZCG(accel_directives).jit_debug & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_OPROFILE|ZEND_JIT_DEBUG_PERF)) { @@ -151,6 +171,8 @@ static void *dasm_link_and_encode(dasm_State **dasm_state, zend_jit_disasm( name, (op_array && op_array->filename) ? ZSTR_VAL(op_array->filename) : NULL, + op_array, + cfg, entry, size); } @@ -479,23 +501,10 @@ ZEND_API int zend_jit(zend_op_array *op_array, zend_script *script) } } - handler = dasm_link_and_encode(&dasm_state, op_array, NULL); + handler = dasm_link_and_encode(&dasm_state, op_array, &ssa.cfg, NULL); if (!handler) { goto jit_failure; } - for (b = 0; b < ssa.cfg.blocks_count; b++) { - if (ssa.cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_ENTRY)) { - opline = op_array->opcodes + ssa.cfg.blocks[b].start; - opline->handler = (void*)(((char*)handler) + - dasm_getpclabel(&dasm_state, ssa.cfg.blocks_count + b)); - /* RECV may be skipped */ - while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) { - opline++; - opline->handler = (void*)(((char*)handler) + - dasm_getpclabel(&dasm_state, ssa.cfg.blocks_count + b)); - } - } - } dasm_free(&dasm_state); zend_arena_release(&CG(arena), checkpoint); @@ -540,7 +549,7 @@ static int zend_jit_make_stubs(void) if (!zend_jit_stubs[i].stub(&dasm_state)) { return 0; } - if (!dasm_link_and_encode(&dasm_state, NULL, zend_jit_stubs[i].name)) { + if (!dasm_link_and_encode(&dasm_state, NULL, NULL, zend_jit_stubs[i].name)) { return 0; } } diff --git a/ext/opcache/jit/zend_jit_disasm_x86.c b/ext/opcache/jit/zend_jit_disasm_x86.c index 0e25edb4e456e..aa26a79bf688a 100644 --- a/ext/opcache/jit/zend_jit_disasm_x86.c +++ b/ext/opcache/jit/zend_jit_disasm_x86.c @@ -234,17 +234,20 @@ static int zend_jit_cmp_labels(Bucket *b1, Bucket *b2) return ((b1->h > b2->h) > 0) ? 1 : -1; } -static int zend_jit_disasm(const char *name, - const char *filename, - const void *start, - size_t size) +static int zend_jit_disasm(const char *name, + const char *filename, + zend_op_array *op_array, + zend_cfg *cfg, + const void *start, + size_t size) { const void *end = (void *)((char *)start + size); zval zv, *z; - zend_long n; + zend_long n, m; HashTable labels; const struct ud_operand *op; uint64_t addr; + int b; if (name) { fprintf(stderr, "%s: ; (%s)\n", name, filename ? filename : "unknown"); @@ -254,7 +257,18 @@ static int zend_jit_disasm(const char *name, ud_set_pc(&ud, (uint64_t)(uintptr_t)start); zend_hash_init(&labels, 8, NULL, NULL, 0); - ZVAL_NULL(&zv); + if (op_array && cfg) { + ZVAL_FALSE(&zv); + for (b = 0; b < cfg->blocks_count; b++) { + if (cfg->blocks[b].flags & ZEND_BB_ENTRY) { + addr = (uint64_t)(uintptr_t)op_array->opcodes[cfg->blocks[b].start].handler; + if (addr >= (uint64_t)(uintptr_t)start && addr < (uint64_t)(uintptr_t)end) { + zend_hash_index_add(&labels, addr, &zv); + } + } + } + } + ZVAL_TRUE(&zv); while (ud_disassemble(&ud)) { op = ud_insn_opr(&ud, 0); if (op && op->type == UD_OP_JIMM) { @@ -268,10 +282,15 @@ static int zend_jit_disasm(const char *name, zend_hash_sort(&labels, (compare_func_t)zend_jit_cmp_labels, 0); /* label numbering */ - n = 0; + n = 0; m = 0; ZEND_HASH_FOREACH_VAL(&labels, z) { - n++; - ZVAL_LONG(z, n); + if (Z_TYPE_P(z) == IS_FALSE) { + m--; + ZVAL_LONG(z, m); + } else { + n++; + ZVAL_LONG(z, n); + } } ZEND_HASH_FOREACH_END(); ud_set_input_buffer(&ud, (uint8_t*)start, (uint8_t*)end - (uint8_t*)start); @@ -281,7 +300,11 @@ static int zend_jit_disasm(const char *name, addr = ud_insn_off(&ud); z = zend_hash_index_find(&labels, addr); if (z) { - fprintf(stderr, ".L" ZEND_LONG_FMT ":\n", Z_LVAL_P(z)); + if (Z_LVAL_P(z) < 0) { + fprintf(stderr, ".ENTRY" ZEND_LONG_FMT ":\n", -Z_LVAL_P(z)); + } else { + fprintf(stderr, ".L" ZEND_LONG_FMT ":\n", Z_LVAL_P(z)); + } } op = ud_insn_opr(&ud, 0); if (op && op->type == UD_OP_JIMM) { @@ -300,7 +323,11 @@ static int zend_jit_disasm(const char *name, while (str[len] == ' ' || str[len] == '\t') { len++; } - fprintf(stderr, "\t%.*s.L" ZEND_LONG_FMT "\n", len, str, Z_LVAL_P(z)); + if (Z_LVAL_P(z) < 0) { + fprintf(stderr, "\t%.*s.ENTRY" ZEND_LONG_FMT "\n", len, str, -Z_LVAL_P(z)); + } else { + fprintf(stderr, "\t%.*s.L" ZEND_LONG_FMT "\n", len, str, Z_LVAL_P(z)); + } continue; } } From 564c53ea207340c5ad6ffad65218dbe76e49766f Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Fri, 26 Aug 2016 15:15:14 +0800 Subject: [PATCH 065/569] Register globals in parts --- ext/opcache/jit/zend_jit_disasm_x86.c | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/ext/opcache/jit/zend_jit_disasm_x86.c b/ext/opcache/jit/zend_jit_disasm_x86.c index aa26a79bf688a..c2cff2d236296 100644 --- a/ext/opcache/jit/zend_jit_disasm_x86.c +++ b/ext/opcache/jit/zend_jit_disasm_x86.c @@ -356,10 +356,24 @@ static int zend_jit_disasm_init(void) ud_set_syntax(&ud, UD_SYN_ATT); #endif ud_set_sym_resolver(&ud, zend_jit_disasm_resolver); + #ifndef ZTS - zend_jit_disasm_add_symbol("executor_globals", (uint64_t)(uintptr_t)&executor_globals, sizeof(executor_globals)); - zend_jit_disasm_add_symbol("compiler_globals", (uint64_t)(uintptr_t)&compiler_globals, sizeof(compiler_globals)); +#define REGISTER_EG(n) \ + zend_jit_disasm_add_symbol("EG("#n")", \ + (uint64_t)(uintptr_t)&executor_globals.n, sizeof(executor_globals.n)) + REGISTER_EG(uninitialized_zval); + REGISTER_EG(exception); + REGISTER_EG(vm_interrupt); + REGISTER_EG(exception_op); + REGISTER_EG(timed_out); +#undef REGISTER_EG +#define REGISTER_CG(n) \ + zend_jit_disasm_add_symbol("CG("#n")", \ + (uint64_t)(uintptr_t)&compiler_globals.n, sizeof(compiler_globals.n)) + REGISTER_CG(known_strings); +#undef REGISTER_CG #endif + zend_elf_load_symbols(); return 1; From 8335bcbb9c004582d8289dd2eff3a3fcc734a1bf Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 26 Aug 2016 14:21:35 +0300 Subject: [PATCH 066/569] Fixed stack frame information --- ext/opcache/jit/zend_jit_gdb.c | 60 +++++++++------------------------- 1 file changed, 16 insertions(+), 44 deletions(-) diff --git a/ext/opcache/jit/zend_jit_gdb.c b/ext/opcache/jit/zend_jit_gdb.c index b3087e1aaa84e..efe104eaf693a 100644 --- a/ext/opcache/jit/zend_jit_gdb.c +++ b/ext/opcache/jit/zend_jit_gdb.c @@ -22,16 +22,6 @@ #include "zend_elf.h" #include "zend_gdb.h" -#if defined(__x86_64) -#define CFRAME_SIZE (12*4) -#define CFRAME_SIZE_JIT CFRAME_SIZE -#elif defined(__i386__) -#define CFRAME_SIZE (10*8) -#define CFRAME_SIZE_JIT (CFRAME_SIZE + 16) -#else -#error "Unsupported target architecture" -#endif - /* DWARF definitions. */ #define DW_CIE_VERSION 1 @@ -46,7 +36,7 @@ enum { }; enum { - DW_EH_PE_udata4 = 3, + DW_EH_PE_udata4 = 0x03, DW_EH_PE_textrel = 0x20 }; @@ -185,8 +175,6 @@ typedef struct _zend_gdbjit_ctx { uint8_t *startp; /* Pointer to start address in obj.space. */ uintptr_t mcaddr; /* Machine code address. */ uint32_t szmcode; /* Size of machine code. */ - uint32_t spadjp; /* Stack adjustment for parent trace or interpreter. */ - uint32_t spadj; /* Stack adjustment for trace itself. */ int32_t lineno; /* Starting line number. */ const char *name; /* JIT function name */ const char *filename; /* Starting file name. */ @@ -304,13 +292,13 @@ static void zend_gdbjit_ehframe(zend_gdbjit_ctx *ctx) uint8_t *p = ctx->p; uint8_t *framep = p; - /* Emit DWARF EH CIE. */ + /* DWARF EH CIE (Call Frame Information). */ DSECT(CIE, - DU32(0); /* Offset to CIE itself. */ - DB(DW_CIE_VERSION); - DSTR("zR"); /* Augmentation. */ - DUV(1); /* Code alignment factor. */ - DSV(-(int32_t)sizeof(uintptr_t)); /* Data alignment factor. */ + DU32(0); /* CIE ID. */ + DB(DW_CIE_VERSION); /* Version */ + DSTR("zR"); /* Augmentation String. */ + DUV(1); /* Code alignment factor. */ + DSV(-(int32_t)sizeof(uintptr_t)); /* Data alignment factor. */ DB(DW_REG_RA); /* Return address register. */ DB(1); DB(DW_EH_PE_textrel|DW_EH_PE_udata4); /* Augmentation data. */ DB(DW_CFA_def_cfa); DUV(DW_REG_SP); DUV(sizeof(uintptr_t)); @@ -318,32 +306,22 @@ static void zend_gdbjit_ehframe(zend_gdbjit_ctx *ctx) DALIGNNOP(sizeof(uintptr_t)); ) - /* Emit DWARF EH FDE. */ + /* DWARF EH FDE (Frame Description Entry). */ DSECT(FDE, - DU32((uint32_t)(p-framep)); /* Offset to CIE. */ - DU32(0); /* Machine code offset relative to .text. */ + DU32((uint32_t)(p-framep)); /* Offset to CIE Pointer. */ + DU32(0); /* Machine code offset relative to .text. */ DU32(ctx->szmcode); /* Machine code length. */ DB(0); /* Augmentation data. */ - /* Registers saved in CFRAME. */ + DB(DW_CFA_def_cfa_offset); DUV(sizeof(uintptr_t)); #if defined(__i386__) - DB(DW_CFA_offset|DW_REG_BP); DUV(2); - DB(DW_CFA_offset|DW_REG_DI); DUV(3); - DB(DW_CFA_offset|DW_REG_SI); DUV(4); - DB(DW_CFA_offset|DW_REG_BX); DUV(5); + DB(DW_CFA_advance_loc|3); /* sub $0xc,%esp */ + DB(DW_CFA_def_cfa_offset); DUV(16); /* Aligned stack frame size. */ #elif defined(__x86_64__) - DB(DW_CFA_offset|DW_REG_BP); DUV(2); - DB(DW_CFA_offset|DW_REG_BX); DUV(3); - DB(DW_CFA_offset|DW_REG_15); DUV(4); - DB(DW_CFA_offset|DW_REG_14); DUV(5); - /* Extra registers saved for JIT-compiled code. */ - DB(DW_CFA_offset|DW_REG_13); DUV(9); - DB(DW_CFA_offset|DW_REG_12); DUV(10); + DB(DW_CFA_advance_loc|4); /* sub $0x8,%rsp */ + DB(DW_CFA_def_cfa_offset); DUV(16); /* Aligned stack frame size. */ #else -#error "Unsupported target architecture" +# error "Unsupported target architecture" #endif - if (ctx->spadjp != ctx->spadj) /* Parent/interpreter stack frame size. */ - (DB(DW_CFA_def_cfa_offset), DUV(ctx->spadjp), DB(DW_CFA_advance_loc|1)); /* Only an approximation. */ - DB(DW_CFA_def_cfa_offset); DUV(ctx->spadj); /* Trace stack frame size. */ DALIGNNOP(sizeof(uintptr_t)); ) @@ -480,12 +458,6 @@ static int zend_jit_gdb_register(const char *name, ctx.op_array = op_array; ctx.mcaddr = (uintptr_t)start; ctx.szmcode = (uint32_t)size; -#if defined(__x86_64__) - ctx.spadjp = CFRAME_SIZE_JIT + 8; -#elif defined(__i386__) - ctx.spadjp = CFRAME_SIZE_JIT + 12; -#endif - ctx.spadj = CFRAME_SIZE_JIT; ctx.name = name; ctx.filename = op_array ? ZSTR_VAL(op_array->filename) : "unknown"; ctx.lineno = op_array ? op_array->line_start : 0; From ac48e610d38cbfafbc6c46d62212ef6aa8ec8315 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 26 Aug 2016 16:16:35 +0300 Subject: [PATCH 067/569] Cleanup --- ext/opcache/jit/zend_jit_gdb.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/ext/opcache/jit/zend_jit_gdb.c b/ext/opcache/jit/zend_jit_gdb.c index efe104eaf693a..f97d77afacca9 100644 --- a/ext/opcache/jit/zend_jit_gdb.c +++ b/ext/opcache/jit/zend_jit_gdb.c @@ -170,7 +170,6 @@ static const zend_elf_header zend_elfhdr_template = { /* Context for generating the ELF object for the GDB JIT API. */ typedef struct _zend_gdbjit_ctx { - zend_op_array *op_array; /* Pointer to op_array */ uint8_t *p; /* Pointer to next address in obj.space. */ uint8_t *startp; /* Pointer to start address in obj.space. */ uintptr_t mcaddr; /* Machine code address. */ @@ -214,7 +213,8 @@ static void zend_gdbjit_sleb128(zend_gdbjit_ctx *ctx, int32_t v) ctx->p = p; } -static void zend_gdbjit_secthdr(zend_gdbjit_ctx *ctx) { +static void zend_gdbjit_secthdr(zend_gdbjit_ctx *ctx) +{ zend_elf_sectheader *sect; *ctx->p++ = '\0'; @@ -369,7 +369,6 @@ static void zend_gdbjit_debugabbrev(zend_gdbjit_ctx *ctx) ctx->p = p; } - #define DLNE(op, s) (DB(DW_LNS_extended_op), DUV(1+(s)), DB((op))) static void zend_gdbjit_debugline(zend_gdbjit_ctx *ctx) @@ -419,14 +418,16 @@ static void zend_gdbjit_debugline(zend_gdbjit_ctx *ctx) typedef void (*zend_gdbjit_initf) (zend_gdbjit_ctx *ctx); -static void zend_gdbjit_initsect(zend_gdbjit_ctx *ctx, int sect, zend_gdbjit_initf initf) { +static void zend_gdbjit_initsect(zend_gdbjit_ctx *ctx, int sect, zend_gdbjit_initf initf) +{ ctx->startp = ctx->p; ctx->obj.sect[sect].ofs = (uintptr_t)((char *)ctx->p - (char *)&ctx->obj); initf(ctx); ctx->obj.sect[sect].size = (uintptr_t)(ctx->p - ctx->startp); } -static void zend_gdbjit_buildobj(zend_gdbjit_ctx *ctx) { +static void zend_gdbjit_buildobj(zend_gdbjit_ctx *ctx) +{ zend_gdbjit_obj *obj = &ctx->obj; /* Fill in ELF header and clear structures. */ @@ -448,14 +449,13 @@ static void zend_gdbjit_buildobj(zend_gdbjit_ctx *ctx) { ZEND_ASSERT(ctx->objsize < sizeof(zend_gdbjit_obj)); } -static int zend_jit_gdb_register(const char *name, +static int zend_jit_gdb_register(const char *name, zend_op_array *op_array, - const void *start, - size_t size) + const void *start, + size_t size) { zend_gdbjit_ctx ctx; - ctx.op_array = op_array; ctx.mcaddr = (uintptr_t)start; ctx.szmcode = (uint32_t)size; ctx.name = name; From f11160d1de0a7924a243d9932244bf2f152c5c4b Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 26 Aug 2016 16:26:23 +0300 Subject: [PATCH 068/569] Added reference to original author. --- ext/opcache/jit/zend_jit_gdb.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ext/opcache/jit/zend_jit_gdb.c b/ext/opcache/jit/zend_jit_gdb.c index f97d77afacca9..d729167fa4941 100644 --- a/ext/opcache/jit/zend_jit_gdb.c +++ b/ext/opcache/jit/zend_jit_gdb.c @@ -15,6 +15,9 @@ | Authors: Dmitry Stogov | | Xinchen Hui | +----------------------------------------------------------------------+ + | Based on Mike Pall's implementation of GDB interface for LuaJIT. | + | LuaJIT -- a Just-In-Time Compiler for Lua. http://luajit.org/ | + +----------------------------------------------------------------------+ */ #define HAVE_GDB From 2b8cd80be41e2ab7fcde5d189e6113cc9f7590fb Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 26 Aug 2016 17:13:00 +0300 Subject: [PATCH 069/569] Avoid generation of unreachable JMPs --- ext/opcache/jit/zend_jit.c | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 979407bdc3dc2..2346b8d0a632b 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -337,6 +337,19 @@ ZEND_API int zend_jit(zend_op_array *op_array, zend_script *script) } } + /* mark hidden branch targets */ + for (b = 0; b < ssa.cfg.blocks_count; b++) { + if (ssa.cfg.blocks[b].flags & ZEND_BB_REACHABLE && + ssa.cfg.blocks[b].len > 1) { + + opline = op_array->opcodes + ssa.cfg.blocks[b].start + ssa.cfg.blocks[b].len - 1; + if (opline->opcode == ZEND_DO_FCALL && + (opline-1)->opcode == ZEND_NEW) { + ssa.cfg.blocks[ssa.cfg.blocks[b].successors[0]].flags |= ZEND_BB_TARGET; + } + } + } + dasm_init(&dasm_state, DASM_MAXSECTION); dasm_setupglobal(&dasm_state, dasm_labels, lbl__MAX); dasm_setup(&dasm_state, dasm_actions); @@ -349,7 +362,8 @@ ZEND_API int zend_jit(zend_op_array *op_array, zend_script *script) continue; } if (ssa.cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_ENTRY)) { - if (ssa.cfg.blocks[b].flags & ZEND_BB_ENTRY) { + if ((ssa.cfg.blocks[b].flags & ZEND_BB_ENTRY) && + (ssa.cfg.blocks[b].flags & ZEND_BB_TARGET)) { zend_jit_jmp(&dasm_state, b); } zend_jit_label(&dasm_state, ssa.cfg.blocks_count + b); From 0ddff9c63d4e0709e0dbda38486aa4e6f9a54b8c Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Fri, 26 Aug 2016 22:42:09 +0800 Subject: [PATCH 070/569] Fixed name --- ext/opcache/jit/zend_jit_gdb.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_gdb.c b/ext/opcache/jit/zend_jit_gdb.c index d729167fa4941..900fac9f52b3a 100644 --- a/ext/opcache/jit/zend_jit_gdb.c +++ b/ext/opcache/jit/zend_jit_gdb.c @@ -28,6 +28,7 @@ /* DWARF definitions. */ #define DW_CIE_VERSION 1 +/* CFA (Canonical frame address) */ enum { DW_CFA_nop = 0x0, DW_CFA_offset_extended = 0x5, @@ -295,7 +296,7 @@ static void zend_gdbjit_ehframe(zend_gdbjit_ctx *ctx) uint8_t *p = ctx->p; uint8_t *framep = p; - /* DWARF EH CIE (Call Frame Information). */ + /* DWARF EH CIE (Common Information Entry) */ DSECT(CIE, DU32(0); /* CIE ID. */ DB(DW_CIE_VERSION); /* Version */ From 5e4e4e0afdd1af8eb9f4675709783b4a34559b12 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 26 Aug 2016 17:52:18 +0300 Subject: [PATCH 071/569] Improved support for context threading (incomplete; it's a bit faster, but doesn't cover many cases and crashes). --- ext/opcache/jit/zend_jit.c | 13 +++++++++++++ ext/opcache/jit/zend_jit_x86.dasc | 2 ++ 2 files changed, 15 insertions(+) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 2346b8d0a632b..463bcfec5ecf4 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -36,6 +36,8 @@ #define ZEND_JIT_LEVEL ZEND_JIT_LEVEL_FULL +//#define CONTEXT_THREDED_JIT + #define JIT_PREFIX "JIT$" #define JIT_STUB_PREFIX "JIT$$" @@ -139,7 +141,11 @@ static void *dasm_link_and_encode(dasm_State **dasm_state, int b; for (b = 0; b < cfg->blocks_count; b++) { +#ifdef CONTEXT_THREDED_JIT + if (cfg->blocks[b].flags & ZEND_BB_START) { +#else if (cfg->blocks[b].flags & (ZEND_BB_START|ZEND_BB_ENTRY)) { +#endif zend_op *opline = op_array->opcodes + cfg->blocks[b].start; opline->handler = (void*)(((char*)entry) + @@ -361,6 +367,12 @@ ZEND_API int zend_jit(zend_op_array *op_array, zend_script *script) if ((ssa.cfg.blocks[b].flags & ZEND_BB_REACHABLE) == 0) { continue; } +#ifdef CONTEXT_THREDED_JIT + if (ssa.cfg.blocks[b].flags & ZEND_BB_START) { + zend_jit_label(&dasm_state, ssa.cfg.blocks_count + b); + zend_jit_prologue(&dasm_state); + } +#else if (ssa.cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_ENTRY)) { if ((ssa.cfg.blocks[b].flags & ZEND_BB_ENTRY) && (ssa.cfg.blocks[b].flags & ZEND_BB_TARGET)) { @@ -369,6 +381,7 @@ ZEND_API int zend_jit(zend_op_array *op_array, zend_script *script) zend_jit_label(&dasm_state, ssa.cfg.blocks_count + b); zend_jit_prologue(&dasm_state); } +#endif zend_jit_label(&dasm_state, b); if (ssa.cfg.blocks[b].flags & ZEND_BB_LOOP_HEADER) { if (!zend_jit_check_timeout(&dasm_state)) { diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 1bd0d1a1d4d51..4d631094ceace 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -249,12 +249,14 @@ static int zend_jit_context_threaded_call(dasm_State **Dst, zend_op *opline) if (!zend_jit_handler(Dst, opline)) return 0; if (opline->opcode == ZEND_DO_UCALL) { | call aword [IP] + zend_jit_check_exception(Dst); } else { zend_op *next_opline = opline + 1; | cmp IPl, next_opline | jz >1 | call aword [IP] + zend_jit_check_exception(Dst); |1: } return 1; From 2f4a2571ab23d5d9053ddb81cbb8c420442d6747 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Fri, 26 Aug 2016 23:34:30 +0800 Subject: [PATCH 072/569] Fixed edge case --- ext/opcache/jit/zend_jit_disasm_x86.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_disasm_x86.c b/ext/opcache/jit/zend_jit_disasm_x86.c index c2cff2d236296..0132eb6050194 100644 --- a/ext/opcache/jit/zend_jit_disasm_x86.c +++ b/ext/opcache/jit/zend_jit_disasm_x86.c @@ -115,6 +115,7 @@ static void zend_jit_disasm_add_symbol(const char *name, /* insert it into rbtree */ do { if (sym->addr > node->addr) { + ZEND_ASSERT(sym->addr >= (node->addr + node->size)); if (node->child[1]) { node = node->child[1]; } else { @@ -196,7 +197,8 @@ static const char* zend_jit_disasm_find_symbol(uint64_t addr, while (node) { if (addr < node->addr) { node = node->child[0]; - } else if (addr > (node->addr + node->size)) { + } else if (addr > node->addr && + addr >= (node->addr + node->size)) { node = node->child[1]; } else { *offset = addr - node->addr; From 36bb640c699e588eed506a48b98306650739365f Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Sat, 27 Aug 2016 10:19:07 +0800 Subject: [PATCH 073/569] Use xor --- ext/opcache/jit/zend_jit_x86.dasc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 4d631094ceace..657ebf02b6cb3 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -79,7 +79,7 @@ static int zend_jit_interrupt_handler_stub(dasm_State **Dst) | je >1 | //zend_timeout(0); |.if X64 - | mov CARG1d, 0 + | xor CARG1d, CARG1d | load_addr rax, zend_timeout | call rax |.else From 8f60e7ceab6f1fcf67f0952d4e21da723f79acac Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Sat, 27 Aug 2016 13:20:52 +0800 Subject: [PATCH 074/569] Rename default prefix --- ext/opcache/jit/zend_jit.c | 4 ++-- ext/opcache/jit/zend_jit_x86.dasc | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 463bcfec5ecf4..1bd2a61cb500a 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -357,7 +357,7 @@ ZEND_API int zend_jit(zend_op_array *op_array, zend_script *script) } dasm_init(&dasm_state, DASM_MAXSECTION); - dasm_setupglobal(&dasm_state, dasm_labels, lbl__MAX); + dasm_setupglobal(&dasm_state, dasm_labels, zend_lb_MAX); dasm_setup(&dasm_state, dasm_actions); dasm_growpc(&dasm_state, ssa.cfg.blocks_count * 2); @@ -569,7 +569,7 @@ static int zend_jit_make_stubs(void) uint32_t i; dasm_init(&dasm_state, DASM_MAXSECTION); - dasm_setupglobal(&dasm_state, dasm_labels, lbl__MAX); + dasm_setupglobal(&dasm_state, dasm_labels, zend_lb_MAX); for (i = 0; i < sizeof(zend_jit_stubs)/sizeof(zend_jit_stubs[0]); i++) { dasm_setup(&dasm_state, dasm_actions); diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 657ebf02b6cb3..2e3a648effb3e 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -55,8 +55,8 @@ |.actionlist dasm_actions -|.globals lbl_ -static void* dasm_labels[lbl__MAX]; +|.globals zend_lb +static void* dasm_labels[zend_lb_MAX]; |.section code From 8e7c71b229e58dd991da92eb81b847409c5211d1 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Sun, 28 Aug 2016 22:07:37 +0800 Subject: [PATCH 075/569] Seems we could do the check while JITing (only one usage is in pctl, it set the interrupt_handler in MINIT) --- ext/opcache/jit/zend_jit_x86.dasc | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 2e3a648effb3e..098766189e9bc 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -88,25 +88,25 @@ static int zend_jit_interrupt_handler_stub(dasm_State **Dst) |.endif |1: | //} else if (zend_interrupt_function) { - | mov r0, aword [zend_interrupt_function] - | test r0, r0 - | je >2 + || if (zend_interrupt_function) { | //SAVE_OPLINE(); | mov EX->opline, IP | //zend_interrupt_function(execute_data); |.if X64 + | load_addr rax, zend_interrupt_function | mov CARG1, FP + | call rax |.else | push FP + | call &zend_interrupt_function |.endif - | call r0 | //ZEND_VM_ENTER(); - | //execute_data = EG(current_execute_data); + | //execute_data = EG(current_execute_data); | mov FP, aword [&EG(current_execute_data)] - | // LOAD_OPLINE(); + | // LOAD_OPLINE(); | mov IP, EX->opline | //} - |2: + ||} | //ZEND_VM_CONTINUE() | add r4, SPAD // stack alignment | jmp aword [IP] From 89eeddb999a98550442aa66e9075870b637bbdcb Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Sun, 28 Aug 2016 23:41:37 +0800 Subject: [PATCH 076/569] Cleanup (http://corsix.github.io/dynasm-doc/instructions.html#memory) --- ext/opcache/jit/zend_jit_x86.dasc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 098766189e9bc..5bffc08f72954 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -84,7 +84,7 @@ static int zend_jit_interrupt_handler_stub(dasm_State **Dst) | call rax |.else | push 0 - | call &zend_timeout + | call dword &zend_timeout |.endif |1: | //} else if (zend_interrupt_function) { @@ -98,7 +98,7 @@ static int zend_jit_interrupt_handler_stub(dasm_State **Dst) | call rax |.else | push FP - | call &zend_interrupt_function + | call dword &zend_interrupt_function |.endif | //ZEND_VM_ENTER(); | //execute_data = EG(current_execute_data); @@ -122,7 +122,7 @@ static int zend_jit_exception_handler_stub(dasm_State **Dst) | load_addr rax, EG(exception_op)->handler | jmp rax |.else - | jmp &(EG(exception_op)->handler) + | jmp dword &(EG(exception_op)->handler) |.endif return 1; @@ -173,7 +173,7 @@ static int zend_jit_handler(dasm_State **Dst, zend_op *opline) | load_addr rax, handler | call rax |.else - | call aword &handler + | call dword &handler |.endif zend_jit_check_exception(Dst); return 1; @@ -188,7 +188,7 @@ static int zend_jit_tail_handler(dasm_State **Dst, zend_op *opline) | load_addr rax, handler | jmp rax |.else - | jmp aword &handler + | jmp dword &handler |.endif return 1; } From 2e33860c8a0f94311e8c729391c903a5de5b0978 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Mon, 29 Aug 2016 20:06:17 +0800 Subject: [PATCH 077/569] For better readability --- ext/opcache/jit/zend_jit_x86.dasc | 38 +++++++++++++++---------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 5bffc08f72954..2f36929beb182 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -88,25 +88,25 @@ static int zend_jit_interrupt_handler_stub(dasm_State **Dst) |.endif |1: | //} else if (zend_interrupt_function) { - || if (zend_interrupt_function) { - | //SAVE_OPLINE(); - | mov EX->opline, IP - | //zend_interrupt_function(execute_data); - |.if X64 - | load_addr rax, zend_interrupt_function - | mov CARG1, FP - | call rax - |.else - | push FP - | call dword &zend_interrupt_function - |.endif - | //ZEND_VM_ENTER(); - | //execute_data = EG(current_execute_data); - | mov FP, aword [&EG(current_execute_data)] - | // LOAD_OPLINE(); - | mov IP, EX->opline - | //} - ||} + if (zend_interrupt_function) { + | //SAVE_OPLINE(); + | mov EX->opline, IP + | //zend_interrupt_function(execute_data); + |.if X64 + | load_addr rax, zend_interrupt_function + | mov CARG1, FP + | call rax + |.else + | push FP + | call dword &zend_interrupt_function + |.endif + | //ZEND_VM_ENTER(); + | //execute_data = EG(current_execute_data); + | mov FP, aword [&EG(current_execute_data)] + | // LOAD_OPLINE(); + | mov IP, EX->opline + | //} + } | //ZEND_VM_CONTINUE() | add r4, SPAD // stack alignment | jmp aword [IP] From e5159fa748bf2e83565e21db0b08a0017dd07bbc Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 29 Aug 2016 16:13:07 +0300 Subject: [PATCH 078/569] Avoid codegeneration for useless exception checks --- ext/opcache/jit/zend_jit.c | 260 +++++++++++++++++++++++++++++- ext/opcache/jit/zend_jit_x86.dasc | 12 +- 2 files changed, 262 insertions(+), 10 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 1bd2a61cb500a..56f6f462dcf0e 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -280,6 +280,254 @@ static void jit_free(void *p, size_t size) #endif } +static int zend_may_throw(zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) +{ + uint32_t t1 = OP1_INFO(); + uint32_t t2 = OP2_INFO(); + + if (opline->op1_type == IS_CV) { + if (t1 & MAY_BE_UNDEF) { + switch (opline->opcode) { + case ZEND_UNSET_VAR: + case ZEND_ISSET_ISEMPTY_VAR: + if (opline->extended_value & ZEND_QUICK_SET) { + break; + } + case ZEND_ISSET_ISEMPTY_DIM_OBJ: + case ZEND_ISSET_ISEMPTY_PROP_OBJ: + case ZEND_ASSIGN: + case ZEND_ASSIGN_REF: + case ZEND_BIND_GLOBAL: + case ZEND_FETCH_DIM_IS: + case ZEND_FETCH_OBJ_IS: + case ZEND_SEND_REF: + break; + default: + /* undefined variable warning */ + return 1; + } + } + } else if (opline->op1_type & (IS_TMP_VAR|IS_VAR)) { + if (t1 & (MAY_BE_OBJECT|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_RESOURCE)) { + switch (opline->opcode) { + case ZEND_CASE: + case ZEND_FE_FETCH_R: + case ZEND_FE_FETCH_RW: + case ZEND_FETCH_LIST: + case ZEND_QM_ASSIGN: + case ZEND_SEND_VAL: + case ZEND_SEND_VAL_EX: + case ZEND_SEND_VAR: + case ZEND_SEND_VAR_EX: + case ZEND_SEND_VAR_NO_REF: + case ZEND_SEND_VAR_NO_REF_EX: + case ZEND_SEND_REF: + case ZEND_SEPARATE: + break; + default: + /* destructor may be called */ + return 1; + } + } + } + + if (opline->op2_type == IS_CV) { + if (t2 & MAY_BE_UNDEF) { + switch (opline->opcode) { + case ZEND_ASSIGN_REF: + break; + default: + /* undefined variable warning */ + return 1; + } + } + } else if (opline->op2_type & (IS_TMP_VAR|IS_VAR)) { + if (t2 & (MAY_BE_OBJECT|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_RESOURCE)) { + switch (opline->opcode) { + case ZEND_ASSIGN: + break; + default: + /* destructor may be called */ + return 1; + } + } + } + + switch (opline->opcode) { + case ZEND_NOP: + case ZEND_IS_IDENTICAL: + case ZEND_IS_NOT_IDENTICAL: + case ZEND_QM_ASSIGN: + case ZEND_JMP: + case ZEND_CHECK_VAR: + case ZEND_MAKE_REF: + case ZEND_SEND_VAR: + case ZEND_BEGIN_SILENCE: + case ZEND_END_SILENCE: + case ZEND_SEND_VAL: + case ZEND_SEND_REF: + case ZEND_FREE: + case ZEND_SEPARATE: + return 0; + case ZEND_BIND_GLOBAL: + if ((opline+1)->opcode == ZEND_BIND_GLOBAL) { + return zend_may_throw(opline + 1, op_array, ssa); + } + return 0; + case ZEND_ADD: + if ((t1 & MAY_BE_ANY) == MAY_BE_ARRAY + && (t2 & MAY_BE_ANY) == MAY_BE_ARRAY) { + return 0; + } + return (t1 & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT)) || + (t2 & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT)); + case ZEND_DIV: + case ZEND_MOD: + if (opline->op2_type != IS_CONST + || Z_TYPE_P(RT_CONSTANT(op_array, opline->op2)) == IS_NULL + || Z_TYPE_P(RT_CONSTANT(op_array, opline->op2)) == IS_FALSE + || (Z_TYPE_P(RT_CONSTANT(op_array, opline->op2)) == IS_LONG + && Z_LVAL_P(RT_CONSTANT(op_array, opline->op2)) == 0) + || (Z_TYPE_P(RT_CONSTANT(op_array, opline->op2)) == IS_DOUBLE + && Z_DVAL_P(RT_CONSTANT(op_array, opline->op2)) == 0.0)) { + return 1; + } + /* break missing intentionally */ + case ZEND_SUB: + case ZEND_MUL: + case ZEND_POW: + return (t1 & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT)) || + (t2 & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT)); + case ZEND_SL: + case ZEND_SR: + if (opline->op2_type != IS_CONST + || (Z_TYPE_P(RT_CONSTANT(op_array, opline->op2)) == IS_LONG + && (zend_ulong)Z_LVAL_P(RT_CONSTANT(op_array, opline->op2)) >= SIZEOF_ZEND_LONG * 8) + || (Z_TYPE_P(RT_CONSTANT(op_array, opline->op2)) == IS_DOUBLE + && (zend_ulong)Z_DVAL_P(RT_CONSTANT(op_array, opline->op2)) >= SIZEOF_ZEND_LONG * 8)) { + return 1; + } + return (t1 & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT)) || + (t2 & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT)); + case ZEND_CONCAT: + case ZEND_FAST_CONCAT: + return (t1 & (MAY_BE_ARRAY|MAY_BE_OBJECT)) || + (t2 & (MAY_BE_ARRAY|MAY_BE_OBJECT)); + case ZEND_BW_OR: + case ZEND_BW_AND: + case ZEND_BW_XOR: + if ((t1 & MAY_BE_ANY) == MAY_BE_STRING + && (t2 & MAY_BE_ANY) == MAY_BE_STRING) { + return 0; + } + return (t1 & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT)) || + (t2 & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT)); + case ZEND_BW_NOT: + return (t1 & (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_ARRAY|MAY_BE_OBJECT)); + case ZEND_BOOL_NOT: + case ZEND_PRE_INC: + case ZEND_POST_INC: + case ZEND_PRE_DEC: + case ZEND_POST_DEC: + case ZEND_JMPZ: + case ZEND_JMPNZ: + case ZEND_JMPZNZ: + case ZEND_JMPZ_EX: + case ZEND_JMPNZ_EX: + case ZEND_BOOL: + return (t1 & MAY_BE_OBJECT); + case ZEND_BOOL_XOR: + return (t1 & MAY_BE_OBJECT) || (t2 & MAY_BE_OBJECT); + case ZEND_IS_EQUAL: + case ZEND_IS_NOT_EQUAL: + case ZEND_IS_SMALLER: + case ZEND_IS_SMALLER_OR_EQUAL: + case ZEND_CASE: + if ((t1 & MAY_BE_ANY) == MAY_BE_NULL + || (t2 & MAY_BE_ANY) == MAY_BE_NULL) { + return 0; + } + return (t1 & MAY_BE_OBJECT) || (t2 & MAY_BE_OBJECT); + case ZEND_ASSIGN_ADD: + if (opline->extended_value != 0) { + return 1; + } + if ((t1 & MAY_BE_ANY) == MAY_BE_ARRAY + && (t2 & MAY_BE_ANY) == MAY_BE_ARRAY) { + return 0; + } + return (t1 & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT)) || + (t2 & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT)); + case ZEND_ASSIGN_DIV: + case ZEND_ASSIGN_MOD: + if (opline->extended_value != 0) { + return 1; + } + if (opline->op2_type != IS_CONST + || Z_TYPE_P(RT_CONSTANT(op_array, opline->op2)) == IS_NULL + || Z_TYPE_P(RT_CONSTANT(op_array, opline->op2)) == IS_FALSE + || (Z_TYPE_P(RT_CONSTANT(op_array, opline->op2)) == IS_LONG + && Z_LVAL_P(RT_CONSTANT(op_array, opline->op2)) == 0) + || (Z_TYPE_P(RT_CONSTANT(op_array, opline->op2)) == IS_DOUBLE + && Z_DVAL_P(RT_CONSTANT(op_array, opline->op2)) == 0.0)) { + return 1; + } + /* break missing intentionally */ + case ZEND_ASSIGN_SUB: + case ZEND_ASSIGN_MUL: + case ZEND_ASSIGN_POW: + if (opline->extended_value != 0) { + return 1; + } + return (t1 & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT)) || + (t2 & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT)); + case ZEND_ASSIGN_SL: + case ZEND_ASSIGN_SR: + if (opline->extended_value != 0) { + return 1; + } + if (opline->op2_type != IS_CONST + || (Z_TYPE_P(RT_CONSTANT(op_array, opline->op2)) == IS_LONG + && (zend_ulong)Z_LVAL_P(RT_CONSTANT(op_array, opline->op2)) >= SIZEOF_ZEND_LONG * 8) + || (Z_TYPE_P(RT_CONSTANT(op_array, opline->op2)) == IS_DOUBLE + && (zend_ulong)Z_DVAL_P(RT_CONSTANT(op_array, opline->op2)) >= SIZEOF_ZEND_LONG * 8)) { + return 1; + } + return (t1 & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT)) || + (t2 & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT)); + case ZEND_ASSIGN_CONCAT: + if (opline->extended_value != 0) { + return 1; + } + return (t1 & (MAY_BE_ARRAY|MAY_BE_OBJECT)) || + (t2 & (MAY_BE_ARRAY|MAY_BE_OBJECT)); + case ZEND_ASSIGN_BW_OR: + case ZEND_ASSIGN_BW_AND: + case ZEND_ASSIGN_BW_XOR: + if (opline->extended_value != 0) { + return 1; + } + if ((t1 & MAY_BE_ANY) == MAY_BE_STRING + && (t2 & MAY_BE_ANY) == MAY_BE_STRING) { + return 0; + } + return (t1 & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT)) || + (t2 & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT)); + case ZEND_ASSIGN: + return (t1 & (MAY_BE_OBJECT|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_RESOURCE)); + case ZEND_ROPE_INIT: + case ZEND_ROPE_ADD: + return t2 & (MAY_BE_ARRAY|MAY_BE_OBJECT); + case ZEND_INIT_ARRAY: + case ZEND_ADD_ARRAY_ELEMENT: + return t2 & (MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE); + case ZEND_STRLEN: + return (t1 & MAY_BE_ANY) != MAY_BE_STRING; + default: + return 1; + } +} + ZEND_API int zend_jit(zend_op_array *op_array, zend_script *script) { uint32_t flags = 0; @@ -341,6 +589,8 @@ ZEND_API int zend_jit(zend_op_array *op_array, zend_script *script) if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_SSA) { zend_dump_op_array(op_array, ZEND_DUMP_HIDE_UNREACHABLE|ZEND_DUMP_RC_INFERENCE|ZEND_DUMP_SSA|ZEND_DUMP_RT_CONSTANTS, "JIT", &ssa); } + } else { + ssa.rt_constants = 1; } /* mark hidden branch targets */ @@ -405,7 +655,7 @@ ZEND_API int zend_jit(zend_op_array *op_array, zend_script *script) case ZEND_BIND_GLOBAL: if (opline->opcode != op_array->opcodes[i+1].opcode) { /* repeatable opcodes */ - if (!zend_jit_handler(&dasm_state, opline)) { + if (!zend_jit_handler(&dasm_state, opline, zend_may_throw(opline, op_array, &ssa))) { goto jit_failure; } } @@ -491,7 +741,7 @@ ZEND_API int zend_jit(zend_op_array *op_array, zend_script *script) case ZEND_FE_RESET_R: case ZEND_FE_RESET_RW: case ZEND_ASSERT_CHECK: - if (!zend_jit_handler(&dasm_state, opline) || + if (!zend_jit_handler(&dasm_state, opline, zend_may_throw(opline, op_array, &ssa)) || !zend_jit_cond_jmp(&dasm_state, opline + 1, ssa.cfg.blocks[b].successors[0])) { goto jit_failure; } @@ -505,7 +755,7 @@ ZEND_API int zend_jit(zend_op_array *op_array, zend_script *script) } break; case ZEND_JMPZNZ: - if (!zend_jit_handler(&dasm_state, opline) || + if (!zend_jit_handler(&dasm_state, opline, zend_may_throw(opline, op_array, &ssa)) || !zend_jit_cond_jmp(&dasm_state, OP_JMP_ADDR(opline, opline->op2), ssa.cfg.blocks[b].successors[1]) || !zend_jit_jmp(&dasm_state, ssa.cfg.blocks[b].successors[0])) { goto jit_failure; @@ -515,13 +765,13 @@ ZEND_API int zend_jit(zend_op_array *op_array, zend_script *script) case ZEND_FE_FETCH_RW: case ZEND_DECLARE_ANON_CLASS: case ZEND_DECLARE_ANON_INHERITED_CLASS: - if (!zend_jit_handler(&dasm_state, opline) || + if (!zend_jit_handler(&dasm_state, opline, zend_may_throw(opline, op_array, &ssa)) || !zend_jit_cond_jmp(&dasm_state, opline + 1, ssa.cfg.blocks[b].successors[0])) { goto jit_failure; } break; default: - if (!zend_jit_handler(&dasm_state, opline)) { + if (!zend_jit_handler(&dasm_state, opline, zend_may_throw(opline, op_array, &ssa))) { goto jit_failure; } } diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 2f36929beb182..0ef423119b43f 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -165,7 +165,7 @@ static int zend_jit_check_exception(dasm_State **Dst) return 1; } -static int zend_jit_handler(dasm_State **Dst, zend_op *opline) +static int zend_jit_handler(dasm_State **Dst, zend_op *opline, int may_throw) { const void *handler = opline->handler; @@ -175,7 +175,9 @@ static int zend_jit_handler(dasm_State **Dst, zend_op *opline) |.else | call dword &handler |.endif - zend_jit_check_exception(Dst); + if (may_throw) { + zend_jit_check_exception(Dst); + } return 1; } @@ -238,7 +240,7 @@ static int zend_jit_recv(dasm_State **Dst, zend_op *opline) { | cmp IPl, opline | jnz >1 - zend_jit_handler(Dst, opline); + zend_jit_handler(Dst, opline, 1); |1: return 1; } @@ -246,7 +248,7 @@ static int zend_jit_recv(dasm_State **Dst, zend_op *opline) #ifdef CONTEXT_THREDED_JIT static int zend_jit_context_threaded_call(dasm_State **Dst, zend_op *opline) { - if (!zend_jit_handler(Dst, opline)) return 0; + if (!zend_jit_handler(Dst, opline, 1)) return 0; if (opline->opcode == ZEND_DO_UCALL) { | call aword [IP] zend_jit_check_exception(Dst); @@ -274,7 +276,7 @@ static int zend_jit_call(dasm_State **Dst, zend_op *opline) static int zend_jit_new(dasm_State **Dst, zend_op *opline) { - if (!zend_jit_handler(Dst, opline)) return 0; + if (!zend_jit_handler(Dst, opline, 1)) return 0; if (EXPECTED(opline->extended_value == 0 && (opline+1)->opcode == ZEND_DO_FCALL)) { zend_op *next_opline = opline + 1; From eb093a3c5dece09242feabd23107397415638ba0 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 29 Aug 2016 16:59:27 +0300 Subject: [PATCH 079/569] Simplify code for smart brnaches. --- ext/opcache/jit/zend_jit.c | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 56f6f462dcf0e..53025f97d3e20 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -712,11 +712,24 @@ ZEND_API int zend_jit(zend_op_array *op_array, zend_script *script) (opline-1)->opcode == ZEND_IS_SMALLER || (opline-1)->opcode == ZEND_IS_SMALLER_OR_EQUAL || (opline-1)->opcode == ZEND_CASE) { - /* might be smart branch */ - if (!zend_jit_smart_branch(&dasm_state, opline, (b + 1), ssa.cfg.blocks[b].successors[0])) { - goto jit_failure; + + uint32_t t1 = _ssa_op1_info(op_array, &ssa, (opline-1)); + uint32_t t2 = _ssa_op2_info(op_array, &ssa, (opline-1)); + + if ((t1 & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) || + (t2 & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { + /* might be smart branch */ + if (!zend_jit_smart_branch(&dasm_state, opline, (b + 1), ssa.cfg.blocks[b].successors[0])) { + goto jit_failure; + } + /* break missing intentionally */ + } else { + /* smart branch */ + if (!zend_jit_cond_jmp(&dasm_state, opline + 1, ssa.cfg.blocks[b].successors[0])) { + goto jit_failure; + } + break; } - /* break missing intentionally */ } else if ((opline-1)->opcode == ZEND_IS_IDENTICAL || (opline-1)->opcode == ZEND_IS_NOT_IDENTICAL || (opline-1)->opcode == ZEND_ISSET_ISEMPTY_VAR || From 3f5779a09525adaec964f76e53ad4a65595ca7ea Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 29 Aug 2016 20:45:53 +0300 Subject: [PATCH 080/569] Use different codegeneration approach for RECV and RECV_INIT --- ext/opcache/jit/zend_jit.c | 50 ++++++++++++++++++--------- ext/opcache/jit/zend_jit_disasm_x86.c | 2 +- ext/opcache/jit/zend_jit_x86.dasc | 9 ----- 3 files changed, 35 insertions(+), 26 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 53025f97d3e20..16f1a2ece313b 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -144,18 +144,12 @@ static void *dasm_link_and_encode(dasm_State **dasm_state, #ifdef CONTEXT_THREDED_JIT if (cfg->blocks[b].flags & ZEND_BB_START) { #else - if (cfg->blocks[b].flags & (ZEND_BB_START|ZEND_BB_ENTRY)) { + if (cfg->blocks[b].flags & (ZEND_BB_START|ZEND_BB_ENTRY|ZEND_BB_RECV_ENTRY)) { #endif zend_op *opline = op_array->opcodes + cfg->blocks[b].start; opline->handler = (void*)(((char*)entry) + dasm_getpclabel(dasm_state, cfg->blocks_count + b)); - /* RECV may be skipped */ - while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) { - opline++; - opline->handler = (void*)(((char*)entry) + - dasm_getpclabel(dasm_state, cfg->blocks_count + b)); - } } } } @@ -547,7 +541,7 @@ ZEND_API int zend_jit(zend_op_array *op_array, zend_script *script) /* Build SSA */ memset(&ssa, 0, sizeof(zend_ssa)); - if (zend_build_cfg(&CG(arena), op_array, ZEND_CFG_STACKLESS | ZEND_RT_CONSTANTS | ZEND_SSA_RC_INFERENCE, &ssa.cfg, &flags) != SUCCESS) { + if (zend_build_cfg(&CG(arena), op_array, ZEND_CFG_STACKLESS | ZEND_CFG_RECV_ENTRY | ZEND_RT_CONSTANTS | ZEND_SSA_RC_INFERENCE, &ssa.cfg, &flags) != SUCCESS) { goto jit_failure; } @@ -623,13 +617,36 @@ ZEND_API int zend_jit(zend_op_array *op_array, zend_script *script) zend_jit_prologue(&dasm_state); } #else - if (ssa.cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_ENTRY)) { - if ((ssa.cfg.blocks[b].flags & ZEND_BB_ENTRY) && - (ssa.cfg.blocks[b].flags & ZEND_BB_TARGET)) { + if (ssa.cfg.blocks[b].flags & ZEND_BB_ENTRY) { + if (ssa.cfg.blocks[b].flags & ZEND_BB_TARGET) { zend_jit_jmp(&dasm_state, b); } zend_jit_label(&dasm_state, ssa.cfg.blocks_count + b); zend_jit_prologue(&dasm_state); + } else if (ssa.cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_RECV_ENTRY)) { + opline = op_array->opcodes + ssa.cfg.blocks[b].start; + if (ssa.cfg.split_at_recv && opline->opcode == ZEND_RECV_INIT) { + if (opline > op_array->opcodes && + (opline-1)->opcode == ZEND_RECV_INIT) { + /* repeatable opcode */ + continue; + } else { + for (i = 0; (opline+i)->opcode == ZEND_RECV_INIT; i++) { + } + zend_jit_jmp(&dasm_state, b + i); + zend_jit_label(&dasm_state, ssa.cfg.blocks_count + b); + for (i = 1; (opline+i)->opcode == ZEND_RECV_INIT; i++) { + zend_jit_label(&dasm_state, ssa.cfg.blocks_count + b + i); + } + zend_jit_prologue(&dasm_state); + } + } else { + if (ssa.cfg.blocks[b].flags & (ZEND_BB_TARGET|ZEND_BB_RECV_ENTRY)) { + zend_jit_jmp(&dasm_state, b); + } + zend_jit_label(&dasm_state, ssa.cfg.blocks_count + b); + zend_jit_prologue(&dasm_state); + } } #endif zend_jit_label(&dasm_state, b); @@ -645,13 +662,14 @@ ZEND_API int zend_jit(zend_op_array *op_array, zend_script *script) for (i = ssa.cfg.blocks[b].start; i <= end; i++) { opline = op_array->opcodes + i; switch (opline->opcode) { - case ZEND_RECV: case ZEND_RECV_INIT: - /* RECV may be skipped */ - if (!zend_jit_recv(&dasm_state, opline)) { - goto jit_failure; + if (ssa.cfg.split_at_recv) { + if (!zend_jit_handler(&dasm_state, opline, zend_may_throw(opline, op_array, &ssa))) { + goto jit_failure; + } + break; } - break; + /* break missing intentionally */ case ZEND_BIND_GLOBAL: if (opline->opcode != op_array->opcodes[i+1].opcode) { /* repeatable opcodes */ diff --git a/ext/opcache/jit/zend_jit_disasm_x86.c b/ext/opcache/jit/zend_jit_disasm_x86.c index 0132eb6050194..f2b4beb0060e7 100644 --- a/ext/opcache/jit/zend_jit_disasm_x86.c +++ b/ext/opcache/jit/zend_jit_disasm_x86.c @@ -262,7 +262,7 @@ static int zend_jit_disasm(const char *name, if (op_array && cfg) { ZVAL_FALSE(&zv); for (b = 0; b < cfg->blocks_count; b++) { - if (cfg->blocks[b].flags & ZEND_BB_ENTRY) { + if (cfg->blocks[b].flags & (ZEND_BB_ENTRY|ZEND_BB_RECV_ENTRY)) { addr = (uint64_t)(uintptr_t)op_array->opcodes[cfg->blocks[b].start].handler; if (addr >= (uint64_t)(uintptr_t)start && addr < (uint64_t)(uintptr_t)end) { zend_hash_index_add(&labels, addr, &zv); diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 0ef423119b43f..673b463323908 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -236,15 +236,6 @@ static int zend_jit_smart_branch(dasm_State **Dst, zend_op *opline, unsigned int return 1; } -static int zend_jit_recv(dasm_State **Dst, zend_op *opline) -{ - | cmp IPl, opline - | jnz >1 - zend_jit_handler(Dst, opline, 1); - |1: - return 1; -} - #ifdef CONTEXT_THREDED_JIT static int zend_jit_context_threaded_call(dasm_State **Dst, zend_op *opline) { From 2ebeed7e084a8cc59730c51e3f9af401c47929e6 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Tue, 30 Aug 2016 11:51:49 +0800 Subject: [PATCH 081/569] minor improvement --- ext/opcache/jit/zend_jit_disasm_x86.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/ext/opcache/jit/zend_jit_disasm_x86.c b/ext/opcache/jit/zend_jit_disasm_x86.c index f2b4beb0060e7..654ed7debe640 100644 --- a/ext/opcache/jit/zend_jit_disasm_x86.c +++ b/ext/opcache/jit/zend_jit_disasm_x86.c @@ -48,7 +48,7 @@ static struct ud ud; typedef struct _sym_node { uint64_t addr; - uint64_t size; + uint64_t end; struct _sym_node *parent; struct _sym_node *child[2]; unsigned char info; @@ -105,7 +105,7 @@ static void zend_jit_disasm_add_symbol(const char *name, return; } sym->addr = addr; - sym->size = size; + sym->end = (addr + size - 1); memcpy((char*)&sym->name, name, len + 1); sym->parent = sym->child[0] = sym->child[1] = NULL; sym->info = 1; @@ -115,7 +115,7 @@ static void zend_jit_disasm_add_symbol(const char *name, /* insert it into rbtree */ do { if (sym->addr > node->addr) { - ZEND_ASSERT(sym->addr >= (node->addr + node->size)); + ZEND_ASSERT(sym->addr > (node->end)); if (node->child[1]) { node = node->child[1]; } else { @@ -197,8 +197,7 @@ static const char* zend_jit_disasm_find_symbol(uint64_t addr, while (node) { if (addr < node->addr) { node = node->child[0]; - } else if (addr > node->addr && - addr >= (node->addr + node->size)) { + } else if (addr > node->end) { node = node->child[1]; } else { *offset = addr - node->addr; From 7cf0240ce58467355fb993278629deeab6f52e9b Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Tue, 30 Aug 2016 14:24:21 +0800 Subject: [PATCH 082/569] Fixed segfault if size is smaller than hugepage size --- ext/opcache/jit/zend_jit.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 16f1a2ece313b..3f426523f56e0 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -235,6 +235,10 @@ static void *jit_alloc(size_t size, int shared) #ifdef _WIN32 return VirtualAlloc(0, size, MEM_COMMIT | MEM_RESERVE, PAGE_EXECUTE_READWRITE); #else +#ifdef MAP_HUGETLB + zend_ulong huge_page_size = 2 * 1024 * 1024; +#endif + void *p; int prot; @@ -249,10 +253,12 @@ static void *jit_alloc(size_t size, int shared) # endif # ifdef MAP_HUGETLB - p = mmap(NULL, size, prot, - (shared ? MAP_SHARED : MAP_PRIVATE) | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0); - if (p != MAP_FAILED) { - return (void*)p; + if (size >= huge_page_size) { + p = mmap(NULL, size, prot, + (shared ? MAP_SHARED : MAP_PRIVATE) | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0); + if (p != MAP_FAILED) { + return (void*)p; + } } # endif p = mmap(NULL, size, prot, From f912c2294f386fbd60ffd5f3b0d1321f4d1efedb Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Tue, 30 Aug 2016 14:26:36 +0800 Subject: [PATCH 083/569] Revert "Fixed segfault if size is smaller than hugepage size" This reverts commit 7cf0240ce58467355fb993278629deeab6f52e9b. --- ext/opcache/jit/zend_jit.c | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 3f426523f56e0..16f1a2ece313b 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -235,10 +235,6 @@ static void *jit_alloc(size_t size, int shared) #ifdef _WIN32 return VirtualAlloc(0, size, MEM_COMMIT | MEM_RESERVE, PAGE_EXECUTE_READWRITE); #else -#ifdef MAP_HUGETLB - zend_ulong huge_page_size = 2 * 1024 * 1024; -#endif - void *p; int prot; @@ -253,12 +249,10 @@ static void *jit_alloc(size_t size, int shared) # endif # ifdef MAP_HUGETLB - if (size >= huge_page_size) { - p = mmap(NULL, size, prot, - (shared ? MAP_SHARED : MAP_PRIVATE) | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0); - if (p != MAP_FAILED) { - return (void*)p; - } + p = mmap(NULL, size, prot, + (shared ? MAP_SHARED : MAP_PRIVATE) | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0); + if (p != MAP_FAILED) { + return (void*)p; } # endif p = mmap(NULL, size, prot, From 6ddfd4d943f7a5b9bcf7137fb18f2e3717f0d6dc Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Tue, 30 Aug 2016 14:27:59 +0800 Subject: [PATCH 084/569] better fix again --- ext/opcache/jit/zend_jit.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 16f1a2ece313b..4f20784dc6b27 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -226,7 +226,12 @@ static size_t jit_page_size(void) GetSystemInfo(&system_info); return system_info.dwPageSize; #else +# ifdef MAP_HUGETLB + /* hardcode hugepage size */ + return 2 * 1024 * 1024; +# else return getpagesize(); +# endif #endif } From 46d2dea9cdd5b7b27370bfd4a787856581907006 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Tue, 30 Aug 2016 14:29:17 +0800 Subject: [PATCH 085/569] Make this configuration consistent with other memory size configration --- ext/opcache/ZendAccelerator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opcache/ZendAccelerator.c b/ext/opcache/ZendAccelerator.c index 52834edaedfe3..dcf933d4046b6 100644 --- a/ext/opcache/ZendAccelerator.c +++ b/ext/opcache/ZendAccelerator.c @@ -2770,7 +2770,7 @@ static int accel_startup(zend_extension *extension) zend_shared_alloc_lock(); #ifdef HAVE_JIT if (ZCG(accel_directives).jit_buffer_size) { - zend_jit_startup(ZCG(accel_directives).jit_buffer_size); + zend_jit_startup(ZCG(accel_directives).jit_buffer_size * 1024 * 1024); } #endif zend_shared_alloc_save_state(); From 558ffba349208f365564d4fc657e61720f13c66d Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 30 Aug 2016 10:36:48 +0300 Subject: [PATCH 086/569] Revert "Make this configuration consistent with other memory size configration" This reverts commit 46d2dea9cdd5b7b27370bfd4a787856581907006. --- ext/opcache/ZendAccelerator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opcache/ZendAccelerator.c b/ext/opcache/ZendAccelerator.c index dcf933d4046b6..52834edaedfe3 100644 --- a/ext/opcache/ZendAccelerator.c +++ b/ext/opcache/ZendAccelerator.c @@ -2770,7 +2770,7 @@ static int accel_startup(zend_extension *extension) zend_shared_alloc_lock(); #ifdef HAVE_JIT if (ZCG(accel_directives).jit_buffer_size) { - zend_jit_startup(ZCG(accel_directives).jit_buffer_size * 1024 * 1024); + zend_jit_startup(ZCG(accel_directives).jit_buffer_size); } #endif zend_shared_alloc_save_state(); From 9c0d7361b1884fd7c177ebf4eb60e9b876eaf343 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 30 Aug 2016 11:08:39 +0300 Subject: [PATCH 087/569] Use HUGE PAGES only if specified jit_buffer_size is multiple of 2MB --- ext/opcache/jit/zend_jit.c | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 4f20784dc6b27..4c2a54e3b0dde 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -226,12 +226,7 @@ static size_t jit_page_size(void) GetSystemInfo(&system_info); return system_info.dwPageSize; #else -# ifdef MAP_HUGETLB - /* hardcode hugepage size */ - return 2 * 1024 * 1024; -# else return getpagesize(); -# endif #endif } @@ -242,6 +237,9 @@ static void *jit_alloc(size_t size, int shared) #else void *p; int prot; +# ifdef MAP_HUGETLB + size_t huge_page_size = 2 * 1024 * 1024; +# endif # ifdef HAVE_MPROTECT if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_GDB) { @@ -254,10 +252,12 @@ static void *jit_alloc(size_t size, int shared) # endif # ifdef MAP_HUGETLB - p = mmap(NULL, size, prot, - (shared ? MAP_SHARED : MAP_PRIVATE) | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0); - if (p != MAP_FAILED) { - return (void*)p; + if (size >= huge_page_size && size % huge_page_size == 0) { + p = mmap(NULL, size, prot, + (shared ? MAP_SHARED : MAP_PRIVATE) | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0); + if (p != MAP_FAILED) { + return (void*)p; + } } # endif p = mmap(NULL, size, prot, @@ -835,7 +835,9 @@ ZEND_API void zend_jit_unprotect(void) { #ifdef HAVE_MPROTECT if (!(ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_GDB)) { - mprotect(dasm_buf, ((char*)dasm_end) - ((char*)dasm_buf), PROT_READ | PROT_WRITE); + if (mprotect(dasm_buf, ((char*)dasm_end) - ((char*)dasm_buf), PROT_READ | PROT_WRITE) != 0) { + fprintf(stderr, "mprotect() failed [%d] %s\n", errno, strerror(errno)); + } } #endif } @@ -844,7 +846,9 @@ ZEND_API void zend_jit_protect(void) { #ifdef HAVE_MPROTECT if (!(ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_GDB)) { - mprotect(dasm_buf, ((char*)dasm_end) - ((char*)dasm_buf), PROT_READ | PROT_EXEC); + if (mprotect(dasm_buf, ((char*)dasm_end) - ((char*)dasm_buf), PROT_READ | PROT_EXEC) != 0) { + fprintf(stderr, "mprotect() failed [%d] %s\n", errno, strerror(errno)); + } } #endif } From d9411789bc83f130d9e6e9fef3c54c08f7f91043 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 30 Aug 2016 15:44:55 +0300 Subject: [PATCH 088/569] Map JIT code buffer into low 32-bit address (if possible), to generate more efficient machine code --- ext/opcache/jit/zend_jit.c | 44 ++++++++++++++++++++++++++----- ext/opcache/jit/zend_jit_x86.dasc | 22 ++++++++++++++++ 2 files changed, 59 insertions(+), 7 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 4c2a54e3b0dde..1e5fc5c5bc0d0 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -37,6 +37,7 @@ #define ZEND_JIT_LEVEL ZEND_JIT_LEVEL_FULL //#define CONTEXT_THREDED_JIT +#define PREFER_MAP_32BIT #define JIT_PREFIX "JIT$" #define JIT_STUB_PREFIX "JIT$$" @@ -53,6 +54,10 @@ typedef struct _zend_jit_stub { #define JIT_STUB(name) \ {JIT_STUB_PREFIX #name, zend_jit_ ## name ## _stub} +static void *dasm_buf = NULL; +static void *dasm_ptr = NULL; +static void *dasm_end = NULL; + #include "dynasm/dasm_x86.h" #include "jit/zend_jit_x86.c" #include "jit/zend_jit_disasm_x86.c" @@ -73,10 +78,6 @@ typedef struct _zend_jit_stub { #define DASM_ALIGNMENT 16 -static void *dasm_buf = NULL; -static void *dasm_ptr = NULL; -static void *dasm_end = NULL; - static zend_string *zend_jit_func_name(zend_op_array *op_array) { smart_str buf = {0}; @@ -253,20 +254,49 @@ static void *jit_alloc(size_t size, int shared) # ifdef MAP_HUGETLB if (size >= huge_page_size && size % huge_page_size == 0) { +# if defined(PREFER_MAP_32BIT) && defined(__x86_64__) && defined(MAP_32BIT) + /* to got HUGE PAGES in low 32-bit address we have to reseve address + space and then remap it using MAP_HUGETLB */ + p = mmap(NULL, size, prot, + (shared ? MAP_SHARED : MAP_PRIVATE) | MAP_ANONYMOUS | MAP_32BIT, -1, 0); + if (p != MAP_FAILED) { + void *p2 = mmap(p, size, prot, + (shared ? MAP_SHARED : MAP_PRIVATE) | MAP_ANONYMOUS | MAP_HUGETLB | MAP_FIXED, -1, 0); + if (p2 != MAP_FAILED) { + return p2; + } else { + return p; + } + } +# endif p = mmap(NULL, size, prot, (shared ? MAP_SHARED : MAP_PRIVATE) | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0); if (p != MAP_FAILED) { - return (void*)p; + return p; } +# if defined(PREFER_MAP_32BIT) && defined(__x86_64__) && defined(MAP_32BIT) + } else { + p = mmap(NULL, size, prot, + (shared ? MAP_SHARED : MAP_PRIVATE) | MAP_ANONYMOUS | MAP_32BIT, -1, 0); + if (p != MAP_FAILED) { + return p; + } +# endif + } +# elif defined(PREFER_MAP_32BIT) && defined(__x86_64__) && defined(MAP_32BIT) + p = mmap(NULL, size, prot, + (shared ? MAP_SHARED : MAP_PRIVATE) | MAP_ANONYMOUS | MAP_32BIT, -1, 0); + if (p != MAP_FAILED) { + return p; } # endif p = mmap(NULL, size, prot, (shared ? MAP_SHARED : MAP_PRIVATE) | MAP_ANONYMOUS, -1, 0); - if (p == MAP_FAILED) { + if (p == MAP_FAILED) { return NULL; } - return (void*)p; + return (void*)p; #endif } diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 673b463323908..4c990dd2ec29c 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -68,6 +68,7 @@ static void* dasm_labels[zend_lb_MAX]; ||} |.endmacro +#define IS_32BIT(addr) (((uintptr_t)(addr)) <= 0x7fffffff) static int zend_jit_interrupt_handler_stub(dasm_State **Dst) { @@ -80,8 +81,12 @@ static int zend_jit_interrupt_handler_stub(dasm_State **Dst) | //zend_timeout(0); |.if X64 | xor CARG1d, CARG1d + if (IS_32BIT(dasm_end) && IS_32BIT(zend_timeout)) { + | call qword &zend_timeout + } else { | load_addr rax, zend_timeout | call rax + } |.else | push 0 | call dword &zend_timeout @@ -93,9 +98,14 @@ static int zend_jit_interrupt_handler_stub(dasm_State **Dst) | mov EX->opline, IP | //zend_interrupt_function(execute_data); |.if X64 + if (IS_32BIT(dasm_end) && IS_32BIT(zend_interrupt_function)) { + | mov CARG1, FP + | call qword &zend_interrupt_function + } else { | load_addr rax, zend_interrupt_function | mov CARG1, FP | call rax + } |.else | push FP | call dword &zend_interrupt_function @@ -119,8 +129,12 @@ static int zend_jit_exception_handler_stub(dasm_State **Dst) |->exception_handler: | add r4, SPAD // stack alignment |.if X64 + if (IS_32BIT(dasm_end) && IS_32BIT(EG(exception_op)->handler)) { + | jmp qword &(EG(exception_op)->handler) + } else { | load_addr rax, EG(exception_op)->handler | jmp rax + } |.else | jmp dword &(EG(exception_op)->handler) |.endif @@ -170,8 +184,12 @@ static int zend_jit_handler(dasm_State **Dst, zend_op *opline, int may_throw) const void *handler = opline->handler; |.if X64 + if (IS_32BIT(dasm_end) && IS_32BIT(handler)) { + | call qword &handler + } else { | load_addr rax, handler | call rax + } |.else | call dword &handler |.endif @@ -187,8 +205,12 @@ static int zend_jit_tail_handler(dasm_State **Dst, zend_op *opline) | add r4, SPAD // stack alignment |.if X64 + if (IS_32BIT(dasm_end) && IS_32BIT(handler)) { + | jmp qword &handler + } else { | load_addr rax, handler | jmp rax + } |.else | jmp dword &handler |.endif From f9abe6b0e3fec721ae865aeaa52649e62e570d16 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 30 Aug 2016 15:56:06 +0300 Subject: [PATCH 089/569] Resolve EG(current_execute_data) address --- ext/opcache/jit/zend_jit_disasm_x86.c | 1 + 1 file changed, 1 insertion(+) diff --git a/ext/opcache/jit/zend_jit_disasm_x86.c b/ext/opcache/jit/zend_jit_disasm_x86.c index 654ed7debe640..c2fb58102a41b 100644 --- a/ext/opcache/jit/zend_jit_disasm_x86.c +++ b/ext/opcache/jit/zend_jit_disasm_x86.c @@ -367,6 +367,7 @@ static int zend_jit_disasm_init(void) REGISTER_EG(vm_interrupt); REGISTER_EG(exception_op); REGISTER_EG(timed_out); + REGISTER_EG(current_execute_data); #undef REGISTER_EG #define REGISTER_CG(n) \ zend_jit_disasm_add_symbol("CG("#n")", \ From 87c85522e6b2f260f5683a88f0a0828b6b81e0f8 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 30 Aug 2016 17:38:35 +0300 Subject: [PATCH 090/569] Improve zend_may_throw() --- ext/opcache/jit/zend_jit.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 1e5fc5c5bc0d0..579851e478131 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -337,7 +337,7 @@ static int zend_may_throw(zend_op *opline, zend_op_array *op_array, zend_ssa *ss } } } else if (opline->op1_type & (IS_TMP_VAR|IS_VAR)) { - if (t1 & (MAY_BE_OBJECT|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_RESOURCE)) { + if (t1 & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_ARRAY)) { switch (opline->opcode) { case ZEND_CASE: case ZEND_FE_FETCH_R: @@ -371,7 +371,7 @@ static int zend_may_throw(zend_op *opline, zend_op_array *op_array, zend_ssa *ss } } } else if (opline->op2_type & (IS_TMP_VAR|IS_VAR)) { - if (t2 & (MAY_BE_OBJECT|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_RESOURCE)) { + if (t2 & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_ARRAY)) { switch (opline->opcode) { case ZEND_ASSIGN: break; @@ -397,6 +397,10 @@ static int zend_may_throw(zend_op *opline, zend_op_array *op_array, zend_ssa *ss case ZEND_SEND_REF: case ZEND_FREE: case ZEND_SEPARATE: + case ZEND_TYPE_CHECK: + case ZEND_DEFINED: + case ZEND_ISSET_ISEMPTY_THIS: + case ZEND_COALESCE: return 0; case ZEND_BIND_GLOBAL: if ((opline+1)->opcode == ZEND_BIND_GLOBAL) { @@ -464,6 +468,7 @@ static int zend_may_throw(zend_op *opline, zend_op_array *op_array, zend_ssa *ss case ZEND_JMPZ_EX: case ZEND_JMPNZ_EX: case ZEND_BOOL: + case ZEND_JMP_SET: return (t1 & MAY_BE_OBJECT); case ZEND_BOOL_XOR: return (t1 & MAY_BE_OBJECT) || (t2 & MAY_BE_OBJECT); @@ -472,6 +477,7 @@ static int zend_may_throw(zend_op *opline, zend_op_array *op_array, zend_ssa *ss case ZEND_IS_SMALLER: case ZEND_IS_SMALLER_OR_EQUAL: case ZEND_CASE: + case ZEND_SPACESHIP: if ((t1 & MAY_BE_ANY) == MAY_BE_NULL || (t2 & MAY_BE_ANY) == MAY_BE_NULL) { return 0; @@ -543,9 +549,10 @@ static int zend_may_throw(zend_op *opline, zend_op_array *op_array, zend_ssa *ss return (t1 & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT)) || (t2 & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT)); case ZEND_ASSIGN: - return (t1 & (MAY_BE_OBJECT|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_RESOURCE)); + return (t1 & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_ARRAY)); case ZEND_ROPE_INIT: case ZEND_ROPE_ADD: + case ZEND_ROPE_END: return t2 & (MAY_BE_ARRAY|MAY_BE_OBJECT); case ZEND_INIT_ARRAY: case ZEND_ADD_ARRAY_ELEMENT: From 406a1a4594c7afde5b3b634def4e36be1bd8565e Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 30 Aug 2016 17:46:29 +0300 Subject: [PATCH 091/569] Don't check exception after ZEND_INIT_FCALL --- ext/opcache/jit/zend_jit.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 579851e478131..ce7ebde79c6a6 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -402,6 +402,9 @@ static int zend_may_throw(zend_op *opline, zend_op_array *op_array, zend_ssa *ss case ZEND_ISSET_ISEMPTY_THIS: case ZEND_COALESCE: return 0; + case ZEND_INIT_FCALL: + /* can't throw, because call is resolved at compile time */ + return 0; case ZEND_BIND_GLOBAL: if ((opline+1)->opcode == ZEND_BIND_GLOBAL) { return zend_may_throw(opline + 1, op_array, ssa); From f2ad5df4106f151266e1b68b4cc71523ed47ab23 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Tue, 30 Aug 2016 23:39:26 +0800 Subject: [PATCH 092/569] good for simply remove it --- ext/opcache/jit/zend_jit_gdb.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ext/opcache/jit/zend_jit_gdb.c b/ext/opcache/jit/zend_jit_gdb.c index 900fac9f52b3a..5062037da849d 100644 --- a/ext/opcache/jit/zend_jit_gdb.c +++ b/ext/opcache/jit/zend_jit_gdb.c @@ -22,6 +22,8 @@ #define HAVE_GDB +#ifdef HAVE_GDB + #include "zend_elf.h" #include "zend_gdb.h" @@ -486,6 +488,7 @@ static void zend_jit_gdb_init(void) #endif } +#endif /* * Local variables: * tab-width: 4 From cd5418c14e423f8407a18944fcf882aa2044759d Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Wed, 31 Aug 2016 12:27:13 +0800 Subject: [PATCH 093/569] Fixed segfault if mmap HUGETLB|FIXED failed --- ext/opcache/jit/zend_jit.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index ce7ebde79c6a6..71b250488daf3 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -265,7 +265,14 @@ static void *jit_alloc(size_t size, int shared) if (p2 != MAP_FAILED) { return p2; } else { - return p; + /* If mmap failed, p becomes unaccessable, + * however, Let's munmap it explictly in case it is not discarded */ + munmap(p, size); + p = mmap(NULL, size, prot, + (shared ? MAP_SHARED : MAP_PRIVATE) | MAP_ANONYMOUS | MAP_32BIT, -1, 0); + if (p != MAP_FAILED) { + return p; + } } } # endif From 8481878b2812b26f2881d8aaa2fd3b28d89cdb0b Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 31 Aug 2016 08:54:16 +0300 Subject: [PATCH 094/569] Revert "good for simply remove it" This reverts commit f2ad5df4106f151266e1b68b4cc71523ed47ab23. --- ext/opcache/jit/zend_jit_gdb.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/ext/opcache/jit/zend_jit_gdb.c b/ext/opcache/jit/zend_jit_gdb.c index 5062037da849d..900fac9f52b3a 100644 --- a/ext/opcache/jit/zend_jit_gdb.c +++ b/ext/opcache/jit/zend_jit_gdb.c @@ -22,8 +22,6 @@ #define HAVE_GDB -#ifdef HAVE_GDB - #include "zend_elf.h" #include "zend_gdb.h" @@ -488,7 +486,6 @@ static void zend_jit_gdb_init(void) #endif } -#endif /* * Local variables: * tab-width: 4 From 1f473de3b27a131d88f2f0b7b33852084686412e Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 31 Aug 2016 09:12:39 +0300 Subject: [PATCH 095/569] Revert "Revert "good for simply remove it"" This reverts commit 8481878b2812b26f2881d8aaa2fd3b28d89cdb0b. --- ext/opcache/jit/zend_jit_gdb.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ext/opcache/jit/zend_jit_gdb.c b/ext/opcache/jit/zend_jit_gdb.c index 900fac9f52b3a..5062037da849d 100644 --- a/ext/opcache/jit/zend_jit_gdb.c +++ b/ext/opcache/jit/zend_jit_gdb.c @@ -22,6 +22,8 @@ #define HAVE_GDB +#ifdef HAVE_GDB + #include "zend_elf.h" #include "zend_gdb.h" @@ -486,6 +488,7 @@ static void zend_jit_gdb_init(void) #endif } +#endif /* * Local variables: * tab-width: 4 From 1d4e00f554787cf87965c81be1ca39fb0ad4a9f0 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Wed, 31 Aug 2016 20:07:31 +0800 Subject: [PATCH 096/569] Fixed 32bits and hugepage supports --- ext/opcache/jit/zend_jit.c | 29 ++++++++++++----------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 71b250488daf3..72c2dc7df43b5 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -252,53 +252,48 @@ static void *jit_alloc(size_t size, int shared) prot = PROT_EXEC | PROT_READ | PROT_WRITE; # endif + shared = shared? MAP_SHARED : MAP_PRIVATE; + # ifdef MAP_HUGETLB if (size >= huge_page_size && size % huge_page_size == 0) { # if defined(PREFER_MAP_32BIT) && defined(__x86_64__) && defined(MAP_32BIT) + void *p2; /* to got HUGE PAGES in low 32-bit address we have to reseve address space and then remap it using MAP_HUGETLB */ - p = mmap(NULL, size, prot, - (shared ? MAP_SHARED : MAP_PRIVATE) | MAP_ANONYMOUS | MAP_32BIT, -1, 0); + p = mmap(NULL, size, prot, shared | MAP_ANONYMOUS | MAP_32BIT, -1, 0); if (p != MAP_FAILED) { - void *p2 = mmap(p, size, prot, - (shared ? MAP_SHARED : MAP_PRIVATE) | MAP_ANONYMOUS | MAP_HUGETLB | MAP_FIXED, -1, 0); + munmap(p, size); + p = (void*)(ZEND_MM_ALIGNED_SIZE_EX((ptrdiff_t)p, huge_page_size)); + p2 = mmap(p, size, prot, shared | MAP_ANONYMOUS | MAP_HUGETLB | MAP_FIXED, -1, 0); if (p2 != MAP_FAILED) { return p2; } else { - /* If mmap failed, p becomes unaccessable, - * however, Let's munmap it explictly in case it is not discarded */ - munmap(p, size); - p = mmap(NULL, size, prot, - (shared ? MAP_SHARED : MAP_PRIVATE) | MAP_ANONYMOUS | MAP_32BIT, -1, 0); + p = mmap(NULL, size, prot, shared | MAP_ANONYMOUS | MAP_32BIT, -1, 0); if (p != MAP_FAILED) { return p; } } } # endif - p = mmap(NULL, size, prot, - (shared ? MAP_SHARED : MAP_PRIVATE) | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0); + p = mmap(NULL, size, prot, shared | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0); if (p != MAP_FAILED) { return p; } # if defined(PREFER_MAP_32BIT) && defined(__x86_64__) && defined(MAP_32BIT) } else { - p = mmap(NULL, size, prot, - (shared ? MAP_SHARED : MAP_PRIVATE) | MAP_ANONYMOUS | MAP_32BIT, -1, 0); + p = mmap(NULL, size, prot, shared | MAP_ANONYMOUS | MAP_32BIT, -1, 0); if (p != MAP_FAILED) { return p; } # endif } # elif defined(PREFER_MAP_32BIT) && defined(__x86_64__) && defined(MAP_32BIT) - p = mmap(NULL, size, prot, - (shared ? MAP_SHARED : MAP_PRIVATE) | MAP_ANONYMOUS | MAP_32BIT, -1, 0); + p = mmap(NULL, size, prot, shared | MAP_ANONYMOUS | MAP_32BIT, -1, 0); if (p != MAP_FAILED) { return p; } # endif - p = mmap(NULL, size, prot, - (shared ? MAP_SHARED : MAP_PRIVATE) | MAP_ANONYMOUS, -1, 0); + p = mmap(NULL, size, prot, shared | MAP_ANONYMOUS, -1, 0); if (p == MAP_FAILED) { return NULL; } From 288f6ba7781ce2005e2d140b70073a97ef277e15 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Thu, 1 Sep 2016 17:11:17 +0800 Subject: [PATCH 097/569] Prevent try to jit if not enough memory is avaliable --- ext/opcache/jit/zend_jit.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 72c2dc7df43b5..ddd1c72891244 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -124,6 +124,7 @@ static void *dasm_link_and_encode(dasm_State **dasm_state, } if ((void*)((char*)dasm_ptr + size) > dasm_end) { + dasm_ptr = dasm_end; //prevent further try // TODO: jit_buffer_size overflow ??? return NULL; } @@ -579,7 +580,7 @@ ZEND_API int zend_jit(zend_op_array *op_array, zend_script *script) dasm_State* dasm_state = NULL; void *handler; - if (!dasm_buf) { + if (dasm_ptr == dasm_end) { return FAILURE; } From aabb73e233b554b5530a581b3087fb1926f92ddf Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 1 Sep 2016 13:35:09 +0300 Subject: [PATCH 098/569] SEND_VAR_EX can't throw, RECV_INIT may throw only if default value is an unresolved constant or because of type hinting. --- ext/opcache/jit/zend_jit.c | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index ddd1c72891244..1aa0aa693c598 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -398,6 +398,7 @@ static int zend_may_throw(zend_op *opline, zend_op_array *op_array, zend_ssa *ss case ZEND_END_SILENCE: case ZEND_SEND_VAL: case ZEND_SEND_REF: + case ZEND_SEND_VAR_EX: case ZEND_FREE: case ZEND_SEPARATE: case ZEND_TYPE_CHECK: @@ -565,6 +566,25 @@ static int zend_may_throw(zend_op *opline, zend_op_array *op_array, zend_ssa *ss return t2 & (MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE); case ZEND_STRLEN: return (t1 & MAY_BE_ANY) != MAY_BE_STRING; + case ZEND_RECV_INIT: + if (Z_CONSTANT_P(RT_CONSTANT(op_array, opline->op2))) { + return 1; + } + if (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) { + uint32_t arg_num = opline->op1.num; + zend_arg_info *cur_arg_info; + + if (EXPECTED(arg_num <= op_array->num_args)) { + cur_arg_info = &op_array->arg_info[arg_num-1]; + } else if (UNEXPECTED(op_array->fn_flags & ZEND_ACC_VARIADIC)) { + cur_arg_info = &op_array->arg_info[op_array->num_args]; + } else { + return 0; + } + return (cur_arg_info->type_hint != 0); + } else { + return 0; + } default: return 1; } From bd9263e811206e4686c4aabc862d241c2eaa9101 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 1 Sep 2016 14:03:53 +0300 Subject: [PATCH 099/569] Eliminated few more useless exception checks --- ext/opcache/jit/zend_jit.c | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 1aa0aa693c598..f27253d55fb7f 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -563,7 +563,7 @@ static int zend_may_throw(zend_op *opline, zend_op_array *op_array, zend_ssa *ss return t2 & (MAY_BE_ARRAY|MAY_BE_OBJECT); case ZEND_INIT_ARRAY: case ZEND_ADD_ARRAY_ELEMENT: - return t2 & (MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE); + return (opline->op2_type != IS_UNUSED) && (t2 & (MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)); case ZEND_STRLEN: return (t1 & MAY_BE_ANY) != MAY_BE_STRING; case ZEND_RECV_INIT: @@ -585,6 +585,27 @@ static int zend_may_throw(zend_op *opline, zend_op_array *op_array, zend_ssa *ss } else { return 0; } + case ZEND_ISSET_ISEMPTY_DIM_OBJ: + case ZEND_FETCH_DIM_IS: + return (t1 & MAY_BE_OBJECT) || (t2 & (MAY_BE_ARRAY|MAY_BE_OBJECT)); + case ZEND_CAST: + switch (opline->extended_value) { + case IS_NULL: + return 0; + case _IS_BOOL: + return (t1 & MAY_BE_OBJECT); + case IS_LONG: + case IS_DOUBLE: + return (t1 & (MAY_BE_STRING|MAY_BE_OBJECT)); + case IS_STRING: + return (t1 & (MAY_BE_ARRAY|MAY_BE_OBJECT)); + case IS_ARRAY: + return (t1 & MAY_BE_OBJECT); + case IS_OBJECT: + return (t1 & MAY_BE_ARRAY); + default: + return 1; + } default: return 1; } From 2fc7a0910c6514fcaa9071ccfa65e7071fe5cec2 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 1 Sep 2016 16:26:36 +0300 Subject: [PATCH 100/569] Allow run-time machine code generation of functions (on first execution). This might be enabled by uncommenting "#define ZEND_RUNTIME_JIT", but it's going to work only in single process environment yet. Support for multiprocess would require some synchronisation, to serialize updates of shared memory. --- ext/opcache/jit/zend_jit.c | 71 +++++++++++++++++++++++++++++++++++++- 1 file changed, 70 insertions(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index f27253d55fb7f..bbb79717786b9 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -18,6 +18,7 @@ #include #include "Zend/zend_execute.h" +#include "Zend/zend_vm.h" #include "zend_smart_str.h" #include "jit/zend_jit.h" @@ -38,6 +39,7 @@ //#define CONTEXT_THREDED_JIT #define PREFER_MAP_32BIT +//#define ZEND_RUNTIME_JIT #define JIT_PREFIX "JIT$" #define JIT_STUB_PREFIX "JIT$$" @@ -611,7 +613,7 @@ static int zend_may_throw(zend_op *opline, zend_op_array *op_array, zend_ssa *ss } } -ZEND_API int zend_jit(zend_op_array *op_array, zend_script *script) +static int zend_real_jit_func(zend_op_array *op_array, zend_script *script) { uint32_t flags = 0; zend_ssa ssa; @@ -915,6 +917,73 @@ ZEND_API int zend_jit(zend_op_array *op_array, zend_script *script) return FAILURE; } + +#ifdef ZEND_RUNTIME_JIT + +/* memory write protection */ +void zend_accel_shared_protect(int mode); + +#define SHM_PROTECT() \ + do { \ + if (ZCG(accel_directives).protect_memory) { \ + zend_accel_shared_protect(1); \ + } \ + } while (0) +#define SHM_UNPROTECT() \ + do { \ + if (ZCG(accel_directives).protect_memory) { \ + zend_accel_shared_protect(0); \ + } \ + } while (0) + +typedef void (ZEND_FASTCALL *zend_vm_opcode_handler_t)(void); + +/* Run-time JIT handler */ +static void ZEND_FASTCALL zend_runtime_jit(void) +{ + zend_execute_data *execute_data = EG(current_execute_data); + zend_op_array *op_array = &EX(func)->op_array; + zend_op *opline = op_array->opcodes; + + SHM_UNPROTECT(); + zend_jit_unprotect(); + + /* restore original opcode handlers */ + while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) { + zend_vm_set_opcode_handler(opline); + opline++; + } + opline->handler = ZEND_FUNC_INFO(op_array); + ZEND_SET_FUNC_INFO(op_array, NULL); + + /* perform real JIT for this function */ + zend_real_jit_func(op_array, NULL); + + zend_jit_protect(); + SHM_PROTECT(); + + /* JIT-ed code is going to be called by VM */ +} +#endif + +ZEND_API int zend_jit(zend_op_array *op_array, zend_script *script) +{ +#ifdef ZEND_RUNTIME_JIT + zend_op *opline = op_array->opcodes; + + /* Set run-time JIT handler */ + while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) { + opline->handler = (const void*)zend_runtime_jit; + opline++; + } + ZEND_SET_FUNC_INFO(op_array, (void*)opline->handler); + opline->handler = (const void*)zend_runtime_jit; + return SUCCESS; +#else + return zend_real_jit_func(op_array, script); +#endif +} + ZEND_API void zend_jit_unprotect(void) { #ifdef HAVE_MPROTECT From 3e6d18a4463f5d52cb96eee831d74f14b861a44a Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 2 Sep 2016 11:06:07 +0300 Subject: [PATCH 101/569] Collect SCC info to enable range inference. --- ext/opcache/jit/zend_jit.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index bbb79717786b9..b9d41c58411fe 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -667,6 +667,10 @@ static int zend_real_jit_func(zend_op_array *op_array, zend_script *script) goto jit_failure; } + if (zend_ssa_find_sccs(op_array, &ssa) != SUCCESS){ + goto jit_failure; + } + if (zend_ssa_inference(&CG(arena), op_array, script, &ssa) != SUCCESS) { goto jit_failure; } From 6025654eba0a3409bbe4c6abe4a47f179c5f017e Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 2 Sep 2016 11:17:36 +0300 Subject: [PATCH 102/569] Implemented JIT for PRE_INC/PRE_DEC(IS_LONG) --- ext/opcache/jit/zend_jit.c | 10 +++++ ext/opcache/jit/zend_jit_x86.dasc | 62 +++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index b9d41c58411fe..401413cb214fd 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -60,6 +60,8 @@ static void *dasm_buf = NULL; static void *dasm_ptr = NULL; static void *dasm_end = NULL; +static int zend_may_throw(zend_op *opline, zend_op_array *op_array, zend_ssa *ssa); + #include "dynasm/dasm_x86.h" #include "jit/zend_jit_x86.c" #include "jit/zend_jit_disasm_x86.c" @@ -757,6 +759,14 @@ static int zend_real_jit_func(zend_op_array *op_array, zend_script *script) for (i = ssa.cfg.blocks[b].start; i <= end; i++) { opline = op_array->opcodes + i; switch (opline->opcode) { +#if ZEND_JIT_LEVEL >= ZEND_JIT_LEVEL_FULL + case ZEND_PRE_INC: + case ZEND_PRE_DEC: + if (!zend_jit_inc_dec(&dasm_state, opline, op_array, &ssa)) { + goto jit_failure; + } + break; +#endif case ZEND_RECV_INIT: if (ssa.cfg.split_at_recv) { if (!zend_jit_handler(&dasm_state, opline, zend_may_throw(opline, op_array, &ssa))) { diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 4c990dd2ec29c..10593e011624f 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -301,6 +301,68 @@ static int zend_jit_new(dasm_State **Dst, zend_op *opline) return 1; } +#if ZEND_JIT_LEVEL >= ZEND_JIT_LEVEL_FULL +static int zend_jit_inc_dec(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) +{ + uint32_t op1_info, op1_def_info; + + if (!ssa->ops || !ssa->var_info || opline->result_type != IS_UNUSED) { + goto fallback; + } + op1_info = OP1_INFO(); + + if (!(op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)-(MAY_BE_LONG/*|MAY_BE_DOUBLE*/)))) { + if (op1_info & MAY_BE_LONG) { + if (op1_info & (MAY_BE_ANY-MAY_BE_LONG)) { + | cmp dword [FP + opline->op1.var + 8], IS_LONG + | jne >2 + } + if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { + | inc aword [FP + opline->op1.var] + } else { + | dec aword [FP + opline->op1.var] + } + op1_def_info = OP1_DEF_INFO(); + if (op1_def_info & MAY_BE_DOUBLE) { + | jno >2 + if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { + |.if X64 + | mov64 rax, 0x43e0000000000000 + | mov qword [FP + opline->op1.var], rax + |.else + | mov dword [FP + opline->op1.var], 0 + | mov dword [FP + opline->op1.var + 4], 0x41e00000 + |.endif + } else { + |.if X64 + | mov64 rax, 0x43e0000000000000 + | mov qword [FP + opline->op1.var], rax + |.else + | mov dword [FP + opline->op1.var], 0x00200000 + | mov dword [FP + opline->op1.var + 4], 0xc1e00000 + |.endif + } + | mov dword [FP + opline->op1.var + 8], IS_DOUBLE + | jmp >1 + } else if (op1_info & (MAY_BE_ANY-MAY_BE_LONG)) { + | jmp >1 + } + } + if (op1_info & MAY_BE_DOUBLE) { + |2: + | + } + |1: + | add IP, sizeof(zend_op) + return 1; + } + +fallback: + /* fallback to subroutine threading */ + return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); +} +#endif + /* * Local variables: * tab-width: 4 From b5fa120a17ddefd64fe123f1df4864016acffb98 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 2 Sep 2016 12:01:15 +0300 Subject: [PATCH 103/569] Implemented JIT for ADD/SUB/MUL (IS_DOUBLE, IS_DOUBLE) --- ext/opcache/jit/zend_jit.c | 8 +++ ext/opcache/jit/zend_jit_x86.dasc | 116 ++++++++++++++++++++++++++++++ 2 files changed, 124 insertions(+) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 401413cb214fd..b7c5a44e7d373 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -766,6 +766,14 @@ static int zend_real_jit_func(zend_op_array *op_array, zend_script *script) goto jit_failure; } break; + case ZEND_ADD: + case ZEND_SUB: + case ZEND_MUL: +// case ZEND_DIV: // TODO: check for division by zero ??? + if (!zend_jit_math(&dasm_state, opline, op_array, &ssa)) { + goto jit_failure; + } + break; #endif case ZEND_RECV_INIT: if (ssa.cfg.split_at_recv) { diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 10593e011624f..498f736df9b98 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -302,6 +302,89 @@ static int zend_jit_new(dasm_State **Dst, zend_op *opline) } #if ZEND_JIT_LEVEL >= ZEND_JIT_LEVEL_FULL + +|.macro fp_op, fp_ins, op_type, op +|| if (op_type == IS_CONST) { +| .if X64 +| mov aword EX->literals, r0 +| fp_ins qword [r0 + op.constant] +| .else +| fp_ins qword [op.zv] +| .endif +|| } else { +| fp_ins qword [FP + op.var] +|| } +|.endmacro +| +|.macro fp_load, op_type, op +| fp_op fld, op_type, op +|.endmacro +| +|.macro fp_math, opcode, op_type, op +|| switch (opcode) { +|| case ZEND_ADD: +| fp_op fadd, op_type, op +|| break; +|| case ZEND_SUB: +| fp_op fsub, op_type, op +|| break; +|| case ZEND_MUL: +| fp_op fmul, op_type, op +|| break; +|| case ZEND_DIV: +| fp_op fdiv, op_type, op +|| break; +|| } +|.endmacro +| +|.macro fp_store, op +| fstp qword [FP + op.var] +| mov dword [FP + op.var + 8], IS_DOUBLE +|.endmacro +| +|.macro sse_op, sse_ins, op_type, op, reg +|| if (op_type == IS_CONST) { +| .if X64 +| mov aword EX->literals, r0 +| sse_ins reg, qword [r0 + op.constant] +| .else +| sse_ins reg, qword [op.zv] +| .endif +|| } else { +| sse_ins reg, qword [FP + op.var] +|| } +|.endmacro +| +|.macro sse_load, op_type, op, reg +| sse_op movsd, op_type, op, reg +|.endmacro +| +|.macro sse_math, opcode, op_type, op, reg +|| switch (opcode) { +|| case ZEND_ADD: +| sse_op addsd, op_type, op, reg +|| break; +|| case ZEND_SUB: +| sse_op subsd, op_type, op, reg +|| break; +|| case ZEND_MUL: +| sse_op mulsd, op_type, op, reg +|| break; +|| case ZEND_DIV: +| sse_op divsd, op_type, op, reg +|| break; +|| } +|.endmacro +| +|.macro sse_store, op, reg +| movsd qword [FP + op.var], reg +| mov dword [FP + op.var + 8], IS_DOUBLE +|.endmacro +| +|.macro next_opline +| add IP, sizeof(zend_op) +|.endmacro + static int zend_jit_inc_dec(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) { uint32_t op1_info, op1_def_info; @@ -357,6 +440,39 @@ static int zend_jit_inc_dec(dasm_State **Dst, zend_op *opline, zend_op_array *op return 1; } +fallback: + /* fallback to subroutine threading */ + return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); +} + +static int zend_jit_math(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) +{ + uint32_t op1_info, op2_info; + + if (!ssa->ops || !ssa->var_info) { + goto fallback; + } + op1_info = OP1_INFO(); + op2_info = OP2_INFO(); + + if (((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) && + ((op2_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE)) { + |.if X64 // TODO: or SSE ??? + | sse_load opline->op1_type, opline->op1, xmm0 + | sse_math opline->opcode, opline->op2_type, opline->op2, xmm0 + | sse_store opline->result, xmm0 + |.else + | fp_load opline->op1_type, opline->op1 + | fp_math opline->opcode, opline->op2_type, opline->op2 + | fp_store opline->result + |.endif + | next_opline + } else { + goto fallback; + } + + return 1; + fallback: /* fallback to subroutine threading */ return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); From 2fae2d9fc2ce457b0f86a2fdaba3e5fb726358b9 Mon Sep 17 00:00:00 2001 From: Yuji Uchiyama Date: Sat, 3 Sep 2016 11:42:35 +0900 Subject: [PATCH 104/569] Fix operands --- ext/opcache/jit/zend_jit_x86.dasc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 498f736df9b98..5085605ee2e12 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -306,7 +306,7 @@ static int zend_jit_new(dasm_State **Dst, zend_op *opline) |.macro fp_op, fp_ins, op_type, op || if (op_type == IS_CONST) { | .if X64 -| mov aword EX->literals, r0 +| mov r0, aword EX->literals | fp_ins qword [r0 + op.constant] | .else | fp_ins qword [op.zv] @@ -345,7 +345,7 @@ static int zend_jit_new(dasm_State **Dst, zend_op *opline) |.macro sse_op, sse_ins, op_type, op, reg || if (op_type == IS_CONST) { | .if X64 -| mov aword EX->literals, r0 +| mov r0, aword EX->literals | sse_ins reg, qword [r0 + op.constant] | .else | sse_ins reg, qword [op.zv] From d2fade7ef190440def1c43f88b3cb67851f33c5a Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 5 Sep 2016 10:16:25 +0300 Subject: [PATCH 105/569] Micro-optimisation for math instructions with the same operands ($a + $a). --- ext/opcache/jit/zend_jit_x86.dasc | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 5085605ee2e12..7e440f3d56c25 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -376,6 +376,23 @@ static int zend_jit_new(dasm_State **Dst, zend_op *opline) || } |.endmacro | +|.macro sse_math2, opcode, reg1, reg2 +|| switch (opcode) { +|| case ZEND_ADD: +| addsd reg2, reg1 +|| break; +|| case ZEND_SUB: +| subsd reg2, reg1 +|| break; +|| case ZEND_MUL: +| mulsd reg2, reg1 +|| break; +|| case ZEND_DIV: +| divsd reg2, reg1 +|| break; +|| } +|.endmacro +| |.macro sse_store, op, reg | movsd qword [FP + op.var], reg | mov dword [FP + op.var + 8], IS_DOUBLE @@ -459,7 +476,11 @@ static int zend_jit_math(dasm_State **Dst, zend_op *opline, zend_op_array *op_ar ((op2_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE)) { |.if X64 // TODO: or SSE ??? | sse_load opline->op1_type, opline->op1, xmm0 - | sse_math opline->opcode, opline->op2_type, opline->op2, xmm0 + if (opline->op1_type == opline->op2_type && opline->op1.var == opline->op2.var) { + | sse_math2 opline->opcode, xmm0, xmm0 + } else { + | sse_math opline->opcode, opline->op2_type, opline->op2, xmm0 + } | sse_store opline->result, xmm0 |.else | fp_load opline->op1_type, opline->op1 From bdeb0e0bd76d16a0da1ca62634b5674ab3ad32f5 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 5 Sep 2016 12:03:46 +0300 Subject: [PATCH 106/569] Set IP value at start of "target" basic blocks (instead of setting it at branch itself) --- ext/opcache/jit/zend_jit.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index b7c5a44e7d373..7f4831b7930bd 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -747,6 +747,11 @@ static int zend_real_jit_func(zend_op_array *op_array, zend_script *script) } #endif zend_jit_label(&dasm_state, b); + if (ssa.cfg.blocks[b].flags & ZEND_BB_TARGET) { + if (!zend_jit_set_opline(&dasm_state, op_array->opcodes + ssa.cfg.blocks[b].start)) { + goto jit_failure; + } + } if (ssa.cfg.blocks[b].flags & ZEND_BB_LOOP_HEADER) { if (!zend_jit_check_timeout(&dasm_state)) { goto jit_failure; @@ -806,8 +811,7 @@ static int zend_real_jit_func(zend_op_array *op_array, zend_script *script) case ZEND_OP_DATA: break; case ZEND_JMP: - if (!zend_jit_set_opline(&dasm_state, OP_JMP_ADDR(opline, opline->op1)) || - !zend_jit_jmp(&dasm_state, ssa.cfg.blocks[b].successors[0])) { + if (!zend_jit_jmp(&dasm_state, ssa.cfg.blocks[b].successors[0])) { goto jit_failure; } break; From 5802f54efa4f1510383989466c075dbbc2fc0f3a Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 5 Sep 2016 12:04:44 +0300 Subject: [PATCH 107/569] Implemented JIT for comparison operators with IS_DOUBLE operands --- ext/opcache/jit/zend_jit.c | 8 +++ ext/opcache/jit/zend_jit_x86.dasc | 115 ++++++++++++++++++++++++++++++ 2 files changed, 123 insertions(+) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 7f4831b7930bd..751339e848ee6 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -779,6 +779,14 @@ static int zend_real_jit_func(zend_op_array *op_array, zend_script *script) goto jit_failure; } break; + case ZEND_IS_EQUAL: + case ZEND_IS_NOT_EQUAL: + case ZEND_IS_SMALLER: + case ZEND_IS_SMALLER_OR_EQUAL: + if (!zend_jit_cmp(&dasm_state, opline, b, &i, op_array, &ssa)) { + goto jit_failure; + } + break; #endif case ZEND_RECV_INIT: if (ssa.cfg.split_at_recv) { diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 7e440f3d56c25..ccd8f541f8ba7 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -494,6 +494,121 @@ static int zend_jit_math(dasm_State **Dst, zend_op *opline, zend_op_array *op_ar return 1; +fallback: + /* fallback to subroutine threading */ + return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); +} + +static int zend_jit_cmp(dasm_State **Dst, zend_op *opline, int b, int *opnum, zend_op_array *op_array, zend_ssa *ssa) +{ + uint32_t op1_info, op2_info; + unsigned int target_label; + + if (!ssa->ops || !ssa->var_info) { + goto fallback; + } + op1_info = OP1_INFO(); + op2_info = OP2_INFO(); + + if (((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) && + ((op2_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE)) { + |.if X64 // TODO: or SSE ??? + | sse_load opline->op1_type, opline->op1, xmm0 + | sse_op ucomisd, opline->op2_type, opline->op2, xmm0 + |.else + | fp_load opline->op2_type, opline->op2 + | fp_load opline->op1_type, opline->op1 + | fucomip st1 + | fstp st0 + |.endif + if ((opline+1)->opcode == ZEND_JMPZ && + (opline+1)->op1_type == IS_TMP_VAR && + (opline+1)->op1.var == opline->result.var) { + (*opnum)++; + target_label = ssa->cfg.blocks[b].successors[0]; + switch (opline->opcode) { + case ZEND_IS_EQUAL: + | jp >1 + | je => target_label + |1: + break; + case ZEND_IS_NOT_EQUAL: + | jp >1 + | jne => target_label + |1: + break; + case ZEND_IS_SMALLER: + | ja => target_label + break; + case ZEND_IS_SMALLER_OR_EQUAL: + | jae => target_label + break; + } + if (!(ssa->cfg.blocks[ssa->cfg.blocks[b].successors[1]].flags & ZEND_BB_TARGET)) { + | add IP, sizeof(zend_op) * 2 + } + } else if ((opline+1)->opcode == ZEND_JMPNZ && + (opline+1)->op1_type == IS_TMP_VAR && + (opline+1)->op1.var == opline->result.var) { + (*opnum)++; + target_label = ssa->cfg.blocks[b].successors[0]; + switch (opline->opcode) { + case ZEND_IS_EQUAL: + | jp >1 + | jne => target_label + |1: + break; + case ZEND_IS_NOT_EQUAL: + | jp >1 + | je => target_label + |1: + break; + case ZEND_IS_SMALLER: + | jna => target_label + break; + case ZEND_IS_SMALLER_OR_EQUAL: + | jnae => target_label + break; + } + if (!(ssa->cfg.blocks[ssa->cfg.blocks[b].successors[1]].flags & ZEND_BB_TARGET)) { + | add IP, sizeof(zend_op) * 2 + } + } else { + switch (opline->opcode) { + case ZEND_IS_EQUAL: + | jp >1 + | mov eax, IS_TRUE + | je >2 + |1: + | mov eax, IS_FALSE + |2: + break; + case ZEND_IS_NOT_EQUAL: + | jp >1 + | mov eax, IS_TRUE + | jne >2 + |1: + | mov eax, IS_FALSE + |2: + break; + case ZEND_IS_SMALLER: + | seta al + | add eax, 2 + break; + case ZEND_IS_SMALLER_OR_EQUAL: + | setae al + | add eax, 2 + break; + } + | mov dword [FP + opline->result.var + 8], eax + | next_opline + } + } else { + goto fallback; + } + + return 1; + fallback: /* fallback to subroutine threading */ return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); From 4da00449b9f4effbe86c863b487e800cacbe394e Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 5 Sep 2016 13:02:52 +0300 Subject: [PATCH 108/569] Implemented JIT for comparison operators with IS_LONG operands --- ext/opcache/jit/zend_jit_x86.dasc | 104 +++++++++++++++++++++++++++++- 1 file changed, 103 insertions(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index ccd8f541f8ba7..4c931ebf3ca79 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -398,6 +398,39 @@ static int zend_jit_new(dasm_State **Dst, zend_op *opline) | mov dword [FP + op.var + 8], IS_DOUBLE |.endmacro | +|.macro long_op, long_ins, op_type, op, reg +|| if (op_type == IS_CONST) { +| .if X64 +|| if (IS_32BIT(Z_LVAL_P(RT_CONSTANT(op_array, op)))) { +| mov64 r1, Z_LVAL_P(RT_CONSTANT(op_array, op)) +| long_ins reg, r1 +|| } else { +| long_ins reg, Z_LVAL_P(RT_CONSTANT(op_array, op)) +|| } +| .else +| long_ins reg, Z_LVAL_P(RT_CONSTANT(op_array, op)) +| .endif +|| } else { +| long_ins reg, aword [FP + op.var] +|| } +|.endmacro +| +|.macro long_load, op_type, op, reg +|| if (op_type == IS_CONST) { +| .if X64 +|| if (IS_32BIT(Z_LVAL_P(RT_CONSTANT(op_array, op)))) { +| mov64 reg, Z_LVAL_P(RT_CONSTANT(op_array, op)) +|| } else { +| mov reg, Z_LVAL_P(RT_CONSTANT(op_array, op)) +|| } +| .else +| mov reg, Z_LVAL_P(RT_CONSTANT(op_array, op)) +| .endif +|| } else { +| mov reg, aword [FP + op.var] +|| } +|.endmacro +| |.macro next_opline | add IP, sizeof(zend_op) |.endmacro @@ -510,7 +543,76 @@ static int zend_jit_cmp(dasm_State **Dst, zend_op *opline, int b, int *opnum, ze op1_info = OP1_INFO(); op2_info = OP2_INFO(); - if (((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) && + if (((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) && + ((op2_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG)) { + | long_load opline->op1_type, opline->op1, r0 + | long_op cmp, opline->op2_type, opline->op2, r0 + if ((opline+1)->opcode == ZEND_JMPZ && + (opline+1)->op1_type == IS_TMP_VAR && + (opline+1)->op1.var == opline->result.var) { + (*opnum)++; + target_label = ssa->cfg.blocks[b].successors[0]; + switch (opline->opcode) { + case ZEND_IS_EQUAL: + | jne => target_label + break; + case ZEND_IS_NOT_EQUAL: + | je => target_label + break; + case ZEND_IS_SMALLER: + | jg => target_label + break; + case ZEND_IS_SMALLER_OR_EQUAL: + | jge => target_label + break; + } + if (!(ssa->cfg.blocks[ssa->cfg.blocks[b].successors[1]].flags & ZEND_BB_TARGET)) { + | add IP, sizeof(zend_op) * 2 + } + } else if ((opline+1)->opcode == ZEND_JMPNZ && + (opline+1)->op1_type == IS_TMP_VAR && + (opline+1)->op1.var == opline->result.var) { + (*opnum)++; + target_label = ssa->cfg.blocks[b].successors[0]; + switch (opline->opcode) { + case ZEND_IS_EQUAL: + | je => target_label + break; + case ZEND_IS_NOT_EQUAL: + | jne => target_label + break; + case ZEND_IS_SMALLER: + | jl => target_label + break; + case ZEND_IS_SMALLER_OR_EQUAL: + | jle => target_label + break; + } + if (!(ssa->cfg.blocks[ssa->cfg.blocks[b].successors[1]].flags & ZEND_BB_TARGET)) { + | add IP, sizeof(zend_op) * 2 + } + } else { + switch (opline->opcode) { + case ZEND_IS_EQUAL: + | sete al + | add eax, 2 + case ZEND_IS_NOT_EQUAL: + | setne al + | add eax, 2 + break; + case ZEND_IS_SMALLER: + | setg al + | add eax, 2 + break; + case ZEND_IS_SMALLER_OR_EQUAL: + | setge al + | add eax, 2 + break; + } + | mov dword [FP + opline->result.var + 8], eax + | next_opline + } + } else if (((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) && ((op2_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE)) { |.if X64 // TODO: or SSE ??? | sse_load opline->op1_type, opline->op1, xmm0 From b9dff7761748b36750d37ca8ad0dffa3aa3bd38b Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Mon, 5 Sep 2016 18:16:17 +0800 Subject: [PATCH 109/569] Defines macros in uppercase --- ext/opcache/jit/zend_jit.c | 2 +- ext/opcache/jit/zend_jit_x86.dasc | 128 +++++++++++++++--------------- 2 files changed, 65 insertions(+), 65 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 751339e848ee6..b87a0bf8a1589 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -921,7 +921,7 @@ static int zend_real_jit_func(zend_op_array *op_array, zend_script *script) case ZEND_FE_FETCH_RW: case ZEND_DECLARE_ANON_CLASS: case ZEND_DECLARE_ANON_INHERITED_CLASS: - if (!zend_jit_handler(&dasm_state, opline, zend_may_throw(opline, op_array, &ssa)) || + if (!zend_jit_handler(&dasm_state, opline, zend_may_throw(opline, op_array, &ssa)) || !zend_jit_cond_jmp(&dasm_state, opline + 1, ssa.cfg.blocks[b].successors[0])) { goto jit_failure; } diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 4c931ebf3ca79..69d0522968a96 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -60,7 +60,7 @@ static void* dasm_labels[zend_lb_MAX]; |.section code -|.macro load_addr, reg, addr +|.macro LOAD_ADDR, reg, addr ||if (((ptrdiff_t)addr) <= 0x7fffffff) { | mov reg, ((ptrdiff_t)addr) // 0x48 0xc7 0xc0 || } else { @@ -84,7 +84,7 @@ static int zend_jit_interrupt_handler_stub(dasm_State **Dst) if (IS_32BIT(dasm_end) && IS_32BIT(zend_timeout)) { | call qword &zend_timeout } else { - | load_addr rax, zend_timeout + | LOAD_ADDR rax, zend_timeout | call rax } |.else @@ -102,7 +102,7 @@ static int zend_jit_interrupt_handler_stub(dasm_State **Dst) | mov CARG1, FP | call qword &zend_interrupt_function } else { - | load_addr rax, zend_interrupt_function + | LOAD_ADDR rax, zend_interrupt_function | mov CARG1, FP | call rax } @@ -132,7 +132,7 @@ static int zend_jit_exception_handler_stub(dasm_State **Dst) if (IS_32BIT(dasm_end) && IS_32BIT(EG(exception_op)->handler)) { | jmp qword &(EG(exception_op)->handler) } else { - | load_addr rax, EG(exception_op)->handler + | LOAD_ADDR rax, EG(exception_op)->handler | jmp rax } |.else @@ -187,7 +187,7 @@ static int zend_jit_handler(dasm_State **Dst, zend_op *opline, int may_throw) if (IS_32BIT(dasm_end) && IS_32BIT(handler)) { | call qword &handler } else { - | load_addr rax, handler + | LOAD_ADDR rax, handler | call rax } |.else @@ -208,7 +208,7 @@ static int zend_jit_tail_handler(dasm_State **Dst, zend_op *opline) if (IS_32BIT(dasm_end) && IS_32BIT(handler)) { | jmp qword &handler } else { - | load_addr rax, handler + | LOAD_ADDR rax, handler | jmp rax } |.else @@ -226,7 +226,7 @@ static int zend_jit_skip_handler(dasm_State **Dst, uint32_t skip) static int zend_jit_set_opline(dasm_State **Dst, zend_op *target_opline) { |.if X64 - | load_addr IP, target_opline + | LOAD_ADDR IP, target_opline |.else | mov IP, target_opline |.endif @@ -303,7 +303,7 @@ static int zend_jit_new(dasm_State **Dst, zend_op *opline) #if ZEND_JIT_LEVEL >= ZEND_JIT_LEVEL_FULL -|.macro fp_op, fp_ins, op_type, op +|.macro FP_OP, fp_ins, op_type, op || if (op_type == IS_CONST) { | .if X64 | mov r0, aword EX->literals @@ -315,34 +315,34 @@ static int zend_jit_new(dasm_State **Dst, zend_op *opline) | fp_ins qword [FP + op.var] || } |.endmacro -| -|.macro fp_load, op_type, op -| fp_op fld, op_type, op + +|.macro FP_LOAD, op_type, op +| FP_OP fld, op_type, op |.endmacro -| -|.macro fp_math, opcode, op_type, op + +|.macro FP_MATH, opcode, op_type, op || switch (opcode) { || case ZEND_ADD: -| fp_op fadd, op_type, op +| FP_OP fadd, op_type, op || break; || case ZEND_SUB: -| fp_op fsub, op_type, op +| FP_OP fsub, op_type, op || break; || case ZEND_MUL: -| fp_op fmul, op_type, op +| FP_OP fmul, op_type, op || break; || case ZEND_DIV: -| fp_op fdiv, op_type, op +| FP_OP fdiv, op_type, op || break; || } |.endmacro -| -|.macro fp_store, op + +|.macro FP_STORE, op | fstp qword [FP + op.var] | mov dword [FP + op.var + 8], IS_DOUBLE |.endmacro -| -|.macro sse_op, sse_ins, op_type, op, reg + +|.macro SSE_OP, sse_ins, op_type, op, reg || if (op_type == IS_CONST) { | .if X64 | mov r0, aword EX->literals @@ -354,29 +354,29 @@ static int zend_jit_new(dasm_State **Dst, zend_op *opline) | sse_ins reg, qword [FP + op.var] || } |.endmacro -| -|.macro sse_load, op_type, op, reg -| sse_op movsd, op_type, op, reg + +|.macro SSE_LOAD, op_type, op, reg +| SSE_OP movsd, op_type, op, reg |.endmacro -| -|.macro sse_math, opcode, op_type, op, reg + +|.macro SSE_MATH, opcode, op_type, op, reg || switch (opcode) { || case ZEND_ADD: -| sse_op addsd, op_type, op, reg +| SSE_OP addsd, op_type, op, reg || break; || case ZEND_SUB: -| sse_op subsd, op_type, op, reg +| SSE_OP subsd, op_type, op, reg || break; || case ZEND_MUL: -| sse_op mulsd, op_type, op, reg +| SSE_OP mulsd, op_type, op, reg || break; || case ZEND_DIV: -| sse_op divsd, op_type, op, reg +| SSE_OP divsd, op_type, op, reg || break; || } |.endmacro -| -|.macro sse_math2, opcode, reg1, reg2 + +|.macro SSE_MATH2, opcode, reg1, reg2 || switch (opcode) { || case ZEND_ADD: | addsd reg2, reg1 @@ -392,46 +392,46 @@ static int zend_jit_new(dasm_State **Dst, zend_op *opline) || break; || } |.endmacro -| -|.macro sse_store, op, reg + +|.macro SSE_STORE, op, reg | movsd qword [FP + op.var], reg | mov dword [FP + op.var + 8], IS_DOUBLE |.endmacro -| -|.macro long_op, long_ins, op_type, op, reg + +|.macro LONG_OP, long_ins, op_type, op, reg || if (op_type == IS_CONST) { | .if X64 || if (IS_32BIT(Z_LVAL_P(RT_CONSTANT(op_array, op)))) { -| mov64 r1, Z_LVAL_P(RT_CONSTANT(op_array, op)) -| long_ins reg, r1 +| mov64 r1, Z_LVAL_P(RT_CONSTANT(op_array, op)) +| long_ins reg, r1 || } else { -| long_ins reg, Z_LVAL_P(RT_CONSTANT(op_array, op)) +| long_ins reg, Z_LVAL_P(RT_CONSTANT(op_array, op)) || } | .else -| long_ins reg, Z_LVAL_P(RT_CONSTANT(op_array, op)) +| long_ins reg, Z_LVAL_P(RT_CONSTANT(op_array, op)) | .endif || } else { | long_ins reg, aword [FP + op.var] || } |.endmacro -| -|.macro long_load, op_type, op, reg + +|.macro LONG_LOAD, op_type, op, reg || if (op_type == IS_CONST) { | .if X64 || if (IS_32BIT(Z_LVAL_P(RT_CONSTANT(op_array, op)))) { -| mov64 reg, Z_LVAL_P(RT_CONSTANT(op_array, op)) +| mov64 reg, Z_LVAL_P(RT_CONSTANT(op_array, op)) || } else { -| mov reg, Z_LVAL_P(RT_CONSTANT(op_array, op)) +| mov reg, Z_LVAL_P(RT_CONSTANT(op_array, op)) || } | .else -| mov reg, Z_LVAL_P(RT_CONSTANT(op_array, op)) +| mov reg, Z_LVAL_P(RT_CONSTANT(op_array, op)) | .endif || } else { | mov reg, aword [FP + op.var] || } |.endmacro -| -|.macro next_opline + +|.macro INC_OPLINE | add IP, sizeof(zend_op) |.endmacro @@ -486,7 +486,7 @@ static int zend_jit_inc_dec(dasm_State **Dst, zend_op *opline, zend_op_array *op | } |1: - | add IP, sizeof(zend_op) + | INC_OPLINE return 1; } @@ -508,19 +508,19 @@ static int zend_jit_math(dasm_State **Dst, zend_op *opline, zend_op_array *op_ar if (((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) && ((op2_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE)) { |.if X64 // TODO: or SSE ??? - | sse_load opline->op1_type, opline->op1, xmm0 + | SSE_LOAD opline->op1_type, opline->op1, xmm0 if (opline->op1_type == opline->op2_type && opline->op1.var == opline->op2.var) { - | sse_math2 opline->opcode, xmm0, xmm0 + | SSE_MATH2 opline->opcode, xmm0, xmm0 } else { - | sse_math opline->opcode, opline->op2_type, opline->op2, xmm0 + | SSE_MATH opline->opcode, opline->op2_type, opline->op2, xmm0 } - | sse_store opline->result, xmm0 + | SSE_STORE opline->result, xmm0 |.else - | fp_load opline->op1_type, opline->op1 - | fp_math opline->opcode, opline->op2_type, opline->op2 - | fp_store opline->result + | FP_LOAD opline->op1_type, opline->op1 + | FP_MATH opline->opcode, opline->op2_type, opline->op2 + | FP_STORE opline->result |.endif - | next_opline + | INC_OPLINE } else { goto fallback; } @@ -545,8 +545,8 @@ static int zend_jit_cmp(dasm_State **Dst, zend_op *opline, int b, int *opnum, ze if (((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) && ((op2_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG)) { - | long_load opline->op1_type, opline->op1, r0 - | long_op cmp, opline->op2_type, opline->op2, r0 + | LONG_LOAD opline->op1_type, opline->op1, r0 + | LONG_OP cmp, opline->op2_type, opline->op2, r0 if ((opline+1)->opcode == ZEND_JMPZ && (opline+1)->op1_type == IS_TMP_VAR && (opline+1)->op1.var == opline->result.var) { @@ -610,16 +610,16 @@ static int zend_jit_cmp(dasm_State **Dst, zend_op *opline, int b, int *opnum, ze break; } | mov dword [FP + opline->result.var + 8], eax - | next_opline + | INC_OPLINE } } else if (((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) && ((op2_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE)) { |.if X64 // TODO: or SSE ??? - | sse_load opline->op1_type, opline->op1, xmm0 - | sse_op ucomisd, opline->op2_type, opline->op2, xmm0 + | SSE_LOAD opline->op1_type, opline->op1, xmm0 + | SSE_OP ucomisd, opline->op2_type, opline->op2, xmm0 |.else - | fp_load opline->op2_type, opline->op2 - | fp_load opline->op1_type, opline->op1 + | FP_LOAD opline->op2_type, opline->op2 + | FP_LOAD opline->op1_type, opline->op1 | fucomip st1 | fstp st0 |.endif @@ -703,7 +703,7 @@ static int zend_jit_cmp(dasm_State **Dst, zend_op *opline, int b, int *opnum, ze break; } | mov dword [FP + opline->result.var + 8], eax - | next_opline + | INC_OPLINE } } else { goto fallback; From 53d311fedbcdc9900de288686955340ef9179127 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Tue, 6 Sep 2016 11:29:54 +0800 Subject: [PATCH 110/569] Implemented long math without overflow --- ext/opcache/jit/zend_jit_x86.dasc | 37 ++++++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 69d0522968a96..8761f5e52a253 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -431,6 +431,28 @@ static int zend_jit_new(dasm_State **Dst, zend_op *opline) || } |.endmacro +|.macro LONG_MATH, opcode, op_type, op, reg +|| switch (opcode) { +|| case ZEND_ADD: +| LONG_OP add, op_type, op, reg +|| break; +|| case ZEND_SUB: +| LONG_OP sub, op_type, op, reg +|| break; +|| case ZEND_MUL: +| LONG_OP imul, op_type, op, reg +|| break; +|| case ZEND_DIV: +| idiv aword [FP + op.var] +|| break; +|| } +|.endmacro + +|.macro LONG_STORE, op, reg +| mov aword [FP + op.var], reg +| mov dword [FP + op.var + 8], IS_LONG +|.endmacro + |.macro INC_OPLINE | add IP, sizeof(zend_op) |.endmacro @@ -497,15 +519,24 @@ fallback: static int zend_jit_math(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) { - uint32_t op1_info, op2_info; + uint32_t op1_info, op2_info, res_info; if (!ssa->ops || !ssa->var_info) { goto fallback; } + op1_info = OP1_INFO(); op2_info = OP2_INFO(); + res_info = RES_INFO(); - if (((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) && + if (((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) && + ((op2_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) && + ((res_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG)) { + //TODO? overflow? + | LONG_LOAD opline->op1_type, opline->op1, r0 + | LONG_MATH opline->opcode, opline->op2_type, opline->op2, r0 + | LONG_STORE opline->result, r0 + } else if (((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) && ((op2_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE)) { |.if X64 // TODO: or SSE ??? | SSE_LOAD opline->op1_type, opline->op1, xmm0 @@ -520,10 +551,10 @@ static int zend_jit_math(dasm_State **Dst, zend_op *opline, zend_op_array *op_ar | FP_MATH opline->opcode, opline->op2_type, opline->op2 | FP_STORE opline->result |.endif - | INC_OPLINE } else { goto fallback; } + | INC_OPLINE return 1; From 67bd0e1920b775dc147babf53c059d0958e6549f Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Tue, 6 Sep 2016 14:20:08 +0800 Subject: [PATCH 111/569] Implemented jit for ZEND_QM_ASSIGN (which is used wildly in WP) --- ext/opcache/jit/zend_jit_x86.dasc | 54 +++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 8761f5e52a253..75b0326957c09 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -742,6 +742,60 @@ static int zend_jit_cmp(dasm_State **Dst, zend_op *opline, int b, int *opnum, ze return 1; +fallback: + /* fallback to subroutine threading */ + return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); +} + +static int zend_jit_qm_assign(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) +{ + uint32_t op1_info; + + if (!ssa->ops || !ssa->var_info) { + goto fallback; + } + + op1_info = OP1_INFO(); + + if (!(op1_info & + ((MAY_BE_ANY|MAY_BE_UNDEF) - + (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE)))) { + if (opline->op1_type == IS_CONST) { + zval *zv = RT_CONSTANT(op_array, opline->op1); + if (Z_TYPE_P(zv) > IS_TRUE) { + |.if X64 + if (IS_32BIT((ptrdiff_t)Z_PTR_P(zv))) { + | mov qword [FP + opline->result.var], (ptrdiff_t)Z_PTR_P(zv) + } else { + | mov64 r0, (ptrdiff_t)Z_PTR_P(zv) + | mov qword [FP + opline->result.var], r0 + } + |.else + | mov dword [FP + opline->result.var], zv->value.ww.w1 + | mov dword [FP + opline->result.var + 4], zv->value.ww.w2 + |.endif + } + | mov dword [FP + opline->result.var + 8], Z_TYPE_INFO_P(zv) + } else { + if (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) { + |.if X64 + | movdqa xmm0, [FP + opline->op1.var] + | movdqa [FP + opline->result.var], xmm0 + |.else + | movsd xmm0, qword [FP + opline->op1.var] + | movsd qword [FP + opline->result.var], xmm0 + | mov eax, [FP + opline->op1.var + 8] + | mov [FP + opline->result.var + 8], eax + |.endif + } else { + | mov eax, [FP + opline->op1.var + 8] + | mov [FP + opline->result.var + 8], eax + } + } + | INC_OPLINE + return 1; + } + fallback: /* fallback to subroutine threading */ return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); From 2619c5954ffa80ee95c034e00064dc4633cbe212 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Tue, 6 Sep 2016 14:23:24 +0800 Subject: [PATCH 112/569] Fixed condition --- ext/opcache/jit/zend_jit_x86.dasc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 75b0326957c09..19fbebbee3019 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -401,7 +401,7 @@ static int zend_jit_new(dasm_State **Dst, zend_op *opline) |.macro LONG_OP, long_ins, op_type, op, reg || if (op_type == IS_CONST) { | .if X64 -|| if (IS_32BIT(Z_LVAL_P(RT_CONSTANT(op_array, op)))) { +|| if (!IS_32BIT(Z_LVAL_P(RT_CONSTANT(op_array, op)))) { | mov64 r1, Z_LVAL_P(RT_CONSTANT(op_array, op)) | long_ins reg, r1 || } else { @@ -418,7 +418,7 @@ static int zend_jit_new(dasm_State **Dst, zend_op *opline) |.macro LONG_LOAD, op_type, op, reg || if (op_type == IS_CONST) { | .if X64 -|| if (IS_32BIT(Z_LVAL_P(RT_CONSTANT(op_array, op)))) { +|| if (!IS_32BIT(Z_LVAL_P(RT_CONSTANT(op_array, op)))) { | mov64 reg, Z_LVAL_P(RT_CONSTANT(op_array, op)) || } else { | mov reg, Z_LVAL_P(RT_CONSTANT(op_array, op)) From 9c747994b34aacdb1b07c733da67f6d6a7b94a83 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Tue, 6 Sep 2016 14:33:40 +0800 Subject: [PATCH 113/569] enable calls to zend_jit_qm_assign --- ext/opcache/jit/zend_jit.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index b87a0bf8a1589..469d83222d46b 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -779,6 +779,11 @@ static int zend_real_jit_func(zend_op_array *op_array, zend_script *script) goto jit_failure; } break; + case ZEND_QM_ASSIGN: + if (!zend_jit_qm_assign(&dasm_state, opline, op_array, &ssa)) { + goto jit_failure; + } + break; case ZEND_IS_EQUAL: case ZEND_IS_NOT_EQUAL: case ZEND_IS_SMALLER: From e3ca7fa250738518500f6379dfe94bc3bf3d795e Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 6 Sep 2016 11:53:48 +0300 Subject: [PATCH 114/569] Implemented JIT for simple ASSIGN and improved JIT for QM_ASSIGN. --- ext/opcache/jit/zend_jit.c | 5 + ext/opcache/jit/zend_jit_x86.dasc | 192 ++++++++++++++++++++++++------ 2 files changed, 162 insertions(+), 35 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 469d83222d46b..453e4be2923d7 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -779,6 +779,11 @@ static int zend_real_jit_func(zend_op_array *op_array, zend_script *script) goto jit_failure; } break; + case ZEND_ASSIGN: + if (!zend_jit_assign(&dasm_state, opline, op_array, &ssa)) { + goto jit_failure; + } + break; case ZEND_QM_ASSIGN: if (!zend_jit_qm_assign(&dasm_state, opline, op_array, &ssa)) { goto jit_failure; diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 19fbebbee3019..a5f213546a521 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -70,6 +70,48 @@ static void* dasm_labels[zend_lb_MAX]; #define IS_32BIT(addr) (((uintptr_t)(addr)) <= 0x7fffffff) +/* bit helpers */ + +/* from http://aggregate.org/MAGIC/ */ +static uint32_t ones32(uint32_t x) +{ + x -= ((x >> 1) & 0x55555555); + x = (((x >> 2) & 0x33333333) + (x & 0x33333333)); + x = (((x >> 4) + x) & 0x0f0f0f0f); + x += (x >> 8); + x += (x >> 16); + return x & 0x0000003f; +} + +static uint32_t floor_log2(uint32_t x) +{ + x |= (x >> 1); + x |= (x >> 2); + x |= (x >> 4); + x |= (x >> 8); + x |= (x >> 16); + return ones32(x) - 1; +} + +static zend_bool is_power_of_two(uint32_t x) +{ + return !(x & (x - 1)); +} + +static zend_bool has_concrete_type(uint32_t value_type) +{ + if (value_type & MAY_BE_UNDEF) { + value_type |= MAY_BE_NULL; + } + value_type &= MAY_BE_ANY; + return is_power_of_two (value_type); +} + +static uint32_t concrete_type(uint32_t value_type) +{ + return floor_log2(value_type & MAY_BE_ANY); +} + static int zend_jit_interrupt_handler_stub(dasm_State **Dst) { |->interrupt_handler: @@ -747,55 +789,135 @@ fallback: return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); } -static int zend_jit_qm_assign(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) +static int zend_jit_simple_assign(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, uint32_t var, zend_uchar val_type, znode_op val, uint32_t val_info) { - uint32_t op1_info; - - if (!ssa->ops || !ssa->var_info) { + if (val_type == IS_CV && (val_info & (MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_STRING|MAY_BE_RESOURCE|MAY_BE_ARRAY|MAY_BE_OBJECT))) { + /* TODO: allow JIT for Z_ADDREF_P() ??? */ goto fallback; - } - - op1_info = OP1_INFO(); + } else if (val_type == IS_VAR && (val_info & MAY_BE_REF)) { + goto fallback; + } else if (val_type == IS_CONST) { + zval *zv = RT_CONSTANT(op_array, val); - if (!(op1_info & - ((MAY_BE_ANY|MAY_BE_UNDEF) - - (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE)))) { - if (opline->op1_type == IS_CONST) { - zval *zv = RT_CONSTANT(op_array, opline->op1); - if (Z_TYPE_P(zv) > IS_TRUE) { - |.if X64 - if (IS_32BIT((ptrdiff_t)Z_PTR_P(zv))) { - | mov qword [FP + opline->result.var], (ptrdiff_t)Z_PTR_P(zv) - } else { - | mov64 r0, (ptrdiff_t)Z_PTR_P(zv) - | mov qword [FP + opline->result.var], r0 - } + if (Z_OPT_REFCOUNTED_P(zv)) { + /* TODO: allow JIT for Z_ADDREF_P() ??? */ + goto fallback; + } + if (Z_TYPE_P(zv) > IS_TRUE) { + if (Z_TYPE_P(zv) == IS_DOUBLE) { + |.if X64 // TODO: or SSE ??? + || if (Z_DVAL_P(zv) == 0.0) { + | xorps xmm0, xmm0 + || } else if (!IS_32BIT(zv)) { + | mov64 r0, ((uintptr_t)zv) + | movsd xmm0, qword [r0] + || } else { + | movsd xmm0, qword [((uint32_t)(uintptr_t)zv)] + || } + | movsd qword [FP + var], xmm0 |.else - | mov dword [FP + opline->result.var], zv->value.ww.w1 - | mov dword [FP + opline->result.var + 4], zv->value.ww.w2 + || if (Z_DVAL_P(zv) == 0.0) { + | fldz + || } else if (Z_DVAL_P(zv) == 1.0) { + | fld1 + || } else { + | fld qword [zv] + || } + | fstp qword [FP + var] |.endif - } - | mov dword [FP + opline->result.var + 8], Z_TYPE_INFO_P(zv) - } else { - if (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) { + } else { |.if X64 - | movdqa xmm0, [FP + opline->op1.var] - | movdqa [FP + opline->result.var], xmm0 + || if (!IS_32BIT(Z_LVAL_P(zv))) { + | mov64 r0, ((uintptr_t)Z_LVAL_P(zv)) + | mov qword [FP + var], r0 + || } else { + | mov qword [FP + var], Z_LVAL_P(zv) + || } |.else - | movsd xmm0, qword [FP + opline->op1.var] - | movsd qword [FP + opline->result.var], xmm0 - | mov eax, [FP + opline->op1.var + 8] - | mov [FP + opline->result.var + 8], eax + | mov dword [FP + var], Z_LVAL_P(zv) |.endif - } else { - | mov eax, [FP + opline->op1.var + 8] - | mov [FP + opline->result.var + 8], eax } } + | mov dword [FP + var + 8], Z_TYPE_INFO_P(zv); | INC_OPLINE return 1; } + if (val_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) { + if (!(val_info & MAY_BE_DOUBLE)) { + | mov r0, aword [FP + val.var] + | mov aword [FP + var], r0 + } else if ((val_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { + |.if X64 // TODO: or SSE ??? + | movsd xmm0, qword [FP + val.var] + | movsd qword [FP + var], xmm0 + |.else + | fld qword [FP + val.var] + | fstp qword [FP + var] + |.endif + } else { + |.if X64 + | mov r0, aword [FP + val.var] + | mov aword [FP + var], r0 + |.else + | mov r0, dword [FP + val.var] + | mov r1, dword [FP + val.var + 4] + | mov dword [FP + var], r0 + | mov dword [FP + var + 4], r1 + |.endif + } + } + + if ((val_info & (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE)) && + has_concrete_type(val_info & MAY_BE_ANY)) { + zend_uchar type = concrete_type(val_info); + | mov dword [FP + var + 8], type + } else { + | mov eax, dword [FP + val.var + 8] + | mov dword [FP + var + 8], eax + } + | INC_OPLINE + + return 1; + +fallback: + /* fallback to subroutine threading */ + return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); +} + +static int zend_jit_qm_assign(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) +{ + uint32_t op1_info; + + if (!ssa->ops || !ssa->var_info) { + goto fallback; + } + + op1_info = OP1_INFO(); + + return zend_jit_simple_assign(Dst, opline, op_array, ssa, opline->result.var, opline->op1_type, opline->op1, op1_info); + +fallback: + /* fallback to subroutine threading */ + return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); +} + +static int zend_jit_assign(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) +{ + uint32_t op1_info, op2_info; + + if (opline->op1_type != IS_CV || opline->result_type != IS_UNDEF || + !ssa->ops || !ssa->var_info) { + goto fallback; + } + + op1_info = OP1_INFO(); + op2_info = OP2_INFO(); + + if (!(op1_info & (MAY_BE_STRING|MAY_BE_RESOURCE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_REF))) { + return zend_jit_simple_assign(Dst, opline, op_array, ssa, opline->op1.var, opline->op2_type, opline->op2, op2_info); + } + fallback: /* fallback to subroutine threading */ return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); From 8c480e53a179b8ae3989dbc583d6588fe21b5806 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 6 Sep 2016 12:25:46 +0300 Subject: [PATCH 115/569] Use SSE instructions even in 32-bit mode --- ext/opcache/jit/zend_jit_x86.dasc | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index a5f213546a521..3753d0f52d935 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -41,6 +41,7 @@ |.define FCARG1, CARG1d // Simulate x86 fastcall. |.define FCARG2, CARG2d |.define SPAD, 8 + |.define SSE, 1 |.else |.define FP, esi |.define IP, edi @@ -48,6 +49,7 @@ |.define FCARG1, ecx // x86 fastcall arguments. |.define FCARG2, edx |.define SPAD, 12 + |.define SSE, 1 |.endif |.type EX, zend_execute_data, FP @@ -580,7 +582,7 @@ static int zend_jit_math(dasm_State **Dst, zend_op *opline, zend_op_array *op_ar | LONG_STORE opline->result, r0 } else if (((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) && ((op2_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE)) { - |.if X64 // TODO: or SSE ??? + |.if X64 or SSE | SSE_LOAD opline->op1_type, opline->op1, xmm0 if (opline->op1_type == opline->op2_type && opline->op1.var == opline->op2.var) { | SSE_MATH2 opline->opcode, xmm0, xmm0 @@ -687,7 +689,7 @@ static int zend_jit_cmp(dasm_State **Dst, zend_op *opline, int b, int *opnum, ze } } else if (((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) && ((op2_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE)) { - |.if X64 // TODO: or SSE ??? + |.if X64 or SSE | SSE_LOAD opline->op1_type, opline->op1, xmm0 | SSE_OP ucomisd, opline->op2_type, opline->op2, xmm0 |.else @@ -805,12 +807,14 @@ static int zend_jit_simple_assign(dasm_State **Dst, zend_op *opline, zend_op_arr } if (Z_TYPE_P(zv) > IS_TRUE) { if (Z_TYPE_P(zv) == IS_DOUBLE) { - |.if X64 // TODO: or SSE ??? + |.if X64 or SSE || if (Z_DVAL_P(zv) == 0.0) { | xorps xmm0, xmm0 + |.if X64 || } else if (!IS_32BIT(zv)) { | mov64 r0, ((uintptr_t)zv) | movsd xmm0, qword [r0] + |.endif || } else { | movsd xmm0, qword [((uint32_t)(uintptr_t)zv)] || } @@ -848,7 +852,7 @@ static int zend_jit_simple_assign(dasm_State **Dst, zend_op *opline, zend_op_arr | mov r0, aword [FP + val.var] | mov aword [FP + var], r0 } else if ((val_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { - |.if X64 // TODO: or SSE ??? + |.if X64 or SSE | movsd xmm0, qword [FP + val.var] | movsd qword [FP + var], xmm0 |.else From 4f466596865129ec4a3b045b908b14e4ae871171 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Tue, 6 Sep 2016 18:32:30 +0800 Subject: [PATCH 116/569] Implemented full jit support for long math --- ext/opcache/jit/zend_jit_x86.dasc | 43 +++++++++++++++++++++++++++---- 1 file changed, 38 insertions(+), 5 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 3753d0f52d935..f613ae58f1c51 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -492,6 +492,28 @@ static int zend_jit_new(dasm_State **Dst, zend_op *opline) || } |.endmacro +|.macro LONGF_LOAD, op_type, op +| FP_OP fild, op_type, op +|.endmacro + +|.macro LONGF_MATH, opcode +|| switch (opcode) { +|| case ZEND_ADD: +| fadd st0, st1 +|| break; +|| case ZEND_SUB: +| fsub st0, st1 +|| break; +|| case ZEND_MUL: +| fmul st0, st1 +|| break; +|| case ZEND_DIV: +| fdiv st0, st1 +|| break; +|| } +|.endmacro + + |.macro LONG_STORE, op, reg | mov aword [FP + op.var], reg | mov dword [FP + op.var + 8], IS_LONG @@ -571,15 +593,24 @@ static int zend_jit_math(dasm_State **Dst, zend_op *opline, zend_op_array *op_ar op1_info = OP1_INFO(); op2_info = OP2_INFO(); - res_info = RES_INFO(); if (((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) && - ((op2_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) && - ((res_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG)) { - //TODO? overflow? + ((op2_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG)) { + res_info = RES_INFO(); | LONG_LOAD opline->op1_type, opline->op1, r0 | LONG_MATH opline->opcode, opline->op2_type, opline->op2, r0 - | LONG_STORE opline->result, r0 + if ((res_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) { + | LONG_STORE opline->result, r0 + } else { + | jo >1 + | LONG_STORE opline->result, r0 + | jmp >2 + |1: + | LONGF_LOAD opline->op2_type, opline->op2 + | LONGF_LOAD opline->op1_type, opline->op1 + | LONGF_MATH opline->opcode + | FP_STORE opline->result + } } else if (((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) && ((op2_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE)) { |.if X64 or SSE @@ -598,6 +629,7 @@ static int zend_jit_math(dasm_State **Dst, zend_op *opline, zend_op_array *op_ar } else { goto fallback; } + |2: | INC_OPLINE return 1; @@ -926,6 +958,7 @@ fallback: /* fallback to subroutine threading */ return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); } + #endif /* From 423eba7e7b2e154bd293fbac8c74a10f604de40a Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Tue, 6 Sep 2016 20:06:05 +0800 Subject: [PATCH 117/569] Implemented ability to delayed generating --- ext/opcache/jit/zend_jit.c | 54 ++++++++++++++++++++++++++++++- ext/opcache/jit/zend_jit_x86.dasc | 36 ++++++++++++++++----- 2 files changed, 81 insertions(+), 9 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 453e4be2923d7..02dd2fdbbcbf5 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -60,7 +60,18 @@ static void *dasm_buf = NULL; static void *dasm_ptr = NULL; static void *dasm_end = NULL; +typedef struct _delayed_entry { + uint32_t from; + uint32_t to; + zend_op_array *op_array; + zend_op *opline; +} delayed_entry; + +static zend_array delayed_entries = {0}; +static uint32_t delayed_offset = 0; + static int zend_may_throw(zend_op *opline, zend_op_array *op_array, zend_ssa *ssa); +static void *zend_jit_delay(zend_op_array *op_array, zend_op *opline, uint32_t *from, uint32_t *to); #include "dynasm/dasm_x86.h" #include "jit/zend_jit_x86.c" @@ -82,6 +93,44 @@ static int zend_may_throw(zend_op *opline, zend_op_array *op_array, zend_ssa *ss #define DASM_ALIGNMENT 16 +static void free_delayed_entry(zval *el) { + efree((void*)Z_PTR_P(el)); +} + +static void zend_jit_delayed_entries_init(uint32_t offset) { + delayed_offset = offset; + if (delayed_entries.arData == NULL) { + zend_hash_init(&delayed_entries, 8, NULL, free_delayed_entry, 0); + } + return; +} + +static void *zend_jit_delay(zend_op_array *op_array, zend_op *opline, uint32_t *from, uint32_t *to) { + delayed_entry *entry = (delayed_entry*)emalloc(sizeof(delayed_entry)); + entry->op_array = op_array; + entry->opline = opline; + entry->from = delayed_offset++; + entry->to = delayed_offset++; + zend_hash_next_index_insert_ptr(&delayed_entries, (void*)entry); + *from = entry->from; + *to = entry->to; + ZEND_ASSERT(delayed_offset < op_array->last * 2); + return; +} + +static void *zend_jit_delayed_entries_shutdown(dasm_State **Dst) { + delayed_entry *entry; + + ZEND_HASH_FOREACH_PTR(&delayed_entries, entry) { + zend_jit_delayed_gen(Dst, entry->op_array, entry->opline, entry->from, entry->to); + } ZEND_HASH_FOREACH_END(); + + zend_hash_destroy(&delayed_entries); + delayed_entries.arData = NULL; + delayed_offset = 0; + return; +} + static zend_string *zend_jit_func_name(zend_op_array *op_array) { smart_str buf = {0}; @@ -701,7 +750,9 @@ static int zend_real_jit_func(zend_op_array *op_array, zend_script *script) dasm_setupglobal(&dasm_state, dasm_labels, zend_lb_MAX); dasm_setup(&dasm_state, dasm_actions); - dasm_growpc(&dasm_state, ssa.cfg.blocks_count * 2); + dasm_growpc(&dasm_state, op_array->last * 2); + + zend_jit_delayed_entries_init(op_array->last); zend_jit_align_func(&dasm_state); for (b = 0; b < ssa.cfg.blocks_count; b++) { @@ -943,6 +994,7 @@ static int zend_real_jit_func(zend_op_array *op_array, zend_script *script) } } } + zend_jit_delayed_entries_shutdown(&dasm_state); handler = dasm_link_and_encode(&dasm_state, op_array, &ssa.cfg, NULL); if (!handler) { diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index f613ae58f1c51..4d808fbfed6c0 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -596,20 +596,21 @@ static int zend_jit_math(dasm_State **Dst, zend_op *opline, zend_op_array *op_ar if (((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) && ((op2_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG)) { + //TODO: imul/idiv overflow? + if (opline->opcode == ZEND_MUL || opline->opcode == ZEND_DIV) { + goto fallback; + } res_info = RES_INFO(); | LONG_LOAD opline->op1_type, opline->op1, r0 | LONG_MATH opline->opcode, opline->op2_type, opline->op2, r0 if ((res_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) { | LONG_STORE opline->result, r0 } else { - | jo >1 + uint32_t from, to; + zend_jit_delay(op_array, opline, &from, &to); + | jo =>from | LONG_STORE opline->result, r0 - | jmp >2 - |1: - | LONGF_LOAD opline->op2_type, opline->op2 - | LONGF_LOAD opline->op1_type, opline->op1 - | LONGF_MATH opline->opcode - | FP_STORE opline->result + |=>to: } } else if (((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) && ((op2_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE)) { @@ -629,7 +630,6 @@ static int zend_jit_math(dasm_State **Dst, zend_op *opline, zend_op_array *op_ar } else { goto fallback; } - |2: | INC_OPLINE return 1; @@ -959,6 +959,26 @@ fallback: return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); } +static void zend_jit_delayed_gen(dasm_State **Dst, zend_op_array *op_array, zend_op *opline, uint32_t from, uint32_t to) { + switch (opline->opcode) { + case ZEND_ADD: + case ZEND_SUB: + case ZEND_MUL: + case ZEND_DIV: { + /* overflow slow path */ + |=>from: + | LONGF_LOAD opline->op2_type, opline->op2 + | LONGF_LOAD opline->op1_type, opline->op1 + | LONGF_MATH opline->opcode + | FP_STORE opline->result + | jmp =>to + } + break; + default: + break; + } +} + #endif /* From 2ac23ca1e7356ffa924111f27923f6469f659159 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Tue, 6 Sep 2016 20:16:19 +0800 Subject: [PATCH 118/569] Delay slow path in PRE/POST_INC/DEC --- ext/opcache/jit/zend_jit_x86.dasc | 56 +++++++++++++++++-------------- 1 file changed, 30 insertions(+), 26 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 4d808fbfed6c0..8b364a08b49ae 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -545,26 +545,10 @@ static int zend_jit_inc_dec(dasm_State **Dst, zend_op *opline, zend_op_array *op } op1_def_info = OP1_DEF_INFO(); if (op1_def_info & MAY_BE_DOUBLE) { - | jno >2 - if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { - |.if X64 - | mov64 rax, 0x43e0000000000000 - | mov qword [FP + opline->op1.var], rax - |.else - | mov dword [FP + opline->op1.var], 0 - | mov dword [FP + opline->op1.var + 4], 0x41e00000 - |.endif - } else { - |.if X64 - | mov64 rax, 0x43e0000000000000 - | mov qword [FP + opline->op1.var], rax - |.else - | mov dword [FP + opline->op1.var], 0x00200000 - | mov dword [FP + opline->op1.var + 4], 0xc1e00000 - |.endif - } - | mov dword [FP + opline->op1.var + 8], IS_DOUBLE - | jmp >1 + uint32_t from, to; + zend_jit_delay(op_array, opline, &from, &to); + | jo =>from + |=>to: } else if (op1_info & (MAY_BE_ANY-MAY_BE_LONG)) { | jmp >1 } @@ -960,23 +944,43 @@ fallback: } static void zend_jit_delayed_gen(dasm_State **Dst, zend_op_array *op_array, zend_op *opline, uint32_t from, uint32_t to) { + |=>from: switch (opline->opcode) { case ZEND_ADD: case ZEND_SUB: case ZEND_MUL: - case ZEND_DIV: { - /* overflow slow path */ - |=>from: + case ZEND_DIV: | LONGF_LOAD opline->op2_type, opline->op2 | LONGF_LOAD opline->op1_type, opline->op1 | LONGF_MATH opline->opcode | FP_STORE opline->result - | jmp =>to - } break; - default: + case ZEND_POST_INC: + case ZEND_PRE_INC: + |.if X64 + | mov64 rax, 0x43e0000000000000 + | mov qword [FP + opline->op1.var], rax + |.else + | mov dword [FP + opline->op1.var], 0 + | mov dword [FP + opline->op1.var + 4], 0x41e00000 + |.endif + | mov dword [FP + opline->op1.var + 8], IS_DOUBLE break; + case ZEND_POST_DEC: + case ZEND_PRE_DEC: + |.if X64 + | mov64 rax, 0x43e0000000000000 + | mov qword [FP + opline->op1.var], rax + |.else + | mov dword [FP + opline->op1.var], 0x00200000 + | mov dword [FP + opline->op1.var + 4], 0xc1e00000 + |.endif + | mov dword [FP + opline->op1.var + 8], IS_DOUBLE + break; + default: + abort(); } + | jmp =>to } #endif From 5bf42c8a796e75f4bb83b8404e60632bc17099fe Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Tue, 6 Sep 2016 20:23:16 +0800 Subject: [PATCH 119/569] Better name --- ext/opcache/jit/zend_jit.c | 18 +++++++++--------- ext/opcache/jit/zend_jit_x86.dasc | 22 +++++++++++----------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 02dd2fdbbcbf5..20e8d7d805837 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -61,8 +61,8 @@ static void *dasm_ptr = NULL; static void *dasm_end = NULL; typedef struct _delayed_entry { - uint32_t from; - uint32_t to; + uint32_t in; + uint32_t out; zend_op_array *op_array; zend_op *opline; } delayed_entry; @@ -71,7 +71,7 @@ static zend_array delayed_entries = {0}; static uint32_t delayed_offset = 0; static int zend_may_throw(zend_op *opline, zend_op_array *op_array, zend_ssa *ssa); -static void *zend_jit_delay(zend_op_array *op_array, zend_op *opline, uint32_t *from, uint32_t *to); +static void *zend_jit_delay(zend_op_array *op_array, zend_op *opline, uint32_t *in, uint32_t *out); #include "dynasm/dasm_x86.h" #include "jit/zend_jit_x86.c" @@ -105,15 +105,15 @@ static void zend_jit_delayed_entries_init(uint32_t offset) { return; } -static void *zend_jit_delay(zend_op_array *op_array, zend_op *opline, uint32_t *from, uint32_t *to) { +static void *zend_jit_delay(zend_op_array *op_array, zend_op *opline, uint32_t *in, uint32_t *out) { delayed_entry *entry = (delayed_entry*)emalloc(sizeof(delayed_entry)); entry->op_array = op_array; entry->opline = opline; - entry->from = delayed_offset++; - entry->to = delayed_offset++; + entry->in = delayed_offset++; + entry->out = delayed_offset++; zend_hash_next_index_insert_ptr(&delayed_entries, (void*)entry); - *from = entry->from; - *to = entry->to; + *in = entry->in; + *out = entry->out; ZEND_ASSERT(delayed_offset < op_array->last * 2); return; } @@ -122,7 +122,7 @@ static void *zend_jit_delayed_entries_shutdown(dasm_State **Dst) { delayed_entry *entry; ZEND_HASH_FOREACH_PTR(&delayed_entries, entry) { - zend_jit_delayed_gen(Dst, entry->op_array, entry->opline, entry->from, entry->to); + zend_jit_delayed_gen(Dst, entry->op_array, entry->opline, entry->in, entry->out); } ZEND_HASH_FOREACH_END(); zend_hash_destroy(&delayed_entries); diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 8b364a08b49ae..1c9ceb18d7d74 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -545,10 +545,10 @@ static int zend_jit_inc_dec(dasm_State **Dst, zend_op *opline, zend_op_array *op } op1_def_info = OP1_DEF_INFO(); if (op1_def_info & MAY_BE_DOUBLE) { - uint32_t from, to; - zend_jit_delay(op_array, opline, &from, &to); - | jo =>from - |=>to: + uint32_t in, out; + zend_jit_delay(op_array, opline, &in, &out); + | jo =>in + |=>out: } else if (op1_info & (MAY_BE_ANY-MAY_BE_LONG)) { | jmp >1 } @@ -590,11 +590,11 @@ static int zend_jit_math(dasm_State **Dst, zend_op *opline, zend_op_array *op_ar if ((res_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) { | LONG_STORE opline->result, r0 } else { - uint32_t from, to; - zend_jit_delay(op_array, opline, &from, &to); - | jo =>from + uint32_t in, out; + zend_jit_delay(op_array, opline, &in, &out); + | jo =>in | LONG_STORE opline->result, r0 - |=>to: + |=>out: } } else if (((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) && ((op2_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE)) { @@ -943,8 +943,8 @@ fallback: return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); } -static void zend_jit_delayed_gen(dasm_State **Dst, zend_op_array *op_array, zend_op *opline, uint32_t from, uint32_t to) { - |=>from: +static void zend_jit_delayed_gen(dasm_State **Dst, zend_op_array *op_array, zend_op *opline, uint32_t in, uint32_t out) { + |=>in: switch (opline->opcode) { case ZEND_ADD: case ZEND_SUB: @@ -980,7 +980,7 @@ static void zend_jit_delayed_gen(dasm_State **Dst, zend_op_array *op_array, zend default: abort(); } - | jmp =>to + | jmp =>out } #endif From 7a1076bda5952e9ebeae78293dc2ad428f7ed49d Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Tue, 6 Sep 2016 23:47:56 +0800 Subject: [PATCH 120/569] Implement ZEND_SEND_VAL(incomplete, need tested in 32bits box) --- ext/opcache/jit/zend_jit.c | 28 +++--- ext/opcache/jit/zend_jit_x86.dasc | 156 ++++++++++++++++++++++-------- 2 files changed, 130 insertions(+), 54 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 20e8d7d805837..2a4b1b5bd4b4b 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -71,7 +71,7 @@ static zend_array delayed_entries = {0}; static uint32_t delayed_offset = 0; static int zend_may_throw(zend_op *opline, zend_op_array *op_array, zend_ssa *ssa); -static void *zend_jit_delay(zend_op_array *op_array, zend_op *opline, uint32_t *in, uint32_t *out); +static int zend_jit_delay(zend_op_array *op_array, zend_op *opline, uint32_t *in, uint32_t *out); #include "dynasm/dasm_x86.h" #include "jit/zend_jit_x86.c" @@ -105,17 +105,19 @@ static void zend_jit_delayed_entries_init(uint32_t offset) { return; } -static void *zend_jit_delay(zend_op_array *op_array, zend_op *opline, uint32_t *in, uint32_t *out) { - delayed_entry *entry = (delayed_entry*)emalloc(sizeof(delayed_entry)); - entry->op_array = op_array; - entry->opline = opline; - entry->in = delayed_offset++; - entry->out = delayed_offset++; - zend_hash_next_index_insert_ptr(&delayed_entries, (void*)entry); - *in = entry->in; - *out = entry->out; - ZEND_ASSERT(delayed_offset < op_array->last * 2); - return; +static int zend_jit_delay(zend_op_array *op_array, zend_op *opline, uint32_t *in, uint32_t *out) { + if (delayed_offset < op_array->last * 2) { + delayed_entry *entry = (delayed_entry*)emalloc(sizeof(delayed_entry)); + entry->op_array = op_array; + entry->opline = opline; + entry->in = delayed_offset++; + entry->out = delayed_offset++; + zend_hash_next_index_insert_ptr(&delayed_entries, (void*)entry); + *in = entry->in; + *out = entry->out; + return 1; + } + return 0; } static void *zend_jit_delayed_entries_shutdown(dasm_State **Dst) { @@ -752,7 +754,7 @@ static int zend_real_jit_func(zend_op_array *op_array, zend_script *script) dasm_growpc(&dasm_state, op_array->last * 2); - zend_jit_delayed_entries_init(op_array->last); + zend_jit_delayed_entries_init(op_array->last + 1); zend_jit_align_func(&dasm_state); for (b = 0; b < ssa.cfg.blocks_count; b++) { diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 1c9ceb18d7d74..3dc802c4bb681 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -523,6 +523,50 @@ static int zend_jit_new(dasm_State **Dst, zend_op *opline) | add IP, sizeof(zend_op) |.endmacro +static void zend_jit_delayed_gen(dasm_State **Dst, zend_op_array *op_array, zend_op *opline, uint32_t in, uint32_t out) { + if (in) { + |=>in: + } + switch (opline->opcode) { + case ZEND_ADD: + case ZEND_SUB: + case ZEND_MUL: + case ZEND_DIV: + | LONGF_LOAD opline->op2_type, opline->op2 + | LONGF_LOAD opline->op1_type, opline->op1 + | LONGF_MATH opline->opcode + | FP_STORE opline->result + break; + case ZEND_POST_INC: + case ZEND_PRE_INC: + |.if X64 + | mov64 rax, 0x43e0000000000000 + | mov qword [FP + opline->op1.var], rax + |.else + | mov dword [FP + opline->op1.var], 0 + | mov dword [FP + opline->op1.var + 4], 0x41e00000 + |.endif + | mov dword [FP + opline->op1.var + 8], IS_DOUBLE + break; + case ZEND_POST_DEC: + case ZEND_PRE_DEC: + |.if X64 + | mov64 rax, 0x43e0000000000000 + | mov qword [FP + opline->op1.var], rax + |.else + | mov dword [FP + opline->op1.var], 0x00200000 + | mov dword [FP + opline->op1.var + 4], 0xc1e00000 + |.endif + | mov dword [FP + opline->op1.var + 8], IS_DOUBLE + break; + default: + abort(); + } + if (out) { + | jmp =>out + } +} + static int zend_jit_inc_dec(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) { uint32_t op1_info, op1_def_info; @@ -546,9 +590,14 @@ static int zend_jit_inc_dec(dasm_State **Dst, zend_op *opline, zend_op_array *op op1_def_info = OP1_DEF_INFO(); if (op1_def_info & MAY_BE_DOUBLE) { uint32_t in, out; - zend_jit_delay(op_array, opline, &in, &out); - | jo =>in - |=>out: + if (zend_jit_delay(op_array, opline, &in, &out)) { + | jo =>in + |=>out: + } else { + | jno >1 + zend_jit_delayed_gen(Dst, op_array, opline, 0, 0); + |1: + } } else if (op1_info & (MAY_BE_ANY-MAY_BE_LONG)) { | jmp >1 } @@ -591,10 +640,16 @@ static int zend_jit_math(dasm_State **Dst, zend_op *opline, zend_op_array *op_ar | LONG_STORE opline->result, r0 } else { uint32_t in, out; - zend_jit_delay(op_array, opline, &in, &out); - | jo =>in - | LONG_STORE opline->result, r0 - |=>out: + if (zend_jit_delay(op_array, opline, &in, &out)) { + | LONG_STORE opline->result, r0 + | jo =>in + |=>out: + } else { + | LONG_STORE opline->result, r0 + | jno >1 + zend_jit_delayed_gen(Dst, op_array, opline, 0, 0); + |1: + } } } else if (((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) && ((op2_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE)) { @@ -943,44 +998,63 @@ fallback: return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); } -static void zend_jit_delayed_gen(dasm_State **Dst, zend_op_array *op_array, zend_op *opline, uint32_t in, uint32_t out) { - |=>in: - switch (opline->opcode) { - case ZEND_ADD: - case ZEND_SUB: - case ZEND_MUL: - case ZEND_DIV: - | LONGF_LOAD opline->op2_type, opline->op2 - | LONGF_LOAD opline->op1_type, opline->op1 - | LONGF_MATH opline->opcode - | FP_STORE opline->result - break; - case ZEND_POST_INC: - case ZEND_PRE_INC: - |.if X64 - | mov64 rax, 0x43e0000000000000 - | mov qword [FP + opline->op1.var], rax - |.else - | mov dword [FP + opline->op1.var], 0 - | mov dword [FP + opline->op1.var + 4], 0x41e00000 - |.endif - | mov dword [FP + opline->op1.var + 8], IS_DOUBLE - break; - case ZEND_POST_DEC: - case ZEND_PRE_DEC: +static int zend_jit_send_val(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) +{ + zval *zv; +#if 0 + if (opline->op1_type != IS_CONST) { + goto fallback; + } + + zv = RT_CONSTANT(op_array, opline->op1); + + if (Z_REFCOUNTED_P(zv)) { + goto fallback; + } + + |.if X64 + | mov r0, EX->call + |.endif + + switch (Z_TYPE_P(zv)) { + case IS_STRING: + case IS_ARRAY: + case IS_LONG: + case IS_DOUBLE: + if (Z_TYPE_P(zv) < IS_STRING) { + |.if X64 + | mov qword [r0 + opline->result.var], (uintptr_t)Z_PTR_P(zv) + |.else + | mov dword [EX->call + opline->result.var], zv->value.ww.w1 + | mov dword [EX->call + opline->result.var + 4], zv->value.ww.w2 + |.endif + } else { + |.if X64 + | mov64 r1, (uintptr_t)(zv) + | mov r1, [r1] + | mov qword [r0 + opline->result.var], r1 + |.else + | movsd qword xmm0, [zv] + | movsd qword [EX->call + opline->result.var + 4], xmm0 + |.endif + } + case IS_NULL: + case IS_FALSE: + case IS_TRUE: |.if X64 - | mov64 rax, 0x43e0000000000000 - | mov qword [FP + opline->op1.var], rax + | mov dword [r0 + opline->result.var + 8], Z_TYPE_INFO_P(zv) |.else - | mov dword [FP + opline->op1.var], 0x00200000 - | mov dword [FP + opline->op1.var + 4], 0xc1e00000 + | movsd qword [EX->call + opline->result.var + 8], Z_TYPE_INFO_P(zv) |.endif - | mov dword [FP + opline->op1.var + 8], IS_DOUBLE - break; - default: - abort(); + break; } - | jmp =>out + | INC_OPLINE + + return 1; +fallback: +#endif + /* fallback to subroutine threading */ + return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); } #endif From 60bd205371b2807497a2f69c29d80b2fdfada53e Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Wed, 7 Sep 2016 10:50:26 +0800 Subject: [PATCH 121/569] Enable jit for ZEND_SEND_VAL --- ext/opcache/jit/zend_jit.c | 5 +++++ ext/opcache/jit/zend_jit_x86.dasc | 16 ++++------------ 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 2a4b1b5bd4b4b..d7aeb5630bf9b 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -842,6 +842,11 @@ static int zend_real_jit_func(zend_op_array *op_array, zend_script *script) goto jit_failure; } break; + case ZEND_SEND_VAL: + if (!zend_jit_send_val(&dasm_state, opline, op_array, &ssa)) { + goto jit_failure; + } + break; case ZEND_IS_EQUAL: case ZEND_IS_NOT_EQUAL: case ZEND_IS_SMALLER: diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 3dc802c4bb681..cc332e63fcd13 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -1001,7 +1001,6 @@ fallback: static int zend_jit_send_val(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) { zval *zv; -#if 0 if (opline->op1_type != IS_CONST) { goto fallback; } @@ -1012,9 +1011,7 @@ static int zend_jit_send_val(dasm_State **Dst, zend_op *opline, zend_op_array *o goto fallback; } - |.if X64 | mov r0, EX->call - |.endif switch (Z_TYPE_P(zv)) { case IS_STRING: @@ -1025,8 +1022,8 @@ static int zend_jit_send_val(dasm_State **Dst, zend_op *opline, zend_op_array *o |.if X64 | mov qword [r0 + opline->result.var], (uintptr_t)Z_PTR_P(zv) |.else - | mov dword [EX->call + opline->result.var], zv->value.ww.w1 - | mov dword [EX->call + opline->result.var + 4], zv->value.ww.w2 + | mov dword [r0 + opline->result.var], (uint32_t)zv->value.ww.w1 + | mov dword [r0 + opline->result.var + 4], (uint32_t)zv->value.ww.w2 |.endif } else { |.if X64 @@ -1034,25 +1031,20 @@ static int zend_jit_send_val(dasm_State **Dst, zend_op *opline, zend_op_array *o | mov r1, [r1] | mov qword [r0 + opline->result.var], r1 |.else - | movsd qword xmm0, [zv] - | movsd qword [EX->call + opline->result.var + 4], xmm0 + | movsd xmm0, qword [zv] + | movsd qword [r0 + opline->result.var], xmm0 |.endif } case IS_NULL: case IS_FALSE: case IS_TRUE: - |.if X64 | mov dword [r0 + opline->result.var + 8], Z_TYPE_INFO_P(zv) - |.else - | movsd qword [EX->call + opline->result.var + 8], Z_TYPE_INFO_P(zv) - |.endif break; } | INC_OPLINE return 1; fallback: -#endif /* fallback to subroutine threading */ return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); } From 1875b5ff31b3eb25cb68b46ea7968a105c2a3cef Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Wed, 7 Sep 2016 11:06:31 +0800 Subject: [PATCH 122/569] Improvement zend_jit_simple_assign for IS_CONST type 1. if const zv is value type(not pointer type), then do memory copy to save one mem access 2. only set type if it's necessary --- ext/opcache/jit/zend_jit_x86.dasc | 113 +++++++++++++++--------------- 1 file changed, 56 insertions(+), 57 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index cc332e63fcd13..f4d8e5a71602a 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -862,7 +862,7 @@ fallback: return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); } -static int zend_jit_simple_assign(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, uint32_t var, zend_uchar val_type, znode_op val, uint32_t val_info) +static int zend_jit_simple_assign(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, uint32_t var, uint32_t var_info, zend_uchar val_type, znode_op val, uint32_t val_info) { if (val_type == IS_CV && (val_info & (MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_STRING|MAY_BE_RESOURCE|MAY_BE_ARRAY|MAY_BE_OBJECT))) { /* TODO: allow JIT for Z_ADDREF_P() ??? */ @@ -877,43 +877,40 @@ static int zend_jit_simple_assign(dasm_State **Dst, zend_op *opline, zend_op_arr goto fallback; } if (Z_TYPE_P(zv) > IS_TRUE) { - if (Z_TYPE_P(zv) == IS_DOUBLE) { - |.if X64 or SSE - || if (Z_DVAL_P(zv) == 0.0) { - | xorps xmm0, xmm0 + if (Z_TYPE_P(zv) < IS_STRING) { + /* Simply copy value */ |.if X64 - || } else if (!IS_32BIT(zv)) { - | mov64 r0, ((uintptr_t)zv) - | movsd xmm0, qword [r0] - |.endif - || } else { - | movsd xmm0, qword [((uint32_t)(uintptr_t)zv)] - || } - | movsd qword [FP + var], xmm0 + if (IS_32BIT(Z_PTR_P(zv))) { + | mov qword [FP + var], (uintptr_t)Z_PTR_P(zv) + } else { + | mov64 r0, (uintptr_t)Z_PTR_P(zv) + | mov qword [FP + var], r0 + } |.else - || if (Z_DVAL_P(zv) == 0.0) { - | fldz - || } else if (Z_DVAL_P(zv) == 1.0) { - | fld1 - || } else { - | fld qword [zv] - || } - | fstp qword [FP + var] + | mov dword [FP + var], zv->value.ww.w1 + if (Z_TYPE_P(zv) == IS_DOUBLE) { + | mov dword [FP + var + 4], zv->value.ww.w2 + } |.endif } else { + /* Z_PTR_P(zv) could be changed, Read from memory */ |.if X64 - || if (!IS_32BIT(Z_LVAL_P(zv))) { - | mov64 r0, ((uintptr_t)Z_LVAL_P(zv)) - | mov qword [FP + var], r0 - || } else { - | mov qword [FP + var], Z_LVAL_P(zv) - || } + if (!IS_32BIT(zv)) { + | mov64 r1, (uintptr_t)(zv) + | mov r1, [r1] + } else { + | mov r1, qword [(uintptr_t)(zv)] + } + | mov qword [FP + var], r1 |.else - | mov dword [FP + var], Z_LVAL_P(zv) + | movsd xmm0, qword [zv] + | movsd qword [FP + var], xmm0 |.endif } } - | mov dword [FP + var + 8], Z_TYPE_INFO_P(zv); + if ((var_info & MAY_BE_ANY) != (1<result.var, opline->op1_type, opline->op1, op1_info); + return zend_jit_simple_assign(Dst, opline, op_array, ssa, opline->result.var, -1, opline->op1_type, opline->op1, op1_info); fallback: /* fallback to subroutine threading */ @@ -990,7 +987,7 @@ static int zend_jit_assign(dasm_State **Dst, zend_op *opline, zend_op_array *op_ op2_info = OP2_INFO(); if (!(op1_info & (MAY_BE_STRING|MAY_BE_RESOURCE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_REF))) { - return zend_jit_simple_assign(Dst, opline, op_array, ssa, opline->op1.var, opline->op2_type, opline->op2, op2_info); + return zend_jit_simple_assign(Dst, opline, op_array, ssa, opline->op1.var, op1_info, opline->op2_type, opline->op2, op2_info); } fallback: @@ -1012,35 +1009,37 @@ static int zend_jit_send_val(dasm_State **Dst, zend_op *opline, zend_op_array *o } | mov r0, EX->call - - switch (Z_TYPE_P(zv)) { - case IS_STRING: - case IS_ARRAY: - case IS_LONG: - case IS_DOUBLE: - if (Z_TYPE_P(zv) < IS_STRING) { - |.if X64 - | mov qword [r0 + opline->result.var], (uintptr_t)Z_PTR_P(zv) - |.else - | mov dword [r0 + opline->result.var], (uint32_t)zv->value.ww.w1 - | mov dword [r0 + opline->result.var + 4], (uint32_t)zv->value.ww.w2 - |.endif + if (Z_TYPE_P(zv) > IS_TRUE) { + if (Z_TYPE_P(zv) < IS_STRING) { + |.if X64 + if (IS_32BIT(Z_PTR_P(zv))) { + | mov qword [r0 + opline->result.var], (uintptr_t)Z_PTR_P(zv) } else { - |.if X64 - | mov64 r1, (uintptr_t)(zv) - | mov r1, [r1] - | mov qword [r0 + opline->result.var], r1 - |.else - | movsd xmm0, qword [zv] - | movsd qword [r0 + opline->result.var], xmm0 - |.endif + | mov64 r1, (uintptr_t)Z_PTR_P(zv) + | mov qword [r0 + opline->result.var], r1 } - case IS_NULL: - case IS_FALSE: - case IS_TRUE: - | mov dword [r0 + opline->result.var + 8], Z_TYPE_INFO_P(zv) - break; + |.else + | mov dword [r0 + opline->result.var], zv->value.ww.w1 + if (Z_TYPE_P(zv) == IS_DOUBLE) { + | mov dword [r0 + opline->result.var + 4], zv->value.ww.w2 + } + |.endif + } else { + |.if X64 + if (IS_32BIT(zv)) { + | mov r1, [(uintptr_t)(zv)] + } else { + | mov64 r1, ((uintptr_t)(zv)) + | mov r1, [r1] + } + | mov qword [r0 + opline->result.var], r1 + |.else + | movsd xmm0, qword [(uintptr_t)(zv)] + | movsd qword [r0 + opline->result.var], xmm0 + |.endif + } } + | mov dword [r0 + opline->result.var + 8], Z_TYPE_INFO_P(zv) | INC_OPLINE return 1; From be21f615eff773f18ca3c506957aee9974b4d355 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Wed, 7 Sep 2016 13:16:38 +0800 Subject: [PATCH 123/569] Cleanup --- ext/opcache/jit/zend_jit_x86.dasc | 108 +++++++++++------------------- 1 file changed, 40 insertions(+), 68 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index f4d8e5a71602a..42675315c8842 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -513,12 +513,49 @@ static int zend_jit_new(dasm_State **Dst, zend_op *opline) || } |.endmacro - |.macro LONG_STORE, op, reg | mov aword [FP + op.var], reg | mov dword [FP + op.var + 8], IS_LONG |.endmacro +|.macro STORE_CTZV, base, reg, var, var_info, zv +||if (Z_TYPE_P(zv) > IS_TRUE) { +|| if (Z_TYPE_P(zv) < IS_STRING) { +|| /* Simply copy value */ +| .if X64 +|| if (IS_32BIT(Z_PTR_P(zv))) { +| mov qword [base + var], (uintptr_t)Z_PTR_P(zv) +|| } else { +| mov64 reg, (uintptr_t)Z_PTR_P(zv) +| mov qword [base + var], reg +|| } +| .else +| mov dword [base + var], zv->value.ww.w1 +|| if (Z_TYPE_P(zv) == IS_DOUBLE) { +| mov dword [base + var + 4], zv->value.ww.w2 +|| } +| .endif +|| } else { +|| /* Z_PTR_P(zv) could be changed, Read from memory */ +| .if X64 +|| if (IS_32BIT(zv)) { +| mov reg, [(uintptr_t)(zv)] +|| } else { +| mov64 reg, ((uintptr_t)(zv)) +| mov reg, [reg] +|| } +| mov qword [base + var], reg +| .else +| movsd xmm0, qword [(uintptr_t)(zv)] +| movsd qword [base + var], xmm0 +| .endif +|| } +||} +||if ((var_info & MAY_BE_ANY) != (1< IS_TRUE) { - if (Z_TYPE_P(zv) < IS_STRING) { - /* Simply copy value */ - |.if X64 - if (IS_32BIT(Z_PTR_P(zv))) { - | mov qword [FP + var], (uintptr_t)Z_PTR_P(zv) - } else { - | mov64 r0, (uintptr_t)Z_PTR_P(zv) - | mov qword [FP + var], r0 - } - |.else - | mov dword [FP + var], zv->value.ww.w1 - if (Z_TYPE_P(zv) == IS_DOUBLE) { - | mov dword [FP + var + 4], zv->value.ww.w2 - } - |.endif - } else { - /* Z_PTR_P(zv) could be changed, Read from memory */ - |.if X64 - if (!IS_32BIT(zv)) { - | mov64 r1, (uintptr_t)(zv) - | mov r1, [r1] - } else { - | mov r1, qword [(uintptr_t)(zv)] - } - | mov qword [FP + var], r1 - |.else - | movsd xmm0, qword [zv] - | movsd qword [FP + var], xmm0 - |.endif - } - } - if ((var_info & MAY_BE_ANY) != (1<call - if (Z_TYPE_P(zv) > IS_TRUE) { - if (Z_TYPE_P(zv) < IS_STRING) { - |.if X64 - if (IS_32BIT(Z_PTR_P(zv))) { - | mov qword [r0 + opline->result.var], (uintptr_t)Z_PTR_P(zv) - } else { - | mov64 r1, (uintptr_t)Z_PTR_P(zv) - | mov qword [r0 + opline->result.var], r1 - } - |.else - | mov dword [r0 + opline->result.var], zv->value.ww.w1 - if (Z_TYPE_P(zv) == IS_DOUBLE) { - | mov dword [r0 + opline->result.var + 4], zv->value.ww.w2 - } - |.endif - } else { - |.if X64 - if (IS_32BIT(zv)) { - | mov r1, [(uintptr_t)(zv)] - } else { - | mov64 r1, ((uintptr_t)(zv)) - | mov r1, [r1] - } - | mov qword [r0 + opline->result.var], r1 - |.else - | movsd xmm0, qword [(uintptr_t)(zv)] - | movsd qword [r0 + opline->result.var], xmm0 - |.endif - } - } - | mov dword [r0 + opline->result.var + 8], Z_TYPE_INFO_P(zv) + | STORE_CTZV r0, r1, opline->result.var, -1, zv | INC_OPLINE return 1; From 88c165c83d8b838cf28a193bf82873cdb476e8f9 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Wed, 7 Sep 2016 14:27:08 +0800 Subject: [PATCH 124/569] Fixed enable for ZEND_MUL/ZEND_DIV overflow --- ext/opcache/jit/zend_jit_x86.dasc | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 42675315c8842..e02a36403c209 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -492,8 +492,21 @@ static int zend_jit_new(dasm_State **Dst, zend_op *opline) || } |.endmacro +|.macro LONGF_OP, fp_ins, op_type, op +|| if (op_type == IS_CONST) { +| .if X64 +| mov r0, aword EX->literals +| fp_ins qword [r0 + op.constant] +| .else +| fp_ins dword [op.zv] +| .endif +|| } else { +| fp_ins aword [FP + op.var] +|| } +|.endmacro + |.macro LONGF_LOAD, op_type, op -| FP_OP fild, op_type, op +| LONGF_OP fild, op_type, op |.endmacro |.macro LONGF_MATH, opcode @@ -666,10 +679,6 @@ static int zend_jit_math(dasm_State **Dst, zend_op *opline, zend_op_array *op_ar if (((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) && ((op2_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG)) { - //TODO: imul/idiv overflow? - if (opline->opcode == ZEND_MUL || opline->opcode == ZEND_DIV) { - goto fallback; - } res_info = RES_INFO(); | LONG_LOAD opline->op1_type, opline->op1, r0 | LONG_MATH opline->opcode, opline->op2_type, opline->op2, r0 From a5f983c8068434696dbc15f47850c30c40ce2dcc Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 7 Sep 2016 11:06:19 +0300 Subject: [PATCH 125/569] Fixed egradation caused by 32/64-bit mess. JIR for SEND_VAL now supports TMP and VAR operands. --- ext/opcache/jit/zend_jit.c | 3 +- ext/opcache/jit/zend_jit_x86.dasc | 161 ++++++++++++++++-------------- 2 files changed, 88 insertions(+), 76 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index d7aeb5630bf9b..c54f3487e594e 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -120,7 +120,7 @@ static int zend_jit_delay(zend_op_array *op_array, zend_op *opline, uint32_t *in return 0; } -static void *zend_jit_delayed_entries_shutdown(dasm_State **Dst) { +static void zend_jit_delayed_entries_shutdown(dasm_State **Dst) { delayed_entry *entry; ZEND_HASH_FOREACH_PTR(&delayed_entries, entry) { @@ -130,7 +130,6 @@ static void *zend_jit_delayed_entries_shutdown(dasm_State **Dst) { zend_hash_destroy(&delayed_entries); delayed_entries.arData = NULL; delayed_offset = 0; - return; } static zend_string *zend_jit_func_name(zend_op_array *op_array) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index e02a36403c209..ade9fea8c44aa 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -531,41 +531,81 @@ static int zend_jit_new(dasm_State **Dst, zend_op *opline) | mov dword [FP + op.var + 8], IS_LONG |.endmacro -|.macro STORE_CTZV, base, reg, var, var_info, zv -||if (Z_TYPE_P(zv) > IS_TRUE) { -|| if (Z_TYPE_P(zv) < IS_STRING) { -|| /* Simply copy value */ -| .if X64 -|| if (IS_32BIT(Z_PTR_P(zv))) { -| mov qword [base + var], (uintptr_t)Z_PTR_P(zv) -|| } else { -| mov64 reg, (uintptr_t)Z_PTR_P(zv) -| mov qword [base + var], reg -|| } -| .else -| mov dword [base + var], zv->value.ww.w1 +|.macro ZVAL_COPY_CONST, dst_base, dst_offset, dst_info, zv, tmp_reg +|| if (Z_TYPE_P(zv) > IS_TRUE) { || if (Z_TYPE_P(zv) == IS_DOUBLE) { -| mov dword [base + var + 4], zv->value.ww.w2 +|.if X64 or SSE +|| if (Z_DVAL_P(zv) == 0.0) { +| xorps xmm0, xmm0 +| .if X64 +|| } else if (!IS_32BIT(zv)) { +| mov64 tmp_reg, ((uintptr_t)zv) +| movsd xmm0, qword [tmp_reg] +| .endif +|| } else { +| movsd xmm0, qword [((uint32_t)(uintptr_t)zv)] +|| } +| movsd qword [dst_base + dst_offset], xmm0 +|.else +|| if (Z_DVAL_P(zv) == 0.0) { +| fldz +|| } else if (Z_DVAL_P(zv) == 1.0) { +| fld1 +|| } else { +| fld qword [zv] +|| } +| fstp qword [dst_base + dst_offset] +|.endif +|| } else { +|.if X64 +|| if (!IS_32BIT(Z_LVAL_P(zv))) { +| mov64 tmp_reg, ((uintptr_t)Z_LVAL_P(zv)) +| mov qword [dst_base + dst_offset], tmp_reg +|| } else { +| mov qword [dst_base + dst_offset], Z_LVAL_P(zv) +|| } +|.else +| mov dword [dst_base + dst_offset], Z_LVAL_P(zv) +|.endif || } -| .endif -|| } else { -|| /* Z_PTR_P(zv) could be changed, Read from memory */ -| .if X64 -|| if (IS_32BIT(zv)) { -| mov reg, [(uintptr_t)(zv)] +|| } +|| if ((dst_info & MAY_BE_ANY) != (1<op1_type != IS_CONST) { - goto fallback; - } + uint32_t op1_info; - zv = RT_CONSTANT(op_array, opline->op1); + if (opline->op1_type == IS_CONST) { + zval *zv = RT_CONSTANT(op_array, opline->op1); - if (Z_REFCOUNTED_P(zv)) { - goto fallback; - } + if (Z_REFCOUNTED_P(zv)) { + /* TODO: allow JIT for Z_ADDREF_P() ??? */ + goto fallback; + } + | mov r0, EX->call + | ZVAL_COPY_CONST r0, opline->result.var, -1, zv, r1 + | INC_OPLINE + return 1; + } + op1_info = OP1_INFO(); | mov r0, EX->call - | STORE_CTZV r0, r1, opline->result.var, -1, zv + | ZVAL_COPY_VALUE r0, opline->result.var, FP, opline->op1.var, op1_info, r1, ecx, r2 | INC_OPLINE - return 1; + fallback: /* fallback to subroutine threading */ return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); From 8059f9695e39cc2f90eef298a2c370753cce37db Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Wed, 7 Sep 2016 16:29:11 +0800 Subject: [PATCH 126/569] Fixed memleak --- ext/opcache/jit/zend_jit.c | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index c54f3487e594e..783a432867aef 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -120,13 +120,14 @@ static int zend_jit_delay(zend_op_array *op_array, zend_op *opline, uint32_t *in return 0; } -static void zend_jit_delayed_entries_shutdown(dasm_State **Dst) { - delayed_entry *entry; - - ZEND_HASH_FOREACH_PTR(&delayed_entries, entry) { - zend_jit_delayed_gen(Dst, entry->op_array, entry->opline, entry->in, entry->out); - } ZEND_HASH_FOREACH_END(); +static void zend_jit_delayed_entries_shutdown(dasm_State **Dst, int flush) { + if (flush) { + delayed_entry *entry; + ZEND_HASH_FOREACH_PTR(&delayed_entries, entry) { + zend_jit_delayed_gen(Dst, entry->op_array, entry->opline, entry->in, entry->out); + } ZEND_HASH_FOREACH_END(); + } zend_hash_destroy(&delayed_entries); delayed_entries.arData = NULL; delayed_offset = 0; @@ -1000,7 +1001,7 @@ static int zend_real_jit_func(zend_op_array *op_array, zend_script *script) } } } - zend_jit_delayed_entries_shutdown(&dasm_state); + zend_jit_delayed_entries_shutdown(&dasm_state, 1); handler = dasm_link_and_encode(&dasm_state, op_array, &ssa.cfg, NULL); if (!handler) { @@ -1012,6 +1013,7 @@ static int zend_real_jit_func(zend_op_array *op_array, zend_script *script) return SUCCESS; jit_failure: + zend_jit_delayed_entries_shutdown(&dasm_state, 0); if (dasm_state) { dasm_free(&dasm_state); } From 1338af06d183f93befde2e88687f7c495c74b675 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 7 Sep 2016 11:50:01 +0300 Subject: [PATCH 127/569] Fixed suppor for -0.0 --- ext/opcache/jit/zend_jit_x86.dasc | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index ade9fea8c44aa..5d0ac6773071d 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -114,6 +114,11 @@ static uint32_t concrete_type(uint32_t value_type) return floor_log2(value_type & MAY_BE_ANY); } +static inline zend_bool is_signed(double d) +{ + return (((unsigned char*)&d)[sizeof(double)-1] & 0x80) != 0; +} + static int zend_jit_interrupt_handler_stub(dasm_State **Dst) { |->interrupt_handler: @@ -535,7 +540,7 @@ static int zend_jit_new(dasm_State **Dst, zend_op *opline) || if (Z_TYPE_P(zv) > IS_TRUE) { || if (Z_TYPE_P(zv) == IS_DOUBLE) { |.if X64 or SSE -|| if (Z_DVAL_P(zv) == 0.0) { +|| if (Z_DVAL_P(zv) == 0.0 && !is_signed(Z_DVAL_P(zv))) { | xorps xmm0, xmm0 | .if X64 || } else if (!IS_32BIT(zv)) { @@ -547,7 +552,7 @@ static int zend_jit_new(dasm_State **Dst, zend_op *opline) || } | movsd qword [dst_base + dst_offset], xmm0 |.else -|| if (Z_DVAL_P(zv) == 0.0) { +|| if (Z_DVAL_P(zv) == 0.0 && !is_signed(Z_DVAL_P(zv))) { | fldz || } else if (Z_DVAL_P(zv) == 1.0) { | fld1 From afc6366273d77c8bf0f883baa96c6e9f21a1d061 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Wed, 7 Sep 2016 18:15:37 +0800 Subject: [PATCH 128/569] Re-implemented delayed codes generating --- ext/opcache/jit/zend_jit.c | 57 +------------------ ext/opcache/jit/zend_jit_x86.dasc | 92 ++++++++++--------------------- 2 files changed, 30 insertions(+), 119 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 783a432867aef..d58ba836e93ed 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -60,18 +60,7 @@ static void *dasm_buf = NULL; static void *dasm_ptr = NULL; static void *dasm_end = NULL; -typedef struct _delayed_entry { - uint32_t in; - uint32_t out; - zend_op_array *op_array; - zend_op *opline; -} delayed_entry; - -static zend_array delayed_entries = {0}; -static uint32_t delayed_offset = 0; - static int zend_may_throw(zend_op *opline, zend_op_array *op_array, zend_ssa *ssa); -static int zend_jit_delay(zend_op_array *op_array, zend_op *opline, uint32_t *in, uint32_t *out); #include "dynasm/dasm_x86.h" #include "jit/zend_jit_x86.c" @@ -93,46 +82,6 @@ static int zend_jit_delay(zend_op_array *op_array, zend_op *opline, uint32_t *in #define DASM_ALIGNMENT 16 -static void free_delayed_entry(zval *el) { - efree((void*)Z_PTR_P(el)); -} - -static void zend_jit_delayed_entries_init(uint32_t offset) { - delayed_offset = offset; - if (delayed_entries.arData == NULL) { - zend_hash_init(&delayed_entries, 8, NULL, free_delayed_entry, 0); - } - return; -} - -static int zend_jit_delay(zend_op_array *op_array, zend_op *opline, uint32_t *in, uint32_t *out) { - if (delayed_offset < op_array->last * 2) { - delayed_entry *entry = (delayed_entry*)emalloc(sizeof(delayed_entry)); - entry->op_array = op_array; - entry->opline = opline; - entry->in = delayed_offset++; - entry->out = delayed_offset++; - zend_hash_next_index_insert_ptr(&delayed_entries, (void*)entry); - *in = entry->in; - *out = entry->out; - return 1; - } - return 0; -} - -static void zend_jit_delayed_entries_shutdown(dasm_State **Dst, int flush) { - if (flush) { - delayed_entry *entry; - - ZEND_HASH_FOREACH_PTR(&delayed_entries, entry) { - zend_jit_delayed_gen(Dst, entry->op_array, entry->opline, entry->in, entry->out); - } ZEND_HASH_FOREACH_END(); - } - zend_hash_destroy(&delayed_entries); - delayed_entries.arData = NULL; - delayed_offset = 0; -} - static zend_string *zend_jit_func_name(zend_op_array *op_array) { smart_str buf = {0}; @@ -752,9 +701,7 @@ static int zend_real_jit_func(zend_op_array *op_array, zend_script *script) dasm_setupglobal(&dasm_state, dasm_labels, zend_lb_MAX); dasm_setup(&dasm_state, dasm_actions); - dasm_growpc(&dasm_state, op_array->last * 2); - - zend_jit_delayed_entries_init(op_array->last + 1); + dasm_growpc(&dasm_state, ssa.cfg.blocks_count * 2); zend_jit_align_func(&dasm_state); for (b = 0; b < ssa.cfg.blocks_count; b++) { @@ -1001,7 +948,6 @@ static int zend_real_jit_func(zend_op_array *op_array, zend_script *script) } } } - zend_jit_delayed_entries_shutdown(&dasm_state, 1); handler = dasm_link_and_encode(&dasm_state, op_array, &ssa.cfg, NULL); if (!handler) { @@ -1013,7 +959,6 @@ static int zend_real_jit_func(zend_op_array *op_array, zend_script *script) return SUCCESS; jit_failure: - zend_jit_delayed_entries_shutdown(&dasm_state, 0); if (dasm_state) { dasm_free(&dasm_state); } diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 5d0ac6773071d..e133f7d41d8a8 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -60,7 +60,7 @@ |.globals zend_lb static void* dasm_labels[zend_lb_MAX]; -|.section code +|.section code, cold_codes |.macro LOAD_ADDR, reg, addr ||if (((ptrdiff_t)addr) <= 0x7fffffff) { @@ -618,50 +618,6 @@ static int zend_jit_new(dasm_State **Dst, zend_op *opline) | add IP, sizeof(zend_op) |.endmacro -static void zend_jit_delayed_gen(dasm_State **Dst, zend_op_array *op_array, zend_op *opline, uint32_t in, uint32_t out) { - if (in) { - |=>in: - } - switch (opline->opcode) { - case ZEND_ADD: - case ZEND_SUB: - case ZEND_MUL: - case ZEND_DIV: - | LONGF_LOAD opline->op2_type, opline->op2 - | LONGF_LOAD opline->op1_type, opline->op1 - | LONGF_MATH opline->opcode - | FP_STORE opline->result - break; - case ZEND_POST_INC: - case ZEND_PRE_INC: - |.if X64 - | mov64 rax, 0x43e0000000000000 - | mov qword [FP + opline->op1.var], rax - |.else - | mov dword [FP + opline->op1.var], 0 - | mov dword [FP + opline->op1.var + 4], 0x41e00000 - |.endif - | mov dword [FP + opline->op1.var + 8], IS_DOUBLE - break; - case ZEND_POST_DEC: - case ZEND_PRE_DEC: - |.if X64 - | mov64 rax, 0x43e0000000000000 - | mov qword [FP + opline->op1.var], rax - |.else - | mov dword [FP + opline->op1.var], 0x00200000 - | mov dword [FP + opline->op1.var + 4], 0xc1e00000 - |.endif - | mov dword [FP + opline->op1.var + 8], IS_DOUBLE - break; - default: - abort(); - } - if (out) { - | jmp =>out - } -} - static int zend_jit_inc_dec(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) { uint32_t op1_info, op1_def_info; @@ -684,15 +640,27 @@ static int zend_jit_inc_dec(dasm_State **Dst, zend_op *opline, zend_op_array *op } op1_def_info = OP1_DEF_INFO(); if (op1_def_info & MAY_BE_DOUBLE) { - uint32_t in, out; - if (zend_jit_delay(op_array, opline, &in, &out)) { - | jo =>in - |=>out: + | jno >1 + |.cold_codes + if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { + |.if X64 + | mov64 rax, 0x43e0000000000000 + | mov qword [FP + opline->op1.var], rax + |.else + | mov dword [FP + opline->op1.var], 0 + | mov dword [FP + opline->op1.var + 4], 0x41e00000 + |.endif } else { - | jno >1 - zend_jit_delayed_gen(Dst, op_array, opline, 0, 0); - |1: + |.if X64 + | mov64 rax, 0x43e0000000000000 + | mov qword [FP + opline->op1.var], rax + |.else + | mov dword [FP + opline->op1.var], 0x00200000 + | mov dword [FP + opline->op1.var + 4], 0xc1e00000 + |.endif } + | mov dword [FP + opline->op1.var + 8], IS_DOUBLE + |.code } else if (op1_info & (MAY_BE_ANY-MAY_BE_LONG)) { | jmp >1 } @@ -730,17 +698,15 @@ static int zend_jit_math(dasm_State **Dst, zend_op *opline, zend_op_array *op_ar if ((res_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) { | LONG_STORE opline->result, r0 } else { - uint32_t in, out; - if (zend_jit_delay(op_array, opline, &in, &out)) { - | LONG_STORE opline->result, r0 - | jo =>in - |=>out: - } else { - | LONG_STORE opline->result, r0 - | jno >1 - zend_jit_delayed_gen(Dst, op_array, opline, 0, 0); - |1: - } + | LONG_STORE opline->result, r0 + | jno >1 + |.cold_codes + | LONGF_LOAD opline->op2_type, opline->op2 + | LONGF_LOAD opline->op1_type, opline->op1 + | LONGF_MATH opline->opcode + | FP_STORE opline->result + |.code + |1: } } else if (((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) && ((op2_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE)) { From e2be07cf95481913e82a03d6cedcf3ce8f363b28 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Wed, 7 Sep 2016 18:55:49 +0800 Subject: [PATCH 129/569] Fixed jmps --- ext/opcache/jit/zend_jit_x86.dasc | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index e133f7d41d8a8..e7568c954d676 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -640,8 +640,9 @@ static int zend_jit_inc_dec(dasm_State **Dst, zend_op *opline, zend_op_array *op } op1_def_info = OP1_DEF_INFO(); if (op1_def_info & MAY_BE_DOUBLE) { - | jno >1 + | jo >1 |.cold_codes + |1: if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { |.if X64 | mov64 rax, 0x43e0000000000000 @@ -660,16 +661,17 @@ static int zend_jit_inc_dec(dasm_State **Dst, zend_op *opline, zend_op_array *op |.endif } | mov dword [FP + opline->op1.var + 8], IS_DOUBLE + | jmp >3 |.code } else if (op1_info & (MAY_BE_ANY-MAY_BE_LONG)) { - | jmp >1 + | jmp >3 } } if (op1_info & MAY_BE_DOUBLE) { |2: | } - |1: + |3: | INC_OPLINE return 1; } @@ -698,15 +700,17 @@ static int zend_jit_math(dasm_State **Dst, zend_op *opline, zend_op_array *op_ar if ((res_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) { | LONG_STORE opline->result, r0 } else { - | LONG_STORE opline->result, r0 - | jno >1 + | jo >1 |.cold_codes + |1: | LONGF_LOAD opline->op2_type, opline->op2 | LONGF_LOAD opline->op1_type, opline->op1 | LONGF_MATH opline->opcode | FP_STORE opline->result + | jmp >2 |.code - |1: + | LONG_STORE opline->result, r0 + |2: } } else if (((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) && ((op2_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE)) { From d99f14290b711b0893c2fe5cc7ede4a49bcf4d14 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 7 Sep 2016 15:49:25 +0300 Subject: [PATCH 130/569] Implemented JIT for PRE_INC/DEC of non IS_LONG operands --- ext/opcache/jit/zend_jit_x86.dasc | 118 ++++++++++++++++++------------ 1 file changed, 71 insertions(+), 47 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index e7568c954d676..fb7dd02f3e73f 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -622,59 +622,83 @@ static int zend_jit_inc_dec(dasm_State **Dst, zend_op *opline, zend_op_array *op { uint32_t op1_info, op1_def_info; - if (!ssa->ops || !ssa->var_info || opline->result_type != IS_UNUSED) { + if (!ssa->ops || !ssa->var_info || opline->op1_type != IS_CV || opline->result_type != IS_UNUSED) { goto fallback; } op1_info = OP1_INFO(); - if (!(op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)-(MAY_BE_LONG/*|MAY_BE_DOUBLE*/)))) { - if (op1_info & MAY_BE_LONG) { - if (op1_info & (MAY_BE_ANY-MAY_BE_LONG)) { - | cmp dword [FP + opline->op1.var + 8], IS_LONG - | jne >2 - } - if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { - | inc aword [FP + opline->op1.var] - } else { - | dec aword [FP + opline->op1.var] - } - op1_def_info = OP1_DEF_INFO(); - if (op1_def_info & MAY_BE_DOUBLE) { - | jo >1 - |.cold_codes - |1: - if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { - |.if X64 - | mov64 rax, 0x43e0000000000000 - | mov qword [FP + opline->op1.var], rax - |.else - | mov dword [FP + opline->op1.var], 0 - | mov dword [FP + opline->op1.var + 4], 0x41e00000 - |.endif - } else { - |.if X64 - | mov64 rax, 0x43e0000000000000 - | mov qword [FP + opline->op1.var], rax - |.else - | mov dword [FP + opline->op1.var], 0x00200000 - | mov dword [FP + opline->op1.var + 4], 0xc1e00000 - |.endif - } - | mov dword [FP + opline->op1.var + 8], IS_DOUBLE - | jmp >3 - |.code - } else if (op1_info & (MAY_BE_ANY-MAY_BE_LONG)) { - | jmp >3 - } - } - if (op1_info & MAY_BE_DOUBLE) { - |2: - | + if (!(op1_info & MAY_BE_LONG)) { + goto fallback; + } + + if (op1_info & (MAY_BE_ANY-MAY_BE_LONG)) { + | cmp dword [FP + opline->op1.var + 8], IS_LONG + | jne >2 + } + if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { + | inc aword [FP + opline->op1.var] + } else { + | dec aword [FP + opline->op1.var] + } + op1_def_info = OP1_DEF_INFO(); + if (op1_def_info & MAY_BE_DOUBLE) { + | jo >1 + |.cold_codes + |1: + if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { + |.if X64 + | mov64 rax, 0x43e0000000000000 + | mov qword [FP + opline->op1.var], rax + |.else + | mov dword [FP + opline->op1.var], 0 + | mov dword [FP + opline->op1.var + 4], 0x41e00000 + |.endif + } else { + |.if X64 + | mov64 rax, 0xc3e0000000000000 + | mov qword [FP + opline->op1.var], rax + |.else + | mov dword [FP + opline->op1.var], 0x00200000 + | mov dword [FP + opline->op1.var + 4], 0xc1e00000 + |.endif } - |3: - | INC_OPLINE - return 1; + | mov dword [FP + opline->op1.var + 8], IS_DOUBLE + | jmp >3 + |.code + } + if (op1_info & (MAY_BE_ANY-MAY_BE_LONG)) { + |.cold_codes + |2: + |.if X64 + | lea CARG1, [FP + opline->op1.var] + || if (IS_32BIT(dasm_end) && IS_32BIT(increment_function)) { + || if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { + | call qword &increment_function + || } else { + | call qword &decrement_function + || } + || } else { + || if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { + | LOAD_ADDR rax, increment_function + || } else { + | LOAD_ADDR rax, decrement_function + || } + | call rax + || } + |.else + | lea FCARG1, [FP + opline->op1.var] + || if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { + | call dword &increment_function + || } else { + | call dword &decrement_function + || } + |.endif + | jmp >3 + |.code } + |3: + | INC_OPLINE + return 1; fallback: /* fallback to subroutine threading */ From 5c068931fdf570b67117a6020e1a58ae5726fa88 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Wed, 7 Sep 2016 22:38:09 +0800 Subject: [PATCH 131/569] Implemented JIT for SEND_VAR(IS_VAR) --- ext/opcache/jit/zend_jit.c | 5 +++++ ext/opcache/jit/zend_jit_x86.dasc | 27 +++++++++++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index d58ba836e93ed..d5ae6eb0703b0 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -794,6 +794,11 @@ static int zend_real_jit_func(zend_op_array *op_array, zend_script *script) goto jit_failure; } break; + case ZEND_SEND_VAR: + if (!zend_jit_send_var(&dasm_state, opline, op_array, &ssa)) { + goto jit_failure; + } + break; case ZEND_IS_EQUAL: case ZEND_IS_NOT_EQUAL: case ZEND_IS_SMALLER: diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index fb7dd02f3e73f..95c1bdf04a541 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -1030,6 +1030,33 @@ static int zend_jit_send_val(dasm_State **Dst, zend_op *opline, zend_op_array *o return 1; } + if (!ssa->ops || !ssa->var_info) { + goto fallback; + } + + op1_info = OP1_INFO(); + | mov r0, EX->call + | ZVAL_COPY_VALUE r0, opline->result.var, FP, opline->op1.var, op1_info, r1, ecx, r2 + | INC_OPLINE + return 1; + +fallback: + /* fallback to subroutine threading */ + return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); +} + +static int zend_jit_send_var(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) +{ + uint32_t op1_info; + if (opline->op1_type == IS_CV || !ssa->ops || !ssa->var_info) { + goto fallback; + } + + op1_info = OP1_INFO(); + if (op1_info & (MAY_BE_UNDEF|MAY_BE_REF)) { + goto fallback; + } + op1_info = OP1_INFO(); | mov r0, EX->call | ZVAL_COPY_VALUE r0, opline->result.var, FP, opline->op1.var, op1_info, r1, ecx, r2 From 1b37d5044bcb1b68c47ec9c89768c161c70d56d2 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Thu, 8 Sep 2016 01:10:15 +0800 Subject: [PATCH 132/569] Implemented JIT for TYPE_CHECK --- ext/opcache/jit/zend_jit.c | 5 +++ ext/opcache/jit/zend_jit_x86.dasc | 57 +++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index d5ae6eb0703b0..06d9edae27f35 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -807,6 +807,11 @@ static int zend_real_jit_func(zend_op_array *op_array, zend_script *script) goto jit_failure; } break; + case ZEND_TYPE_CHECK: + if (!zend_jit_type_check(&dasm_state, opline, b, &i, op_array, &ssa)) { + goto jit_failure; + } + break; #endif case ZEND_RECV_INIT: if (ssa.cfg.split_at_recv) { diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 95c1bdf04a541..83ee85fab44af 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -1068,6 +1068,63 @@ fallback: return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); } +static int zend_jit_type_check(dasm_State **Dst, zend_op *opline, int b, int *opnum, zend_op_array *op_array, zend_ssa *ssa) +{ + uint32_t op1_info; + unsigned int target_label; + zend_uchar type; + + if (!ssa->ops || !ssa->var_info || opline->extended_value == IS_RESOURCE) { + goto fallback; + } + + op1_info = OP1_INFO(); + type = opline->extended_value; + if ((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) == (1<opcode == ZEND_JMPZ && + (opline+1)->op1_type == IS_TMP_VAR && + (opline+1)->op1.var == opline->result.var) { + (*opnum)++; + | add IP, sizeof(zend_op) * 2 + } else if ((opline+1)->opcode == ZEND_JMPNZ && + (opline+1)->op1_type == IS_TMP_VAR && + (opline+1)->op1.var == opline->result.var) { + (*opnum)++; + target_label = ssa->cfg.blocks[b].successors[0]; + | add IP, sizeof(zend_op) * 2 + | jmp =>target_label + } else { + | mov dword [FP + opline->result.var + 8], IS_TRUE + | INC_OPLINE + } + } else if (!(op1_info & (1<opcode == ZEND_JMPZ && + (opline+1)->op1_type == IS_TMP_VAR && + (opline+1)->op1.var == opline->result.var) { + (*opnum)++; + target_label = ssa->cfg.blocks[b].successors[0]; + | add IP, sizeof(zend_op) * 2 + | jmp =>target_label + } else if ((opline+1)->opcode == ZEND_JMPNZ && + (opline+1)->op1_type == IS_TMP_VAR && + (opline+1)->op1.var == opline->result.var) { + (*opnum)++; + | add IP, sizeof(zend_op) * 2 + } else { + | mov dword [FP + opline->result.var + 8], IS_FALSE + | INC_OPLINE + } + } else { + goto fallback; + } + + return 1; + +fallback: + /* fallback to subroutine threading */ + return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); +} + #endif /* From 956824f272083de3a785a24fb6cb9b0b95a69b83 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Thu, 8 Sep 2016 16:16:27 +0800 Subject: [PATCH 133/569] Make script avaliable in jit phase --- ext/opcache/zend_persist.c | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/ext/opcache/zend_persist.c b/ext/opcache/zend_persist.c index dc13001745749..906ce517300c0 100644 --- a/ext/opcache/zend_persist.c +++ b/ext/opcache/zend_persist.c @@ -67,13 +67,16 @@ } while (0) typedef void (*zend_persist_func_t)(zval*); +typedef void (*zend_persist_func_ex_t)(zval*, zend_persistent_script*); static void zend_persist_zval(zval *z); static const uint32_t uninitialized_bucket[-HT_MIN_MASK] = {HT_INVALID_IDX, HT_INVALID_IDX}; -static void zend_hash_persist(HashTable *ht, zend_persist_func_t pPersistElement) +#define zend_hash_persist(ht, func) zend_hash_persist_ex(ht, func, NULL) + +static void zend_hash_persist_ex(HashTable *ht, void *pPersistElement, zend_persistent_script *script) { uint32_t idx, nIndex; Bucket *p; @@ -125,7 +128,11 @@ static void zend_hash_persist(HashTable *ht, zend_persist_func_t pPersistElement } /* persist the data itself */ - pPersistElement(&p->val); + if (script) { + ((zend_persist_func_ex_t)pPersistElement)(&p->val, script); + } else { + ((zend_persist_func_t)pPersistElement)(&p->val); + } nIndex = p->h | ht->nTableMask; Z_NEXT(p->val) = HT_HASH(ht, nIndex); @@ -153,7 +160,11 @@ static void zend_hash_persist(HashTable *ht, zend_persist_func_t pPersistElement } /* persist the data itself */ - pPersistElement(&p->val); + if (script) { + ((zend_persist_func_ex_t)pPersistElement)(&p->val, script); + } else { + ((zend_persist_func_t)pPersistElement)(&p->val); + } } } @@ -576,7 +587,7 @@ static void zend_persist_op_array_ex(zend_op_array *op_array, zend_persistent_sc #endif } -static void zend_persist_op_array(zval *zv) +static void zend_persist_op_array(zval *zv, zend_persistent_script *script) { zend_op_array *op_array = Z_PTR_P(zv); zend_op_array *old_op_array = zend_shared_alloc_get_xlat_entry(op_array); @@ -591,7 +602,7 @@ static void zend_persist_op_array(zval *zv) zend_shared_alloc_register_xlat_entry(Z_PTR_P(zv), ZCG(arena_mem)); Z_PTR_P(zv) = ZCG(arena_mem); ZCG(arena_mem) = (void*)((char*)ZCG(arena_mem) + ZEND_ALIGNED_SIZE(sizeof(zend_op_array))); - zend_persist_op_array_ex(Z_PTR_P(zv), NULL); + zend_persist_op_array_ex(Z_PTR_P(zv), script); } static void zend_persist_property_info(zval *zv) @@ -654,7 +665,7 @@ static void zend_persist_class_constant(zval *zv) } } -static void zend_persist_class_entry(zval *zv) +static void zend_persist_class_entry(zval *zv, zend_persistent_script *script) { zend_class_entry *ce = Z_PTR_P(zv); @@ -664,7 +675,7 @@ static void zend_persist_class_entry(zval *zv) ce = Z_PTR_P(zv) = ZCG(arena_mem); ZCG(arena_mem) = (void*)((char*)ZCG(arena_mem) + ZEND_ALIGNED_SIZE(sizeof(zend_class_entry))); zend_accel_store_interned_string(ce->name); - zend_hash_persist(&ce->function_table, zend_persist_op_array); + zend_hash_persist_ex(&ce->function_table, zend_persist_op_array, script); if (ce->default_properties_table) { int i; @@ -827,9 +838,9 @@ static int zend_update_parent_ce(zval *zv) return 0; } -static void zend_accel_persist_class_table(HashTable *class_table) +static void zend_accel_persist_class_table(HashTable *class_table, zend_persistent_script *script) { - zend_hash_persist(class_table, zend_persist_class_entry); + zend_hash_persist_ex(class_table, zend_persist_class_entry, script); zend_hash_apply(class_table, (apply_func_t) zend_update_parent_ce); } @@ -860,8 +871,8 @@ zend_persistent_script *zend_accel_script_persist(zend_persistent_script *script zend_jit_unprotect(); #endif - zend_accel_persist_class_table(&script->script.class_table); - zend_hash_persist(&script->script.function_table, zend_persist_op_array); + zend_accel_persist_class_table(&script->script.class_table, script); + zend_hash_persist_ex(&script->script.function_table, zend_persist_op_array, script); zend_persist_op_array_ex(&script->script.main_op_array, script); #ifdef HAVE_JIT From f018411eb81066dd8c566f6e48839c566fa777b8 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Thu, 8 Sep 2016 16:38:17 +0800 Subject: [PATCH 134/569] Save one compare if possible in zend_jit_new --- ext/opcache/jit/zend_jit.c | 5 +- ext/opcache/jit/zend_jit_x86.dasc | 576 ++++++++++++++++-------------- 2 files changed, 302 insertions(+), 279 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 06d9edae27f35..e67dc05bd5c12 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -928,12 +928,9 @@ static int zend_real_jit_func(zend_op_array *op_array, zend_script *script) } break; case ZEND_NEW: - if (!zend_jit_new(&dasm_state, opline)) { + if (!zend_jit_new(&dasm_state, opline, &i, op_array, &ssa)) { goto jit_failure; } - if (EXPECTED(opline->extended_value == 0 && (opline+1)->opcode == ZEND_DO_FCALL)) { - i++; - } break; case ZEND_JMPZNZ: if (!zend_jit_handler(&dasm_state, opline, zend_may_throw(opline, op_array, &ssa)) || diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 83ee85fab44af..f59af4b626abc 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -72,6 +72,276 @@ static void* dasm_labels[zend_lb_MAX]; #define IS_32BIT(addr) (((uintptr_t)(addr)) <= 0x7fffffff) +#if ZEND_JIT_LEVEL >= ZEND_JIT_LEVEL_FULL + +|.macro FP_OP, fp_ins, op_type, op +||if (op_type == IS_CONST) { +| .if X64 +| mov r0, aword EX->literals +| fp_ins qword [r0 + op.constant] +| .else +| fp_ins qword [op.zv] +| .endif +||} else { +| fp_ins qword [FP + op.var] +||} +|.endmacro + +|.macro FP_LOAD, op_type, op +| FP_OP fld, op_type, op +|.endmacro + +|.macro FP_MATH, opcode, op_type, op +||switch (opcode) { +|| case ZEND_ADD: +| FP_OP fadd, op_type, op +|| break; +|| case ZEND_SUB: +| FP_OP fsub, op_type, op +|| break; +|| case ZEND_MUL: +| FP_OP fmul, op_type, op +|| break; +|| case ZEND_DIV: +| FP_OP fdiv, op_type, op +|| break; +||} +|.endmacro + +|.macro FP_STORE, op +| fstp qword [FP + op.var] +| mov dword [FP + op.var + 8], IS_DOUBLE +|.endmacro + +|.macro SSE_OP, sse_ins, op_type, op, reg +||if (op_type == IS_CONST) { +| .if X64 +| mov r0, aword EX->literals +| sse_ins reg, qword [r0 + op.constant] +| .else +| sse_ins reg, qword [op.zv] +| .endif +||} else { +| sse_ins reg, qword [FP + op.var] +||} +|.endmacro + +|.macro SSE_LOAD, op_type, op, reg +| SSE_OP movsd, op_type, op, reg +|.endmacro + +|.macro SSE_MATH, opcode, op_type, op, reg +||switch (opcode) { +|| case ZEND_ADD: +| SSE_OP addsd, op_type, op, reg +|| break; +|| case ZEND_SUB: +| SSE_OP subsd, op_type, op, reg +|| break; +|| case ZEND_MUL: +| SSE_OP mulsd, op_type, op, reg +|| break; +|| case ZEND_DIV: +| SSE_OP divsd, op_type, op, reg +|| break; +||} +|.endmacro + +|.macro SSE_MATH2, opcode, reg1, reg2 +||switch (opcode) { +|| case ZEND_ADD: +| addsd reg2, reg1 +|| break; +|| case ZEND_SUB: +| subsd reg2, reg1 +|| break; +|| case ZEND_MUL: +| mulsd reg2, reg1 +|| break; +|| case ZEND_DIV: +| divsd reg2, reg1 +|| break; +||} +|.endmacro + +|.macro SSE_STORE, op, reg +| movsd qword [FP + op.var], reg +| mov dword [FP + op.var + 8], IS_DOUBLE +|.endmacro + +|.macro LONG_OP, long_ins, op_type, op, reg +||if (op_type == IS_CONST) { +| .if X64 +|| if (!IS_32BIT(Z_LVAL_P(RT_CONSTANT(op_array, op)))) { +| mov64 r1, Z_LVAL_P(RT_CONSTANT(op_array, op)) +| long_ins reg, r1 +|| } else { +| long_ins reg, Z_LVAL_P(RT_CONSTANT(op_array, op)) +|| } +| .else +| long_ins reg, Z_LVAL_P(RT_CONSTANT(op_array, op)) +| .endif +||} else { +| long_ins reg, aword [FP + op.var] +||} +|.endmacro + +|.macro LONG_LOAD, op_type, op, reg +||if (op_type == IS_CONST) { +| .if X64 +|| if (!IS_32BIT(Z_LVAL_P(RT_CONSTANT(op_array, op)))) { +| mov64 reg, Z_LVAL_P(RT_CONSTANT(op_array, op)) +|| } else { +| mov reg, Z_LVAL_P(RT_CONSTANT(op_array, op)) +|| } +| .else +| mov reg, Z_LVAL_P(RT_CONSTANT(op_array, op)) +| .endif +||} else { +| mov reg, aword [FP + op.var] +||} +|.endmacro + +|.macro LONG_MATH, opcode, op_type, op, reg +||switch (opcode) { +|| case ZEND_ADD: +| LONG_OP add, op_type, op, reg +|| break; +|| case ZEND_SUB: +| LONG_OP sub, op_type, op, reg +|| break; +|| case ZEND_MUL: +| LONG_OP imul, op_type, op, reg +|| break; +|| case ZEND_DIV: +| idiv aword [FP + op.var] +|| break; +||} +|.endmacro + +|.macro LONGF_OP, fp_ins, op_type, op +||if (op_type == IS_CONST) { +| .if X64 +| mov r0, aword EX->literals +| fp_ins qword [r0 + op.constant] +| .else +| fp_ins dword [op.zv] +| .endif +||} else { +| fp_ins aword [FP + op.var] +||} +|.endmacro + +|.macro LONGF_LOAD, op_type, op +| LONGF_OP fild, op_type, op +|.endmacro + +|.macro LONGF_MATH, opcode +||switch (opcode) { +|| case ZEND_ADD: +| fadd st0, st1 +|| break; +|| case ZEND_SUB: +| fsub st0, st1 +|| break; +|| case ZEND_MUL: +| fmul st0, st1 +|| break; +|| case ZEND_DIV: +| fdiv st0, st1 +|| break; +||} +|.endmacro + +|.macro LONG_STORE, op, reg +| mov aword [FP + op.var], reg +| mov dword [FP + op.var + 8], IS_LONG +|.endmacro + +|.macro ZVAL_COPY_CONST, dst_base, dst_offset, dst_info, zv, tmp_reg +||if (Z_TYPE_P(zv) > IS_TRUE) { +|| if (Z_TYPE_P(zv) == IS_DOUBLE) { +|.if X64 or SSE +|| if (Z_DVAL_P(zv) == 0.0 && !is_signed(Z_DVAL_P(zv))) { +| xorps xmm0, xmm0 +| .if X64 +|| } else if (!IS_32BIT(zv)) { +| mov64 tmp_reg, ((uintptr_t)zv) +| movsd xmm0, qword [tmp_reg] +| .endif +|| } else { +| movsd xmm0, qword [((uint32_t)(uintptr_t)zv)] +|| } +| movsd qword [dst_base + dst_offset], xmm0 +|.else +|| if (Z_DVAL_P(zv) == 0.0 && !is_signed(Z_DVAL_P(zv))) { +| fldz +|| } else if (Z_DVAL_P(zv) == 1.0) { +| fld1 +|| } else { +| fld qword [zv] +|| } +| fstp qword [dst_base + dst_offset] +|.endif +|| } else { +|.if X64 +|| if (!IS_32BIT(Z_LVAL_P(zv))) { +| mov64 tmp_reg, ((uintptr_t)Z_LVAL_P(zv)) +| mov qword [dst_base + dst_offset], tmp_reg +|| } else { +| mov qword [dst_base + dst_offset], Z_LVAL_P(zv) +|| } +|.else +| mov dword [dst_base + dst_offset], Z_LVAL_P(zv) +|.endif +|| } +||} +||if ((dst_info & MAY_BE_ANY) != (1<extended_value == 0 && (opline+1)->opcode == ZEND_DO_FCALL)) { - zend_op *next_opline = opline + 1; - - | cmp IPl, next_opline - | jnz >1 - zend_jit_call(Dst, next_opline); - |1: + if (!zend_jit_handler(Dst, opline, 1)) { + return 0; + } + if (opline->extended_value == 0 && (opline+1)->opcode == ZEND_DO_FCALL) { + zend_class_entry *ce = NULL; + if (opline->op1_type == IS_CONST) { + zval *zv = RT_CONSTANT(op_array, opline->op1); + if (Z_TYPE_P(zv) == IS_STRING) { + zval *lc = zv + 1; + ce = (zend_class_entry*)zend_hash_find_ptr(EG(class_table), Z_STR_P(lc)); + } + } +#if ZEND_JIT_LEVEL >= ZEND_JIT_LEVEL_FULL + if (ssa->ops && ssa->var_info) { + zend_ssa_var_info *res_ssa = &ssa->var_info[ssa->ops[opline - op_array->opcodes].result_def]; + if (res_ssa->ce && !res_ssa->is_instanceof) { + ce = res_ssa->ce; + } + } +#endif + (*opnum)++; + if (!ce || ce->constructor) { + zend_op *next_opline = opline + 1; + | cmp IPl, next_opline + | jnz >1 + zend_jit_call(Dst, next_opline); + |1: + } else { + //| INC_OPLINE + } } return 1; } #if ZEND_JIT_LEVEL >= ZEND_JIT_LEVEL_FULL -|.macro FP_OP, fp_ins, op_type, op -|| if (op_type == IS_CONST) { -| .if X64 -| mov r0, aword EX->literals -| fp_ins qword [r0 + op.constant] -| .else -| fp_ins qword [op.zv] -| .endif -|| } else { -| fp_ins qword [FP + op.var] -|| } -|.endmacro - -|.macro FP_LOAD, op_type, op -| FP_OP fld, op_type, op -|.endmacro - -|.macro FP_MATH, opcode, op_type, op -|| switch (opcode) { -|| case ZEND_ADD: -| FP_OP fadd, op_type, op -|| break; -|| case ZEND_SUB: -| FP_OP fsub, op_type, op -|| break; -|| case ZEND_MUL: -| FP_OP fmul, op_type, op -|| break; -|| case ZEND_DIV: -| FP_OP fdiv, op_type, op -|| break; -|| } -|.endmacro - -|.macro FP_STORE, op -| fstp qword [FP + op.var] -| mov dword [FP + op.var + 8], IS_DOUBLE -|.endmacro - -|.macro SSE_OP, sse_ins, op_type, op, reg -|| if (op_type == IS_CONST) { -| .if X64 -| mov r0, aword EX->literals -| sse_ins reg, qword [r0 + op.constant] -| .else -| sse_ins reg, qword [op.zv] -| .endif -|| } else { -| sse_ins reg, qword [FP + op.var] -|| } -|.endmacro - -|.macro SSE_LOAD, op_type, op, reg -| SSE_OP movsd, op_type, op, reg -|.endmacro - -|.macro SSE_MATH, opcode, op_type, op, reg -|| switch (opcode) { -|| case ZEND_ADD: -| SSE_OP addsd, op_type, op, reg -|| break; -|| case ZEND_SUB: -| SSE_OP subsd, op_type, op, reg -|| break; -|| case ZEND_MUL: -| SSE_OP mulsd, op_type, op, reg -|| break; -|| case ZEND_DIV: -| SSE_OP divsd, op_type, op, reg -|| break; -|| } -|.endmacro - -|.macro SSE_MATH2, opcode, reg1, reg2 -|| switch (opcode) { -|| case ZEND_ADD: -| addsd reg2, reg1 -|| break; -|| case ZEND_SUB: -| subsd reg2, reg1 -|| break; -|| case ZEND_MUL: -| mulsd reg2, reg1 -|| break; -|| case ZEND_DIV: -| divsd reg2, reg1 -|| break; -|| } -|.endmacro - -|.macro SSE_STORE, op, reg -| movsd qword [FP + op.var], reg -| mov dword [FP + op.var + 8], IS_DOUBLE -|.endmacro - -|.macro LONG_OP, long_ins, op_type, op, reg -|| if (op_type == IS_CONST) { -| .if X64 -|| if (!IS_32BIT(Z_LVAL_P(RT_CONSTANT(op_array, op)))) { -| mov64 r1, Z_LVAL_P(RT_CONSTANT(op_array, op)) -| long_ins reg, r1 -|| } else { -| long_ins reg, Z_LVAL_P(RT_CONSTANT(op_array, op)) -|| } -| .else -| long_ins reg, Z_LVAL_P(RT_CONSTANT(op_array, op)) -| .endif -|| } else { -| long_ins reg, aword [FP + op.var] -|| } -|.endmacro - -|.macro LONG_LOAD, op_type, op, reg -|| if (op_type == IS_CONST) { -| .if X64 -|| if (!IS_32BIT(Z_LVAL_P(RT_CONSTANT(op_array, op)))) { -| mov64 reg, Z_LVAL_P(RT_CONSTANT(op_array, op)) -|| } else { -| mov reg, Z_LVAL_P(RT_CONSTANT(op_array, op)) -|| } -| .else -| mov reg, Z_LVAL_P(RT_CONSTANT(op_array, op)) -| .endif -|| } else { -| mov reg, aword [FP + op.var] -|| } -|.endmacro - -|.macro LONG_MATH, opcode, op_type, op, reg -|| switch (opcode) { -|| case ZEND_ADD: -| LONG_OP add, op_type, op, reg -|| break; -|| case ZEND_SUB: -| LONG_OP sub, op_type, op, reg -|| break; -|| case ZEND_MUL: -| LONG_OP imul, op_type, op, reg -|| break; -|| case ZEND_DIV: -| idiv aword [FP + op.var] -|| break; -|| } -|.endmacro - -|.macro LONGF_OP, fp_ins, op_type, op -|| if (op_type == IS_CONST) { -| .if X64 -| mov r0, aword EX->literals -| fp_ins qword [r0 + op.constant] -| .else -| fp_ins dword [op.zv] -| .endif -|| } else { -| fp_ins aword [FP + op.var] -|| } -|.endmacro - -|.macro LONGF_LOAD, op_type, op -| LONGF_OP fild, op_type, op -|.endmacro - -|.macro LONGF_MATH, opcode -|| switch (opcode) { -|| case ZEND_ADD: -| fadd st0, st1 -|| break; -|| case ZEND_SUB: -| fsub st0, st1 -|| break; -|| case ZEND_MUL: -| fmul st0, st1 -|| break; -|| case ZEND_DIV: -| fdiv st0, st1 -|| break; -|| } -|.endmacro - -|.macro LONG_STORE, op, reg -| mov aword [FP + op.var], reg -| mov dword [FP + op.var + 8], IS_LONG -|.endmacro - -|.macro ZVAL_COPY_CONST, dst_base, dst_offset, dst_info, zv, tmp_reg -|| if (Z_TYPE_P(zv) > IS_TRUE) { -|| if (Z_TYPE_P(zv) == IS_DOUBLE) { -|.if X64 or SSE -|| if (Z_DVAL_P(zv) == 0.0 && !is_signed(Z_DVAL_P(zv))) { -| xorps xmm0, xmm0 -| .if X64 -|| } else if (!IS_32BIT(zv)) { -| mov64 tmp_reg, ((uintptr_t)zv) -| movsd xmm0, qword [tmp_reg] -| .endif -|| } else { -| movsd xmm0, qword [((uint32_t)(uintptr_t)zv)] -|| } -| movsd qword [dst_base + dst_offset], xmm0 -|.else -|| if (Z_DVAL_P(zv) == 0.0 && !is_signed(Z_DVAL_P(zv))) { -| fldz -|| } else if (Z_DVAL_P(zv) == 1.0) { -| fld1 -|| } else { -| fld qword [zv] -|| } -| fstp qword [dst_base + dst_offset] -|.endif -|| } else { -|.if X64 -|| if (!IS_32BIT(Z_LVAL_P(zv))) { -| mov64 tmp_reg, ((uintptr_t)Z_LVAL_P(zv)) -| mov qword [dst_base + dst_offset], tmp_reg -|| } else { -| mov qword [dst_base + dst_offset], Z_LVAL_P(zv) -|| } -|.else -| mov dword [dst_base + dst_offset], Z_LVAL_P(zv) -|.endif -|| } -|| } -|| if ((dst_info & MAY_BE_ANY) != (1< Date: Thu, 8 Sep 2016 16:41:10 +0800 Subject: [PATCH 135/569] Thay are alternative --- ext/opcache/jit/zend_jit_x86.dasc | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index f59af4b626abc..565c98c5f033f 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -613,13 +613,6 @@ static int zend_jit_new(dasm_State **Dst, zend_op *opline, uint32_t *opnum, zend } if (opline->extended_value == 0 && (opline+1)->opcode == ZEND_DO_FCALL) { zend_class_entry *ce = NULL; - if (opline->op1_type == IS_CONST) { - zval *zv = RT_CONSTANT(op_array, opline->op1); - if (Z_TYPE_P(zv) == IS_STRING) { - zval *lc = zv + 1; - ce = (zend_class_entry*)zend_hash_find_ptr(EG(class_table), Z_STR_P(lc)); - } - } #if ZEND_JIT_LEVEL >= ZEND_JIT_LEVEL_FULL if (ssa->ops && ssa->var_info) { zend_ssa_var_info *res_ssa = &ssa->var_info[ssa->ops[opline - op_array->opcodes].result_def]; @@ -627,6 +620,14 @@ static int zend_jit_new(dasm_State **Dst, zend_op *opline, uint32_t *opnum, zend ce = res_ssa->ce; } } +#else + if (opline->op1_type == IS_CONST) { + zval *zv = RT_CONSTANT(op_array, opline->op1); + if (Z_TYPE_P(zv) == IS_STRING) { + zval *lc = zv + 1; + ce = (zend_class_entry*)zend_hash_find_ptr(EG(class_table), Z_STR_P(lc)); + } + } #endif (*opnum)++; if (!ce || ce->constructor) { From cdc83645aed4f2ed65a0354c14d9f51639b44600 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Thu, 8 Sep 2016 16:56:33 +0800 Subject: [PATCH 136/569] Remove useless set --- ext/opcache/jit/zend_jit_x86.dasc | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 565c98c5f033f..668df50528504 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -1112,13 +1112,14 @@ static int zend_jit_type_check(dasm_State **Dst, zend_op *opline, int b, int *op (opline+1)->op1_type == IS_TMP_VAR && (opline+1)->op1.var == opline->result.var) { (*opnum)++; - | add IP, sizeof(zend_op) * 2 + if (!(ssa->cfg.blocks[ssa->cfg.blocks[b].successors[1]].flags & ZEND_BB_TARGET)) { + | add IP, sizeof(zend_op) * 2 + } } else if ((opline+1)->opcode == ZEND_JMPNZ && (opline+1)->op1_type == IS_TMP_VAR && (opline+1)->op1.var == opline->result.var) { (*opnum)++; target_label = ssa->cfg.blocks[b].successors[0]; - | add IP, sizeof(zend_op) * 2 | jmp =>target_label } else { | mov dword [FP + opline->result.var + 8], IS_TRUE @@ -1130,13 +1131,14 @@ static int zend_jit_type_check(dasm_State **Dst, zend_op *opline, int b, int *op (opline+1)->op1.var == opline->result.var) { (*opnum)++; target_label = ssa->cfg.blocks[b].successors[0]; - | add IP, sizeof(zend_op) * 2 | jmp =>target_label } else if ((opline+1)->opcode == ZEND_JMPNZ && (opline+1)->op1_type == IS_TMP_VAR && (opline+1)->op1.var == opline->result.var) { (*opnum)++; + if (!(ssa->cfg.blocks[ssa->cfg.blocks[b].successors[1]].flags & ZEND_BB_TARGET)) { | add IP, sizeof(zend_op) * 2 + } } else { | mov dword [FP + opline->result.var + 8], IS_FALSE | INC_OPLINE From 1de41363d9c341a81afc3d855eddebf3161ee7db Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 8 Sep 2016 12:48:51 +0300 Subject: [PATCH 137/569] Revert "Make script avaliable in jit phase" This reverts commit 956824f272083de3a785a24fb6cb9b0b95a69b83. --- ext/opcache/zend_persist.c | 33 +++++++++++---------------------- 1 file changed, 11 insertions(+), 22 deletions(-) diff --git a/ext/opcache/zend_persist.c b/ext/opcache/zend_persist.c index 906ce517300c0..dc13001745749 100644 --- a/ext/opcache/zend_persist.c +++ b/ext/opcache/zend_persist.c @@ -67,16 +67,13 @@ } while (0) typedef void (*zend_persist_func_t)(zval*); -typedef void (*zend_persist_func_ex_t)(zval*, zend_persistent_script*); static void zend_persist_zval(zval *z); static const uint32_t uninitialized_bucket[-HT_MIN_MASK] = {HT_INVALID_IDX, HT_INVALID_IDX}; -#define zend_hash_persist(ht, func) zend_hash_persist_ex(ht, func, NULL) - -static void zend_hash_persist_ex(HashTable *ht, void *pPersistElement, zend_persistent_script *script) +static void zend_hash_persist(HashTable *ht, zend_persist_func_t pPersistElement) { uint32_t idx, nIndex; Bucket *p; @@ -128,11 +125,7 @@ static void zend_hash_persist_ex(HashTable *ht, void *pPersistElement, zend_pers } /* persist the data itself */ - if (script) { - ((zend_persist_func_ex_t)pPersistElement)(&p->val, script); - } else { - ((zend_persist_func_t)pPersistElement)(&p->val); - } + pPersistElement(&p->val); nIndex = p->h | ht->nTableMask; Z_NEXT(p->val) = HT_HASH(ht, nIndex); @@ -160,11 +153,7 @@ static void zend_hash_persist_ex(HashTable *ht, void *pPersistElement, zend_pers } /* persist the data itself */ - if (script) { - ((zend_persist_func_ex_t)pPersistElement)(&p->val, script); - } else { - ((zend_persist_func_t)pPersistElement)(&p->val); - } + pPersistElement(&p->val); } } @@ -587,7 +576,7 @@ static void zend_persist_op_array_ex(zend_op_array *op_array, zend_persistent_sc #endif } -static void zend_persist_op_array(zval *zv, zend_persistent_script *script) +static void zend_persist_op_array(zval *zv) { zend_op_array *op_array = Z_PTR_P(zv); zend_op_array *old_op_array = zend_shared_alloc_get_xlat_entry(op_array); @@ -602,7 +591,7 @@ static void zend_persist_op_array(zval *zv, zend_persistent_script *script) zend_shared_alloc_register_xlat_entry(Z_PTR_P(zv), ZCG(arena_mem)); Z_PTR_P(zv) = ZCG(arena_mem); ZCG(arena_mem) = (void*)((char*)ZCG(arena_mem) + ZEND_ALIGNED_SIZE(sizeof(zend_op_array))); - zend_persist_op_array_ex(Z_PTR_P(zv), script); + zend_persist_op_array_ex(Z_PTR_P(zv), NULL); } static void zend_persist_property_info(zval *zv) @@ -665,7 +654,7 @@ static void zend_persist_class_constant(zval *zv) } } -static void zend_persist_class_entry(zval *zv, zend_persistent_script *script) +static void zend_persist_class_entry(zval *zv) { zend_class_entry *ce = Z_PTR_P(zv); @@ -675,7 +664,7 @@ static void zend_persist_class_entry(zval *zv, zend_persistent_script *script) ce = Z_PTR_P(zv) = ZCG(arena_mem); ZCG(arena_mem) = (void*)((char*)ZCG(arena_mem) + ZEND_ALIGNED_SIZE(sizeof(zend_class_entry))); zend_accel_store_interned_string(ce->name); - zend_hash_persist_ex(&ce->function_table, zend_persist_op_array, script); + zend_hash_persist(&ce->function_table, zend_persist_op_array); if (ce->default_properties_table) { int i; @@ -838,9 +827,9 @@ static int zend_update_parent_ce(zval *zv) return 0; } -static void zend_accel_persist_class_table(HashTable *class_table, zend_persistent_script *script) +static void zend_accel_persist_class_table(HashTable *class_table) { - zend_hash_persist_ex(class_table, zend_persist_class_entry, script); + zend_hash_persist(class_table, zend_persist_class_entry); zend_hash_apply(class_table, (apply_func_t) zend_update_parent_ce); } @@ -871,8 +860,8 @@ zend_persistent_script *zend_accel_script_persist(zend_persistent_script *script zend_jit_unprotect(); #endif - zend_accel_persist_class_table(&script->script.class_table, script); - zend_hash_persist_ex(&script->script.function_table, zend_persist_op_array, script); + zend_accel_persist_class_table(&script->script.class_table); + zend_hash_persist(&script->script.function_table, zend_persist_op_array); zend_persist_op_array_ex(&script->script.main_op_array, script); #ifdef HAVE_JIT From 15d4386b3dd2d5ca4eb07540f0524d94368bd959 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 8 Sep 2016 13:00:47 +0300 Subject: [PATCH 138/569] Use different approach to pass "script" into zend_jit() --- ext/opcache/zend_persist.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ext/opcache/zend_persist.c b/ext/opcache/zend_persist.c index dc13001745749..dedc8a0aa856c 100644 --- a/ext/opcache/zend_persist.c +++ b/ext/opcache/zend_persist.c @@ -571,7 +571,7 @@ static void zend_persist_op_array_ex(zend_op_array *op_array, zend_persistent_sc #ifdef HAVE_JIT if (do_jit && ZCG(accel_directives).jit_buffer_size) { - zend_jit(op_array, &main_persistent_script->script); + zend_jit(op_array, ZCG(current_persistent_script) ? &ZCG(current_persistent_script)->script : NULL); } #endif } @@ -860,9 +860,11 @@ zend_persistent_script *zend_accel_script_persist(zend_persistent_script *script zend_jit_unprotect(); #endif + ZCG(current_persistent_script) = script; zend_accel_persist_class_table(&script->script.class_table); zend_hash_persist(&script->script.function_table, zend_persist_op_array); zend_persist_op_array_ex(&script->script.main_op_array, script); + ZCG(current_persistent_script) = NULL; #ifdef HAVE_JIT zend_jit_protect(); From c5af750203699560345339e2e35eab98d14c2ab5 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 8 Sep 2016 13:33:49 +0300 Subject: [PATCH 139/569] Extended JIT for SEND_VAR to support IS_TYPE_REFCOUNTED --- ext/opcache/jit/zend_jit_x86.dasc | 33 ++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 668df50528504..6a9f75bc74570 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -304,8 +304,8 @@ static void* dasm_labels[zend_lb_MAX]; |.macro ZVAL_COPY_VALUE, dst_base, dst_offset, src_base, src_offset, src_info, tmp_reg1, tmp_reg1d, tmp_reg2 ||if (src_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) { || if (!(src_info & MAY_BE_DOUBLE)) { -| mov tmp_reg1, aword [src_base + src_offset] -| mov aword [dst_base + dst_offset], tmp_reg1 +| mov tmp_reg2, aword [src_base + src_offset] +| mov aword [dst_base + dst_offset], tmp_reg2 || } else if ((src_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { |.if X64 or SSE | movsd xmm0, qword [src_base + src_offset] @@ -316,13 +316,13 @@ static void* dasm_labels[zend_lb_MAX]; |.endif || } else { |.if X64 -| mov tmp_reg1, aword [src_base + src_offset] -| mov aword [dst_base + dst_offset], tmp_reg1 +| mov tmp_reg2, aword [src_base + src_offset] +| mov aword [dst_base + dst_offset], tmp_reg2 |.else -| mov tmp_reg1, dword [src_base + src_offset] -| mov tmp_reg2, dword [src_base + src_offset + 4] -| mov dword [dst_base + dst_offset], tmp_reg1 -| mov dword [dst_base + dst_offset + 4], tmp_reg2 +| mov tmp_reg2, dword [src_base + src_offset] +| mov tmp_reg1, dword [src_base + src_offset + 4] +| mov dword [dst_base + dst_offset], tmp_reg2 +| mov dword [dst_base + dst_offset + 4], tmp_reg1 |.endif || } ||} @@ -336,6 +336,17 @@ static void* dasm_labels[zend_lb_MAX]; ||} |.endmacro +|.macro TRY_ADDREF, val_info, type_flags_reg, value_ptr_reg +||if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { +|| if (val_info & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { +| and type_flags_reg, IS_TYPE_REFCOUNTED +| je >1 +|| } +| inc dword [value_ptr_reg] +|1: +||} +|.endmacro + |.macro INC_OPLINE | add IP, sizeof(zend_op) |.endmacro @@ -1075,18 +1086,22 @@ fallback: static int zend_jit_send_var(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) { uint32_t op1_info; - if (opline->op1_type == IS_CV || !ssa->ops || !ssa->var_info) { + if (!ssa->ops || !ssa->var_info) { goto fallback; } op1_info = OP1_INFO(); if (op1_info & (MAY_BE_UNDEF|MAY_BE_REF)) { + // TODO: support for references ??? goto fallback; } op1_info = OP1_INFO(); | mov r0, EX->call | ZVAL_COPY_VALUE r0, opline->result.var, FP, opline->op1.var, op1_info, r1, ecx, r2 + || if (opline->op1_type == IS_CV) { + | TRY_ADDREF op1_info, ch, r2 + || } | INC_OPLINE return 1; From e8185875747cbf4c1ec1f9847e4516f1c041c3ed Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 8 Sep 2016 14:02:38 +0300 Subject: [PATCH 140/569] Implemented JIT support for IS_TYPE_REFCOUNTED in QM_ASSIGN, ASSIGN and SEND_VAL. --- ext/opcache/jit/zend_jit_x86.dasc | 36 ++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 6a9f75bc74570..2d66bf8e43689 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -336,6 +336,19 @@ static void* dasm_labels[zend_lb_MAX]; ||} |.endmacro +|.macro ADDREF_CONST, zv, tmp_reg +|.if X64 +|| if (!IS_32BIT(Z_LVAL_P(zv))) { +| mov64 tmp_reg, ((uintptr_t)Z_LVAL_P(zv)) +| inc dword [tmp_reg] +|| } else { +| inc dword [Z_LVAL_P(zv)] +|| } +|.else +| inc dword [Z_LVAL_P(zv)] +|.endif +|.endmacro + |.macro TRY_ADDREF, val_info, type_flags_reg, value_ptr_reg ||if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { || if (val_info & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { @@ -987,23 +1000,23 @@ fallback: static int zend_jit_simple_assign(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, uint32_t var, uint32_t var_info, zend_uchar val_type, znode_op val, uint32_t val_info) { - if (val_type == IS_CV && (val_info & (MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_STRING|MAY_BE_RESOURCE|MAY_BE_ARRAY|MAY_BE_OBJECT))) { - /* TODO: allow JIT for Z_ADDREF_P() ??? */ - goto fallback; - } else if (val_type == IS_VAR && (val_info & MAY_BE_REF)) { + if (val_info & (MAY_BE_UNDEF|MAY_BE_REF)) { + /* TODO: Support for references ??? */ goto fallback; } else if (val_type == IS_CONST) { zval *zv = RT_CONSTANT(op_array, val); - if (Z_OPT_REFCOUNTED_P(zv)) { - /* TODO: allow JIT for Z_ADDREF_P() ??? */ - goto fallback; - } | ZVAL_COPY_CONST FP, var, var_info, zv, r0 + || if (Z_REFCOUNTED_P(zv)) { + | ADDREF_CONST zv, r0 + || } | INC_OPLINE return 1; } | ZVAL_COPY_VALUE FP, var, FP, val.var, val_info, r0, eax, r1 + || if (val_type == IS_CV) { + | TRY_ADDREF val_info, ah, r1 + || } | INC_OPLINE return 1; @@ -1058,12 +1071,11 @@ static int zend_jit_send_val(dasm_State **Dst, zend_op *opline, zend_op_array *o if (opline->op1_type == IS_CONST) { zval *zv = RT_CONSTANT(op_array, opline->op1); - if (Z_REFCOUNTED_P(zv)) { - /* TODO: allow JIT for Z_ADDREF_P() ??? */ - goto fallback; - } | mov r0, EX->call | ZVAL_COPY_CONST r0, opline->result.var, -1, zv, r1 + || if (Z_REFCOUNTED_P(zv)) { + | ADDREF_CONST zv, r1 + || } | INC_OPLINE return 1; } From 8c3664f16b9818aae2836b216441e796d4c40749 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 8 Sep 2016 14:18:29 +0300 Subject: [PATCH 141/569] Allow JIT strategy for the whole script at once (just API, not implemeted yet) --- ext/opcache/jit/zend_jit.c | 26 ++++++++++++++------------ ext/opcache/jit/zend_jit.h | 11 ++++++++++- ext/opcache/jit/zend_jit_x86.dasc | 6 +++--- ext/opcache/zend_persist.c | 9 +++++++-- 4 files changed, 34 insertions(+), 18 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index e67dc05bd5c12..a24c308ae506c 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -29,14 +29,6 @@ #include "Optimizer/zend_inference.h" #include "Optimizer/zend_dump.h" - -#define ZEND_JIT_LEVEL_NONE 0 /* no JIT */ -#define ZEND_JIT_LEVEL_MINI 1 /* minimal JIT (subroutine threading) */ -#define ZEND_JIT_LEVEL_INLINE 2 /* selective inline threading */ -#define ZEND_JIT_LEVEL_FULL 3 /* optimized JIT based on Type-Inference */ - -#define ZEND_JIT_LEVEL ZEND_JIT_LEVEL_FULL - //#define CONTEXT_THREDED_JIT #define PREFER_MAP_32BIT //#define ZEND_RUNTIME_JIT @@ -652,7 +644,7 @@ static int zend_real_jit_func(zend_op_array *op_array, zend_script *script) goto jit_failure; } - if ((ZEND_JIT_LEVEL >= ZEND_JIT_LEVEL_FULL) + if ((ZEND_JIT_LEVEL >= ZEND_JIT_LEVEL_OPT_FUNC) && ssa.cfg.blocks && op_array->last_try_catch == 0 && !(op_array->fn_flags & ZEND_ACC_GENERATOR) @@ -764,7 +756,7 @@ static int zend_real_jit_func(zend_op_array *op_array, zend_script *script) for (i = ssa.cfg.blocks[b].start; i <= end; i++) { opline = op_array->opcodes + i; switch (opline->opcode) { -#if ZEND_JIT_LEVEL >= ZEND_JIT_LEVEL_FULL +#if ZEND_JIT_LEVEL >= ZEND_JIT_LEVEL_OPT_FUNC case ZEND_PRE_INC: case ZEND_PRE_DEC: if (!zend_jit_inc_dec(&dasm_state, opline, op_array, &ssa)) { @@ -1022,7 +1014,7 @@ static void ZEND_FASTCALL zend_runtime_jit(void) } #endif -ZEND_API int zend_jit(zend_op_array *op_array, zend_script *script) +ZEND_API int zend_jit_op_array(zend_op_array *op_array, zend_script *script) { #ifdef ZEND_RUNTIME_JIT zend_op *opline = op_array->opcodes; @@ -1040,6 +1032,11 @@ ZEND_API int zend_jit(zend_op_array *op_array, zend_script *script) #endif } +ZEND_API int zend_jit_script(zend_script *script) +{ + return FAILURE; +} + ZEND_API void zend_jit_unprotect(void) { #ifdef HAVE_MPROTECT @@ -1170,7 +1167,12 @@ ZEND_API void zend_jit_shutdown(void) #else /* HAVE_JIT */ -ZEND_API int zend_jit(zend_op_array *op_array, zend_script *script) +ZEND_API int zend_jit_op_array(zend_op_array *op_array, zend_script *script) +{ + return FAILURE; +} + +ZEND_API int zend_jit_script(zend_script *script) { return FAILURE; } diff --git a/ext/opcache/jit/zend_jit.h b/ext/opcache/jit/zend_jit.h index 1369211da5c7d..75311ed0454d3 100644 --- a/ext/opcache/jit/zend_jit.h +++ b/ext/opcache/jit/zend_jit.h @@ -21,6 +21,14 @@ #ifndef HAVE_JIT_H #define HAVE_JIT_H +#define ZEND_JIT_LEVEL_NONE 0 /* no JIT */ +#define ZEND_JIT_LEVEL_MINIMAL 1 /* minimal JIT (subroutine threading) */ +#define ZEND_JIT_LEVEL_INLINE 2 /* selective inline threading */ +#define ZEND_JIT_LEVEL_OPT_FUNC 3 /* optimized JIT based on Type-Inference */ +#define ZEND_JIT_LEVEL_OPT_SCRIPT 4 /* optimized JIT based on Type-Inference and inner-procedute analises */ + +#define ZEND_JIT_LEVEL ZEND_JIT_LEVEL_OPT_FUNC + #define ZEND_JIT_DEBUG_ASM (1<<0) #define ZEND_JIT_DEBUG_SSA (1<<1) @@ -29,7 +37,8 @@ #define ZEND_JIT_DEBUG_OPROFILE (1<<6) -ZEND_API int zend_jit(zend_op_array *op_array, zend_script *script); +ZEND_API int zend_jit_op_array(zend_op_array *op_array, zend_script *script); +ZEND_API int zend_jit_script(zend_script *script); ZEND_API void zend_jit_unprotect(void); ZEND_API void zend_jit_protect(void); ZEND_API int zend_jit_startup(size_t size); diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 2d66bf8e43689..4a891c57d4a64 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -72,7 +72,7 @@ static void* dasm_labels[zend_lb_MAX]; #define IS_32BIT(addr) (((uintptr_t)(addr)) <= 0x7fffffff) -#if ZEND_JIT_LEVEL >= ZEND_JIT_LEVEL_FULL +#if ZEND_JIT_LEVEL >= ZEND_JIT_LEVEL_OT_FUNC |.macro FP_OP, fp_ins, op_type, op ||if (op_type == IS_CONST) { @@ -637,7 +637,7 @@ static int zend_jit_new(dasm_State **Dst, zend_op *opline, uint32_t *opnum, zend } if (opline->extended_value == 0 && (opline+1)->opcode == ZEND_DO_FCALL) { zend_class_entry *ce = NULL; -#if ZEND_JIT_LEVEL >= ZEND_JIT_LEVEL_FULL +#if ZEND_JIT_LEVEL >= ZEND_JIT_LEVEL_OPT_FUNC if (ssa->ops && ssa->var_info) { zend_ssa_var_info *res_ssa = &ssa->var_info[ssa->ops[opline - op_array->opcodes].result_def]; if (res_ssa->ce && !res_ssa->is_instanceof) { @@ -667,7 +667,7 @@ static int zend_jit_new(dasm_State **Dst, zend_op *opline, uint32_t *opnum, zend return 1; } -#if ZEND_JIT_LEVEL >= ZEND_JIT_LEVEL_FULL +#if ZEND_JIT_LEVEL >= ZEND_JIT_LEVEL_OPT_FUNC static int zend_jit_inc_dec(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) { diff --git a/ext/opcache/zend_persist.c b/ext/opcache/zend_persist.c index dedc8a0aa856c..20a81755431d9 100644 --- a/ext/opcache/zend_persist.c +++ b/ext/opcache/zend_persist.c @@ -570,8 +570,9 @@ static void zend_persist_op_array_ex(zend_op_array *op_array, zend_persistent_sc ZCG(mem) = (void*)((char*)ZCG(mem) + ZEND_ALIGNED_SIZE(zend_extensions_op_array_persist(op_array, ZCG(mem)))); #ifdef HAVE_JIT - if (do_jit && ZCG(accel_directives).jit_buffer_size) { - zend_jit(op_array, ZCG(current_persistent_script) ? &ZCG(current_persistent_script)->script : NULL); + if (ZEND_JIT_LEVEL <= ZEND_JIT_LEVEL_OPT_FUNC && + do_jit && ZCG(accel_directives).jit_buffer_size) { + zend_jit_op_array(op_array, ZCG(current_persistent_script) ? &ZCG(current_persistent_script)->script : NULL); } #endif } @@ -867,6 +868,10 @@ zend_persistent_script *zend_accel_script_persist(zend_persistent_script *script ZCG(current_persistent_script) = NULL; #ifdef HAVE_JIT + if (ZEND_JIT_LEVEL >= ZEND_JIT_LEVEL_OPT_SCRIPT && + ZCG(accel_directives).jit_buffer_size) { + zend_jit_script(&script->script); + } zend_jit_protect(); #endif From ab70f3b472b1dba51958adcdf0328678005cc65c Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 8 Sep 2016 15:19:48 +0300 Subject: [PATCH 142/569] Enable optimization for the whole script --- ext/opcache/jit/zend_jit.c | 294 +++++++++++++++++++++++++------------ ext/opcache/jit/zend_jit.h | 2 +- 2 files changed, 199 insertions(+), 97 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index a24c308ae506c..96609c5d7605f 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -27,6 +27,7 @@ #include "Optimizer/zend_func_info.h" #include "Optimizer/zend_ssa.h" #include "Optimizer/zend_inference.h" +#include "Optimizer/zend_call_graph.h" #include "Optimizer/zend_dump.h" //#define CONTEXT_THREDED_JIT @@ -607,84 +608,85 @@ static int zend_may_throw(zend_op *opline, zend_op_array *op_array, zend_ssa *ss } } -static int zend_real_jit_func(zend_op_array *op_array, zend_script *script) +static int zend_jit_op_array_analyze1(zend_op_array *op_array, zend_script *script, zend_ssa *ssa, uint32_t *flags) { - uint32_t flags = 0; - zend_ssa ssa; - void *checkpoint; - int b, i, end; - zend_op *opline; - dasm_State* dasm_state = NULL; - void *handler; - - if (dasm_ptr == dasm_end) { + if (zend_build_cfg(&CG(arena), op_array, ZEND_CFG_STACKLESS | ZEND_CFG_RECV_ENTRY | ZEND_RT_CONSTANTS | ZEND_SSA_RC_INFERENCE, &ssa->cfg, flags) != SUCCESS) { return FAILURE; } - checkpoint = zend_arena_checkpoint(CG(arena)); - - /* Build SSA */ - memset(&ssa, 0, sizeof(zend_ssa)); - - if (zend_build_cfg(&CG(arena), op_array, ZEND_CFG_STACKLESS | ZEND_CFG_RECV_ENTRY | ZEND_RT_CONSTANTS | ZEND_SSA_RC_INFERENCE, &ssa.cfg, &flags) != SUCCESS) { - goto jit_failure; - } - - if (zend_cfg_build_predecessors(&CG(arena), &ssa.cfg) != SUCCESS) { - goto jit_failure; + if (zend_cfg_build_predecessors(&CG(arena), &ssa->cfg) != SUCCESS) { + return FAILURE; } /* Compute Dominators Tree */ - if (zend_cfg_compute_dominators_tree(op_array, &ssa.cfg) != SUCCESS) { - goto jit_failure; + if (zend_cfg_compute_dominators_tree(op_array, &ssa->cfg) != SUCCESS) { + return FAILURE; } /* Identify reducible and irreducible loops */ - if (zend_cfg_identify_loops(op_array, &ssa.cfg, &flags) != SUCCESS) { - goto jit_failure; + if (zend_cfg_identify_loops(op_array, &ssa->cfg, flags) != SUCCESS) { + return FAILURE; } if ((ZEND_JIT_LEVEL >= ZEND_JIT_LEVEL_OPT_FUNC) - && ssa.cfg.blocks + && ssa->cfg.blocks && op_array->last_try_catch == 0 && !(op_array->fn_flags & ZEND_ACC_GENERATOR) - && !(flags & ZEND_FUNC_INDIRECT_VAR_ACCESS)) { - if (zend_build_ssa(&CG(arena), script, op_array, ZEND_RT_CONSTANTS | ZEND_SSA_RC_INFERENCE, &ssa, &flags) != SUCCESS) { - goto jit_failure; + && !(*flags & ZEND_FUNC_INDIRECT_VAR_ACCESS)) { + if (zend_build_ssa(&CG(arena), script, op_array, ZEND_RT_CONSTANTS | ZEND_SSA_RC_INFERENCE, ssa, flags) != SUCCESS) { + return FAILURE; } - if (zend_ssa_compute_use_def_chains(&CG(arena), op_array, &ssa) != SUCCESS) { - goto jit_failure; + if (zend_ssa_compute_use_def_chains(&CG(arena), op_array, ssa) != SUCCESS) { + return FAILURE; } - if (zend_ssa_find_false_dependencies(op_array, &ssa) != SUCCESS) { - goto jit_failure; + if (zend_ssa_find_false_dependencies(op_array, ssa) != SUCCESS) { + return FAILURE; } - if (zend_ssa_find_sccs(op_array, &ssa) != SUCCESS){ - goto jit_failure; + if (zend_ssa_find_sccs(op_array, ssa) != SUCCESS){ + return FAILURE; } + } else { + ssa->rt_constants = 1; + } - if (zend_ssa_inference(&CG(arena), op_array, script, &ssa) != SUCCESS) { - goto jit_failure; - } + return SUCCESS; +} + +static int zend_jit_op_array_analyze2(zend_op_array *op_array, zend_script *script, zend_ssa *ssa, uint32_t *flags) +{ + if ((ZEND_JIT_LEVEL >= ZEND_JIT_LEVEL_OPT_FUNC) + && ssa->cfg.blocks + && op_array->last_try_catch == 0 + && !(op_array->fn_flags & ZEND_ACC_GENERATOR) + && !(*flags & ZEND_FUNC_INDIRECT_VAR_ACCESS)) { - if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_SSA) { - zend_dump_op_array(op_array, ZEND_DUMP_HIDE_UNREACHABLE|ZEND_DUMP_RC_INFERENCE|ZEND_DUMP_SSA|ZEND_DUMP_RT_CONSTANTS, "JIT", &ssa); + if (zend_ssa_inference(&CG(arena), op_array, script, ssa) != SUCCESS) { + return FAILURE; } - } else { - ssa.rt_constants = 1; } + return SUCCESS; +} + +static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) +{ + int b, i, end; + zend_op *opline; + dasm_State* dasm_state = NULL; + void *handler; + /* mark hidden branch targets */ - for (b = 0; b < ssa.cfg.blocks_count; b++) { - if (ssa.cfg.blocks[b].flags & ZEND_BB_REACHABLE && - ssa.cfg.blocks[b].len > 1) { + for (b = 0; b < ssa->cfg.blocks_count; b++) { + if (ssa->cfg.blocks[b].flags & ZEND_BB_REACHABLE && + ssa->cfg.blocks[b].len > 1) { - opline = op_array->opcodes + ssa.cfg.blocks[b].start + ssa.cfg.blocks[b].len - 1; + opline = op_array->opcodes + ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len - 1; if (opline->opcode == ZEND_DO_FCALL && (opline-1)->opcode == ZEND_NEW) { - ssa.cfg.blocks[ssa.cfg.blocks[b].successors[0]].flags |= ZEND_BB_TARGET; + ssa->cfg.blocks[ssa->cfg.blocks[b].successors[0]].flags |= ZEND_BB_TARGET; } } } @@ -693,28 +695,28 @@ static int zend_real_jit_func(zend_op_array *op_array, zend_script *script) dasm_setupglobal(&dasm_state, dasm_labels, zend_lb_MAX); dasm_setup(&dasm_state, dasm_actions); - dasm_growpc(&dasm_state, ssa.cfg.blocks_count * 2); + dasm_growpc(&dasm_state, ssa->cfg.blocks_count * 2); zend_jit_align_func(&dasm_state); - for (b = 0; b < ssa.cfg.blocks_count; b++) { - if ((ssa.cfg.blocks[b].flags & ZEND_BB_REACHABLE) == 0) { + for (b = 0; b < ssa->cfg.blocks_count; b++) { + if ((ssa->cfg.blocks[b].flags & ZEND_BB_REACHABLE) == 0) { continue; } #ifdef CONTEXT_THREDED_JIT - if (ssa.cfg.blocks[b].flags & ZEND_BB_START) { - zend_jit_label(&dasm_state, ssa.cfg.blocks_count + b); + if (ssa->cfg.blocks[b].flags & ZEND_BB_START) { + zend_jit_label(&dasm_state, ssa->cfg.blocks_count + b); zend_jit_prologue(&dasm_state); } #else - if (ssa.cfg.blocks[b].flags & ZEND_BB_ENTRY) { - if (ssa.cfg.blocks[b].flags & ZEND_BB_TARGET) { + if (ssa->cfg.blocks[b].flags & ZEND_BB_ENTRY) { + if (ssa->cfg.blocks[b].flags & ZEND_BB_TARGET) { zend_jit_jmp(&dasm_state, b); } - zend_jit_label(&dasm_state, ssa.cfg.blocks_count + b); + zend_jit_label(&dasm_state, ssa->cfg.blocks_count + b); zend_jit_prologue(&dasm_state); - } else if (ssa.cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_RECV_ENTRY)) { - opline = op_array->opcodes + ssa.cfg.blocks[b].start; - if (ssa.cfg.split_at_recv && opline->opcode == ZEND_RECV_INIT) { + } else if (ssa->cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_RECV_ENTRY)) { + opline = op_array->opcodes + ssa->cfg.blocks[b].start; + if (ssa->cfg.split_at_recv && opline->opcode == ZEND_RECV_INIT) { if (opline > op_array->opcodes && (opline-1)->opcode == ZEND_RECV_INIT) { /* repeatable opcode */ @@ -723,43 +725,43 @@ static int zend_real_jit_func(zend_op_array *op_array, zend_script *script) for (i = 0; (opline+i)->opcode == ZEND_RECV_INIT; i++) { } zend_jit_jmp(&dasm_state, b + i); - zend_jit_label(&dasm_state, ssa.cfg.blocks_count + b); + zend_jit_label(&dasm_state, ssa->cfg.blocks_count + b); for (i = 1; (opline+i)->opcode == ZEND_RECV_INIT; i++) { - zend_jit_label(&dasm_state, ssa.cfg.blocks_count + b + i); + zend_jit_label(&dasm_state, ssa->cfg.blocks_count + b + i); } zend_jit_prologue(&dasm_state); } } else { - if (ssa.cfg.blocks[b].flags & (ZEND_BB_TARGET|ZEND_BB_RECV_ENTRY)) { + if (ssa->cfg.blocks[b].flags & (ZEND_BB_TARGET|ZEND_BB_RECV_ENTRY)) { zend_jit_jmp(&dasm_state, b); } - zend_jit_label(&dasm_state, ssa.cfg.blocks_count + b); + zend_jit_label(&dasm_state, ssa->cfg.blocks_count + b); zend_jit_prologue(&dasm_state); } } #endif zend_jit_label(&dasm_state, b); - if (ssa.cfg.blocks[b].flags & ZEND_BB_TARGET) { - if (!zend_jit_set_opline(&dasm_state, op_array->opcodes + ssa.cfg.blocks[b].start)) { + if (ssa->cfg.blocks[b].flags & ZEND_BB_TARGET) { + if (!zend_jit_set_opline(&dasm_state, op_array->opcodes + ssa->cfg.blocks[b].start)) { goto jit_failure; } } - if (ssa.cfg.blocks[b].flags & ZEND_BB_LOOP_HEADER) { + if (ssa->cfg.blocks[b].flags & ZEND_BB_LOOP_HEADER) { if (!zend_jit_check_timeout(&dasm_state)) { goto jit_failure; } } - if (!ssa.cfg.blocks[b].len) { + if (!ssa->cfg.blocks[b].len) { continue; } - end = ssa.cfg.blocks[b].start + ssa.cfg.blocks[b].len - 1; - for (i = ssa.cfg.blocks[b].start; i <= end; i++) { + end = ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len - 1; + for (i = ssa->cfg.blocks[b].start; i <= end; i++) { opline = op_array->opcodes + i; switch (opline->opcode) { #if ZEND_JIT_LEVEL >= ZEND_JIT_LEVEL_OPT_FUNC case ZEND_PRE_INC: case ZEND_PRE_DEC: - if (!zend_jit_inc_dec(&dasm_state, opline, op_array, &ssa)) { + if (!zend_jit_inc_dec(&dasm_state, opline, op_array, ssa)) { goto jit_failure; } break; @@ -767,27 +769,27 @@ static int zend_real_jit_func(zend_op_array *op_array, zend_script *script) case ZEND_SUB: case ZEND_MUL: // case ZEND_DIV: // TODO: check for division by zero ??? - if (!zend_jit_math(&dasm_state, opline, op_array, &ssa)) { + if (!zend_jit_math(&dasm_state, opline, op_array, ssa)) { goto jit_failure; } break; case ZEND_ASSIGN: - if (!zend_jit_assign(&dasm_state, opline, op_array, &ssa)) { + if (!zend_jit_assign(&dasm_state, opline, op_array, ssa)) { goto jit_failure; } break; case ZEND_QM_ASSIGN: - if (!zend_jit_qm_assign(&dasm_state, opline, op_array, &ssa)) { + if (!zend_jit_qm_assign(&dasm_state, opline, op_array, ssa)) { goto jit_failure; } break; case ZEND_SEND_VAL: - if (!zend_jit_send_val(&dasm_state, opline, op_array, &ssa)) { + if (!zend_jit_send_val(&dasm_state, opline, op_array, ssa)) { goto jit_failure; } break; case ZEND_SEND_VAR: - if (!zend_jit_send_var(&dasm_state, opline, op_array, &ssa)) { + if (!zend_jit_send_var(&dasm_state, opline, op_array, ssa)) { goto jit_failure; } break; @@ -795,19 +797,19 @@ static int zend_real_jit_func(zend_op_array *op_array, zend_script *script) case ZEND_IS_NOT_EQUAL: case ZEND_IS_SMALLER: case ZEND_IS_SMALLER_OR_EQUAL: - if (!zend_jit_cmp(&dasm_state, opline, b, &i, op_array, &ssa)) { + if (!zend_jit_cmp(&dasm_state, opline, b, &i, op_array, ssa)) { goto jit_failure; } break; case ZEND_TYPE_CHECK: - if (!zend_jit_type_check(&dasm_state, opline, b, &i, op_array, &ssa)) { + if (!zend_jit_type_check(&dasm_state, opline, b, &i, op_array, ssa)) { goto jit_failure; } break; #endif case ZEND_RECV_INIT: - if (ssa.cfg.split_at_recv) { - if (!zend_jit_handler(&dasm_state, opline, zend_may_throw(opline, op_array, &ssa))) { + if (ssa->cfg.split_at_recv) { + if (!zend_jit_handler(&dasm_state, opline, zend_may_throw(opline, op_array, ssa))) { goto jit_failure; } break; @@ -816,7 +818,7 @@ static int zend_real_jit_func(zend_op_array *op_array, zend_script *script) case ZEND_BIND_GLOBAL: if (opline->opcode != op_array->opcodes[i+1].opcode) { /* repeatable opcodes */ - if (!zend_jit_handler(&dasm_state, opline, zend_may_throw(opline, op_array, &ssa))) { + if (!zend_jit_handler(&dasm_state, opline, zend_may_throw(opline, op_array, ssa))) { goto jit_failure; } } @@ -836,7 +838,7 @@ static int zend_real_jit_func(zend_op_array *op_array, zend_script *script) case ZEND_OP_DATA: break; case ZEND_JMP: - if (!zend_jit_jmp(&dasm_state, ssa.cfg.blocks[b].successors[0])) { + if (!zend_jit_jmp(&dasm_state, ssa->cfg.blocks[b].successors[0])) { goto jit_failure; } break; @@ -873,19 +875,19 @@ static int zend_real_jit_func(zend_op_array *op_array, zend_script *script) (opline-1)->opcode == ZEND_IS_SMALLER_OR_EQUAL || (opline-1)->opcode == ZEND_CASE) { - uint32_t t1 = _ssa_op1_info(op_array, &ssa, (opline-1)); - uint32_t t2 = _ssa_op2_info(op_array, &ssa, (opline-1)); + uint32_t t1 = _ssa_op1_info(op_array, ssa, (opline-1)); + uint32_t t2 = _ssa_op2_info(op_array, ssa, (opline-1)); if ((t1 & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) || (t2 & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { /* might be smart branch */ - if (!zend_jit_smart_branch(&dasm_state, opline, (b + 1), ssa.cfg.blocks[b].successors[0])) { + if (!zend_jit_smart_branch(&dasm_state, opline, (b + 1), ssa->cfg.blocks[b].successors[0])) { goto jit_failure; } /* break missing intentionally */ } else { /* smart branch */ - if (!zend_jit_cond_jmp(&dasm_state, opline + 1, ssa.cfg.blocks[b].successors[0])) { + if (!zend_jit_cond_jmp(&dasm_state, opline + 1, ssa->cfg.blocks[b].successors[0])) { goto jit_failure; } break; @@ -900,7 +902,7 @@ static int zend_real_jit_func(zend_op_array *op_array, zend_script *script) (opline-1)->opcode == ZEND_TYPE_CHECK || (opline-1)->opcode == ZEND_DEFINED) { /* smart branch */ - if (!zend_jit_cond_jmp(&dasm_state, opline + 1, ssa.cfg.blocks[b].successors[0])) { + if (!zend_jit_cond_jmp(&dasm_state, opline + 1, ssa->cfg.blocks[b].successors[0])) { goto jit_failure; } break; @@ -914,20 +916,20 @@ static int zend_real_jit_func(zend_op_array *op_array, zend_script *script) case ZEND_FE_RESET_R: case ZEND_FE_RESET_RW: case ZEND_ASSERT_CHECK: - if (!zend_jit_handler(&dasm_state, opline, zend_may_throw(opline, op_array, &ssa)) || - !zend_jit_cond_jmp(&dasm_state, opline + 1, ssa.cfg.blocks[b].successors[0])) { + if (!zend_jit_handler(&dasm_state, opline, zend_may_throw(opline, op_array, ssa)) || + !zend_jit_cond_jmp(&dasm_state, opline + 1, ssa->cfg.blocks[b].successors[0])) { goto jit_failure; } break; case ZEND_NEW: - if (!zend_jit_new(&dasm_state, opline, &i, op_array, &ssa)) { + if (!zend_jit_new(&dasm_state, opline, &i, op_array, ssa)) { goto jit_failure; } break; case ZEND_JMPZNZ: - if (!zend_jit_handler(&dasm_state, opline, zend_may_throw(opline, op_array, &ssa)) || - !zend_jit_cond_jmp(&dasm_state, OP_JMP_ADDR(opline, opline->op2), ssa.cfg.blocks[b].successors[1]) || - !zend_jit_jmp(&dasm_state, ssa.cfg.blocks[b].successors[0])) { + if (!zend_jit_handler(&dasm_state, opline, zend_may_throw(opline, op_array, ssa)) || + !zend_jit_cond_jmp(&dasm_state, OP_JMP_ADDR(opline, opline->op2), ssa->cfg.blocks[b].successors[1]) || + !zend_jit_jmp(&dasm_state, ssa->cfg.blocks[b].successors[0])) { goto jit_failure; } break; @@ -935,32 +937,69 @@ static int zend_real_jit_func(zend_op_array *op_array, zend_script *script) case ZEND_FE_FETCH_RW: case ZEND_DECLARE_ANON_CLASS: case ZEND_DECLARE_ANON_INHERITED_CLASS: - if (!zend_jit_handler(&dasm_state, opline, zend_may_throw(opline, op_array, &ssa)) || - !zend_jit_cond_jmp(&dasm_state, opline + 1, ssa.cfg.blocks[b].successors[0])) { + if (!zend_jit_handler(&dasm_state, opline, zend_may_throw(opline, op_array, ssa)) || + !zend_jit_cond_jmp(&dasm_state, opline + 1, ssa->cfg.blocks[b].successors[0])) { goto jit_failure; } break; default: - if (!zend_jit_handler(&dasm_state, opline, zend_may_throw(opline, op_array, &ssa))) { + if (!zend_jit_handler(&dasm_state, opline, zend_may_throw(opline, op_array, ssa))) { goto jit_failure; } } } } - handler = dasm_link_and_encode(&dasm_state, op_array, &ssa.cfg, NULL); + handler = dasm_link_and_encode(&dasm_state, op_array, &ssa->cfg, NULL); if (!handler) { goto jit_failure; } dasm_free(&dasm_state); - zend_arena_release(&CG(arena), checkpoint); return SUCCESS; jit_failure: if (dasm_state) { dasm_free(&dasm_state); } + return FAILURE; +} + +static int zend_real_jit_func(zend_op_array *op_array, zend_script *script) +{ + uint32_t flags = 0; + zend_ssa ssa; + void *checkpoint; + + if (dasm_ptr == dasm_end) { + return FAILURE; + } + + checkpoint = zend_arena_checkpoint(CG(arena)); + + /* Build SSA */ + memset(&ssa, 0, sizeof(zend_ssa)); + + if (zend_jit_op_array_analyze1(op_array, script, &ssa, &flags) != SUCCESS) { + goto jit_failure; + } + + if (zend_jit_op_array_analyze2(op_array, script, &ssa, &flags) != SUCCESS) { + goto jit_failure; + } + + if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_SSA) { + zend_dump_op_array(op_array, ZEND_DUMP_HIDE_UNREACHABLE|ZEND_DUMP_RC_INFERENCE|ZEND_DUMP_SSA|ZEND_DUMP_RT_CONSTANTS, "JIT", &ssa); + } + + if (zend_jit(op_array, &ssa) != SUCCESS) { + goto jit_failure; + } + + zend_arena_release(&CG(arena), checkpoint); + return SUCCESS; + +jit_failure: zend_arena_release(&CG(arena), checkpoint); return FAILURE; } @@ -1034,6 +1073,69 @@ ZEND_API int zend_jit_op_array(zend_op_array *op_array, zend_script *script) ZEND_API int zend_jit_script(zend_script *script) { + void *checkpoint; + zend_call_graph call_graph; + zend_func_info *info; + int i; + + if (dasm_ptr == dasm_end) { + return FAILURE; + } + + checkpoint = zend_arena_checkpoint(CG(arena)); + + call_graph.op_arrays_count = 0; + if (zend_build_call_graph(&CG(arena), script, ZEND_RT_CONSTANTS, &call_graph) != SUCCESS) { + goto jit_failure; + } + + for (i = 0; i < call_graph.op_arrays_count; i++) { + info = ZEND_FUNC_INFO(call_graph.op_arrays[i]); + if (info) { + if (zend_jit_op_array_analyze1(call_graph.op_arrays[i], script, &info->ssa, &info->flags) != SUCCESS) { + goto jit_failure; + } + } + } + + for (i = 0; i < call_graph.op_arrays_count; i++) { + info = ZEND_FUNC_INFO(call_graph.op_arrays[i]); + if (info) { + if (zend_jit_op_array_analyze2(call_graph.op_arrays[i], script, &info->ssa, &info->flags) != SUCCESS) { + goto jit_failure; + } + } + } + + if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_SSA) { + for (i = 0; i < call_graph.op_arrays_count; i++) { + info = ZEND_FUNC_INFO(call_graph.op_arrays[i]); + if (info) { + zend_dump_op_array(call_graph.op_arrays[i], ZEND_DUMP_HIDE_UNREACHABLE|ZEND_DUMP_RC_INFERENCE|ZEND_DUMP_SSA|ZEND_DUMP_RT_CONSTANTS, "JIT", &info->ssa); + } + } + } + + for (i = 0; i < call_graph.op_arrays_count; i++) { + info = ZEND_FUNC_INFO(call_graph.op_arrays[i]); + if (info) { + if (zend_jit(call_graph.op_arrays[i], &info->ssa) != SUCCESS) { + goto jit_failure; + } + } + } + + for (i = 0; i < call_graph.op_arrays_count; i++) { + ZEND_SET_FUNC_INFO(call_graph.op_arrays[i], NULL); + } + zend_arena_release(&CG(arena), checkpoint); + return SUCCESS; + +jit_failure: + for (i = 0; i < call_graph.op_arrays_count; i++) { + ZEND_SET_FUNC_INFO(call_graph.op_arrays[i], NULL); + } + zend_arena_release(&CG(arena), checkpoint); return FAILURE; } diff --git a/ext/opcache/jit/zend_jit.h b/ext/opcache/jit/zend_jit.h index 75311ed0454d3..3e1bd920b7284 100644 --- a/ext/opcache/jit/zend_jit.h +++ b/ext/opcache/jit/zend_jit.h @@ -27,7 +27,7 @@ #define ZEND_JIT_LEVEL_OPT_FUNC 3 /* optimized JIT based on Type-Inference */ #define ZEND_JIT_LEVEL_OPT_SCRIPT 4 /* optimized JIT based on Type-Inference and inner-procedute analises */ -#define ZEND_JIT_LEVEL ZEND_JIT_LEVEL_OPT_FUNC +#define ZEND_JIT_LEVEL ZEND_JIT_LEVEL_OPT_SCRIPT #define ZEND_JIT_DEBUG_ASM (1<<0) #define ZEND_JIT_DEBUG_SSA (1<<1) From 58ba693de95abb1e156fd86dcc490c15b0f67337 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 8 Sep 2016 15:46:32 +0300 Subject: [PATCH 143/569] Fixed incorrect Jcc conditions --- ext/opcache/jit/zend_jit_x86.dasc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 4a891c57d4a64..336c7048c935c 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -883,11 +883,11 @@ static int zend_jit_cmp(dasm_State **Dst, zend_op *opline, int b, int *opnum, ze | add eax, 2 break; case ZEND_IS_SMALLER: - | setg al + | setl al | add eax, 2 break; case ZEND_IS_SMALLER_OR_EQUAL: - | setge al + | setle al | add eax, 2 break; } From 148d69151c3fc91a26f01e85858ab15d18e5e981 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 8 Sep 2016 19:27:34 +0300 Subject: [PATCH 144/569] Use more DynAsm macros. JIT for INIT_FCALL (incomplete). --- ext/opcache/jit/zend_jit.c | 5 + ext/opcache/jit/zend_jit_x86.dasc | 263 ++++++++++++++++++------------ 2 files changed, 168 insertions(+), 100 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 96609c5d7605f..c1e7c4bc2c141 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -783,6 +783,11 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) goto jit_failure; } break; +// case ZEND_INIT_FCALL: +// if (!zend_jit_init_fcall(&dasm_state, opline, op_array)) { +// goto jit_failure; +// } +// break; case ZEND_SEND_VAL: if (!zend_jit_send_val(&dasm_state, opline, op_array, ssa)) { goto jit_failure; diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 336c7048c935c..828f6e24cab55 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -23,33 +23,37 @@ |.endif |.if X64 - |.define FP, r14 - |.define IP, r15 - |.define IPl, r15d - |.define CARG1, rdi // x64/POSIX C call arguments. - |.define CARG2, rsi - |.define CARG3, rdx - |.define CARG4, rcx - |.define CARG5, r8 - |.define CARG6, r9 - |.define CARG1d, edi - |.define CARG2d, esi - |.define CARG3d, edx - |.define CARG4d, ecx - |.define CARG5d, r8d - |.define CARG6d, r9d - |.define FCARG1, CARG1d // Simulate x86 fastcall. - |.define FCARG2, CARG2d - |.define SPAD, 8 - |.define SSE, 1 + |.define FP, r14 + |.define IP, r15 + |.define IPl, r15d + |.define CARG1, rdi // x64/POSIX C call arguments. + |.define CARG2, rsi + |.define CARG3, rdx + |.define CARG4, rcx + |.define CARG5, r8 + |.define CARG6, r9 + |.define CARG1d, edi + |.define CARG2d, esi + |.define CARG3d, edx + |.define CARG4d, ecx + |.define CARG5d, r8d + |.define CARG6d, r9d + |.define FCARG1a, CARG1 // Simulate x86 fastcall. + |.define FCARG2a, CARG2 + |.define FCARG1d, CARG1d + |.define FCARG2d, CARG2d + |.define SPAD, 8 + |.define SSE, 1 |.else - |.define FP, esi - |.define IP, edi - |.define IPl, edi - |.define FCARG1, ecx // x86 fastcall arguments. - |.define FCARG2, edx - |.define SPAD, 12 - |.define SSE, 1 + |.define FP, esi + |.define IP, edi + |.define IPl, edi + |.define FCARG1a, ecx // x86 fastcall arguments. + |.define FCARG2a, edx + |.define FCARG1d, ecx + |.define FCARG2d, edx + |.define SPAD, 12 + |.define SSE, 1 |.endif |.type EX, zend_execute_data, FP @@ -63,11 +67,41 @@ static void* dasm_labels[zend_lb_MAX]; |.section code, cold_codes |.macro LOAD_ADDR, reg, addr +|.if X64 ||if (((ptrdiff_t)addr) <= 0x7fffffff) { | mov reg, ((ptrdiff_t)addr) // 0x48 0xc7 0xc0 || } else { | mov64 reg, ((ptrdiff_t)addr) // 0x48 0xb8 ||} +|.else +| mov reg, addr +|.endif +|.endmacro + +|.macro EXT_CALL, func, tmp_reg +|.if X64 +||if (IS_32BIT(dasm_end) && IS_32BIT(func)) { +| call qword &func +||} else { +| LOAD_ADDR tmp_reg, func +| call tmp_reg +||} +|.else +| call dword &func +|.endif +|.endmacro + +|.macro EXT_JMP, func, tmp_reg +|.if X64 +||if (IS_32BIT(dasm_end) && IS_32BIT(func)) { +| jmp qword &func +||} else { +| LOAD_ADDR tmp_reg, func +| jmp tmp_reg +||} +|.else +| jmp dword &func +|.endif |.endmacro #define IS_32BIT(addr) (((uintptr_t)(addr)) <= 0x7fffffff) @@ -424,16 +458,10 @@ static int zend_jit_interrupt_handler_stub(dasm_State **Dst) | //zend_timeout(0); |.if X64 | xor CARG1d, CARG1d - if (IS_32BIT(dasm_end) && IS_32BIT(zend_timeout)) { - | call qword &zend_timeout - } else { - | LOAD_ADDR rax, zend_timeout - | call rax - } |.else | push 0 - | call dword &zend_timeout |.endif + | EXT_CALL zend_timeout, r0 |1: | //} else if (zend_interrupt_function) { if (zend_interrupt_function) { @@ -441,18 +469,11 @@ static int zend_jit_interrupt_handler_stub(dasm_State **Dst) | mov EX->opline, IP | //zend_interrupt_function(execute_data); |.if X64 - if (IS_32BIT(dasm_end) && IS_32BIT(zend_interrupt_function)) { - | mov CARG1, FP - | call qword &zend_interrupt_function - } else { - | LOAD_ADDR rax, zend_interrupt_function | mov CARG1, FP - | call rax - } |.else | push FP - | call dword &zend_interrupt_function |.endif + | EXT_CALL zend_interrupt_function, r0 | //ZEND_VM_ENTER(); | //execute_data = EG(current_execute_data); | mov FP, aword [&EG(current_execute_data)] @@ -471,16 +492,7 @@ static int zend_jit_exception_handler_stub(dasm_State **Dst) { |->exception_handler: | add r4, SPAD // stack alignment - |.if X64 - if (IS_32BIT(dasm_end) && IS_32BIT(EG(exception_op)->handler)) { - | jmp qword &(EG(exception_op)->handler) - } else { - | LOAD_ADDR rax, EG(exception_op)->handler - | jmp rax - } - |.else - | jmp dword &(EG(exception_op)->handler) - |.endif + | EXT_JMP EG(exception_op)->handler, r0 return 1; } @@ -524,18 +536,7 @@ static int zend_jit_check_exception(dasm_State **Dst) static int zend_jit_handler(dasm_State **Dst, zend_op *opline, int may_throw) { - const void *handler = opline->handler; - - |.if X64 - if (IS_32BIT(dasm_end) && IS_32BIT(handler)) { - | call qword &handler - } else { - | LOAD_ADDR rax, handler - | call rax - } - |.else - | call dword &handler - |.endif + | EXT_CALL opline->handler, r0 if (may_throw) { zend_jit_check_exception(Dst); } @@ -544,19 +545,8 @@ static int zend_jit_handler(dasm_State **Dst, zend_op *opline, int may_throw) static int zend_jit_tail_handler(dasm_State **Dst, zend_op *opline) { - const void *handler = opline->handler; - | add r4, SPAD // stack alignment - |.if X64 - if (IS_32BIT(dasm_end) && IS_32BIT(handler)) { - | jmp qword &handler - } else { - | LOAD_ADDR rax, handler - | jmp rax - } - |.else - | jmp dword &handler - |.endif + | EXT_JMP opline->handler, r0 return 1; } @@ -568,11 +558,7 @@ static int zend_jit_skip_handler(dasm_State **Dst, uint32_t skip) static int zend_jit_set_opline(dasm_State **Dst, zend_op *target_opline) { - |.if X64 | LOAD_ADDR IP, target_opline - |.else - | mov IP, target_opline - |.endif return 1; } @@ -720,30 +706,12 @@ static int zend_jit_inc_dec(dasm_State **Dst, zend_op *opline, zend_op_array *op if (op1_info & (MAY_BE_ANY-MAY_BE_LONG)) { |.cold_codes |2: - |.if X64 - | lea CARG1, [FP + opline->op1.var] - || if (IS_32BIT(dasm_end) && IS_32BIT(increment_function)) { - || if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { - | call qword &increment_function - || } else { - | call qword &decrement_function - || } - || } else { - || if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { - | LOAD_ADDR rax, increment_function - || } else { - | LOAD_ADDR rax, decrement_function - || } - | call rax - || } - |.else - | lea FCARG1, [FP + opline->op1.var] + | lea FCARG1a, [FP + opline->op1.var] || if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { - | call dword &increment_function + | EXT_CALL increment_function, r0 || } else { - | call dword &decrement_function + | EXT_CALL decrement_function, r0 || } - |.endif | jmp >3 |.code } @@ -1064,6 +1032,101 @@ fallback: return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); } +static int zend_jit_push_stack_frame(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_function *func) +{ + if (!func) { + assert(0); + } + + uint32_t used_stack = zend_vm_calc_used_stack(opline->extended_value, func); + + | // if (UNEXPECTED(used_stack > (size_t)(((char*)EG(vm_stack_end)) - (char*)call))) { + | LOAD_ADDR r1, EG(vm_stack_top) + | LOAD_ADDR r2, EG(vm_stack_end) + | sub r2,r1 + | cmp r2, used_stack + | jz >1 // TODO proper condition + // EG(vm_stack_top) = (zval*)((char*)call + used_stack); + | // ??? + | // zend_vm_init_call_frame(call, call_info, func, num_args, called_scope, object); + | //??? + |.cold_codes + |1: + | //??? + | jmp >2 + |.code + |2: + + return 1; +} + +static int zend_jit_init_fcall(dasm_State **Dst, zend_op *opline, zend_op_array *op_array) +{ + zend_func_info *info = ZEND_FUNC_INFO(op_array); + zend_function *func = NULL; + + if (info) { + zend_call_info *call_info = info->callee_info; + + while (call_info && call_info->caller_init_opline != opline) { + call_info = call_info->next_callee; + } + if (call_info && call_info->callee_func) { + func = call_info->callee_func; + } + } + if (func && func->type == ZEND_INTERNAL_FUNCTION) { + | LOAD_ADDR r0, func + } else if (func && op_array == &func->op_array) { + /* recursive call */ + | mov r0, EX->func + } else { + zval *zv = RT_CONSTANT(op_array, opline->op2); + + | // if (CACHED_PTR(Z_CACHE_SLOT_P(fname))) { + | mov r0, EX->run_time_cache + | mov r0, aword [r0 + Z_CACHE_SLOT_P(zv)] + | test r0, r0 + | jz >1 + |.cold_codes + |1: + | // if ((func = zend_hash_find(EG(function_table), Z_STR_P(fname))) == NULL) { + | LOAD_ADDR FCARG1a, Z_STR_P(zv); + | EXT_CALL zend_hash_find, r0 + | test r0, r0 + | jz >2 + | mov r0, aword [r0] + | jmp >3 + |2: + | // SAVE_OPLINE(); + | mov EX->opline, IP + | // zend_throw_error(NULL, "Call to undefined function %s()", Z_STRVAL_P(fname)); + |.if X64 + | xor CARG1, CARG1 + | LOAD_ADDR CARG2, "Call to undefined function %s()" + | LOAD_ADDR CARG3, Z_STRVAL_P(zv) + |.else + | push 0 + | push "Call to undefined function %s()" + | push Z_STRVAL_P(zv) + |.endif + | EXT_CALL zend_throw_error, r0 + | jmp ->exception_handler + |.code + |3: + } + if (!zend_jit_push_stack_frame(Dst, opline, op_array, func)) { + return 0; + } + | // call->prev_execute_data = EX(call); + | mov r1, EX->call + | mov EX:r0->prev_execute_data, r1 + | // EX(call) = call; + | mov EX->call, r0 + + return 1; +} + static int zend_jit_send_val(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) { uint32_t op1_info; From b7f108ecae3ce7bc832417bd9d56b0accd04e837 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Fri, 9 Sep 2016 13:16:35 +0800 Subject: [PATCH 145/569] Only increase opline if it's really needed --- ext/opcache/jit/zend_jit.c | 40 ++++++++++++++---- ext/opcache/jit/zend_jit_x86.dasc | 68 ++++++++++++++++++++----------- 2 files changed, 77 insertions(+), 31 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index c1e7c4bc2c141..d79a831d0e893 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -671,6 +671,22 @@ static int zend_jit_op_array_analyze2(zend_op_array *op_array, zend_script *scri return SUCCESS; } +static int zend_need_inc_op(zend_op_array *op_array, zend_op *opline, int b, zend_ssa *ssa) { + zend_op *next_opline = opline + 1; + if (next_opline->opcode == ZEND_JMP || + next_opline->opcode == ZEND_JMPZNZ) { + return 0; + } else if (opline == (op_array->opcodes + ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len - 1)) { + if ((ssa->cfg.blocks[b].successors[0] >= 0 && + (ssa->cfg.blocks[ssa->cfg.blocks[b].successors[0]].flags & ZEND_BB_TARGET)) && + (ssa->cfg.blocks[b].successors[1] < 0 || + (ssa->cfg.blocks[ssa->cfg.blocks[b].successors[1]].flags & ZEND_BB_TARGET))) { + return 0; + } + } + return 1; +} + static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) { int b, i, end; @@ -761,7 +777,8 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) #if ZEND_JIT_LEVEL >= ZEND_JIT_LEVEL_OPT_FUNC case ZEND_PRE_INC: case ZEND_PRE_DEC: - if (!zend_jit_inc_dec(&dasm_state, opline, op_array, ssa)) { + if (!zend_jit_inc_dec(&dasm_state, + opline, op_array, ssa, zend_need_inc_op(op_array, opline, b, ssa))) { goto jit_failure; } break; @@ -769,17 +786,20 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) case ZEND_SUB: case ZEND_MUL: // case ZEND_DIV: // TODO: check for division by zero ??? - if (!zend_jit_math(&dasm_state, opline, op_array, ssa)) { + if (!zend_jit_math(&dasm_state, + opline, op_array, ssa, zend_need_inc_op(op_array, opline, b, ssa))) { goto jit_failure; } break; case ZEND_ASSIGN: - if (!zend_jit_assign(&dasm_state, opline, op_array, ssa)) { + if (!zend_jit_assign(&dasm_state, + opline, op_array, ssa, zend_need_inc_op(op_array, opline, b, ssa))) { goto jit_failure; } break; case ZEND_QM_ASSIGN: - if (!zend_jit_qm_assign(&dasm_state, opline, op_array, ssa)) { + if (!zend_jit_qm_assign(&dasm_state, + opline, op_array, ssa, zend_need_inc_op(op_array, opline, b, ssa))) { goto jit_failure; } break; @@ -789,12 +809,14 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) // } // break; case ZEND_SEND_VAL: - if (!zend_jit_send_val(&dasm_state, opline, op_array, ssa)) { + if (!zend_jit_send_val(&dasm_state, + opline, op_array, ssa, zend_need_inc_op(op_array, opline, b, ssa))) { goto jit_failure; } break; case ZEND_SEND_VAR: - if (!zend_jit_send_var(&dasm_state, opline, op_array, ssa)) { + if (!zend_jit_send_var(&dasm_state, + opline, op_array, ssa, zend_need_inc_op(op_array, opline, b, ssa))) { goto jit_failure; } break; @@ -802,12 +824,14 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) case ZEND_IS_NOT_EQUAL: case ZEND_IS_SMALLER: case ZEND_IS_SMALLER_OR_EQUAL: - if (!zend_jit_cmp(&dasm_state, opline, b, &i, op_array, ssa)) { + if (!zend_jit_cmp(&dasm_state, + opline, b, &i, op_array, ssa, zend_need_inc_op(op_array, opline, b, ssa))) { goto jit_failure; } break; case ZEND_TYPE_CHECK: - if (!zend_jit_type_check(&dasm_state, opline, b, &i, op_array, ssa)) { + if (!zend_jit_type_check(&dasm_state, + opline, b, &i, op_array, ssa, zend_need_inc_op(op_array, opline, b, ssa))) { goto jit_failure; } break; diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 828f6e24cab55..1bc8d2b181807 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -655,7 +655,7 @@ static int zend_jit_new(dasm_State **Dst, zend_op *opline, uint32_t *opnum, zend #if ZEND_JIT_LEVEL >= ZEND_JIT_LEVEL_OPT_FUNC -static int zend_jit_inc_dec(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) +static int zend_jit_inc_dec(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, int inc_op) { uint32_t op1_info, op1_def_info; @@ -716,7 +716,9 @@ static int zend_jit_inc_dec(dasm_State **Dst, zend_op *opline, zend_op_array *op |.code } |3: - | INC_OPLINE + if (inc_op) { + | INC_OPLINE + } return 1; fallback: @@ -724,7 +726,7 @@ fallback: return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); } -static int zend_jit_math(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) +static int zend_jit_math(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, int inc_op) { uint32_t op1_info, op2_info, res_info; @@ -773,7 +775,9 @@ static int zend_jit_math(dasm_State **Dst, zend_op *opline, zend_op_array *op_ar } else { goto fallback; } - | INC_OPLINE + if (inc_op) { + | INC_OPLINE + } return 1; @@ -782,7 +786,7 @@ fallback: return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); } -static int zend_jit_cmp(dasm_State **Dst, zend_op *opline, int b, int *opnum, zend_op_array *op_array, zend_ssa *ssa) +static int zend_jit_cmp(dasm_State **Dst, zend_op *opline, int b, int *opnum, zend_op_array *op_array, zend_ssa *ssa, int inc_op) { uint32_t op1_info, op2_info; unsigned int target_label; @@ -816,7 +820,7 @@ static int zend_jit_cmp(dasm_State **Dst, zend_op *opline, int b, int *opnum, ze | jge => target_label break; } - if (!(ssa->cfg.blocks[ssa->cfg.blocks[b].successors[1]].flags & ZEND_BB_TARGET)) { + if (inc_op) { | add IP, sizeof(zend_op) * 2 } } else if ((opline+1)->opcode == ZEND_JMPNZ && @@ -860,7 +864,9 @@ static int zend_jit_cmp(dasm_State **Dst, zend_op *opline, int b, int *opnum, ze break; } | mov dword [FP + opline->result.var + 8], eax - | INC_OPLINE + if (inc_op) { + | INC_OPLINE + } } } else if (((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) && ((op2_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE)) { @@ -953,7 +959,9 @@ static int zend_jit_cmp(dasm_State **Dst, zend_op *opline, int b, int *opnum, ze break; } | mov dword [FP + opline->result.var + 8], eax - | INC_OPLINE + if (inc_op) { + | INC_OPLINE + } } } else { goto fallback; @@ -966,7 +974,7 @@ fallback: return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); } -static int zend_jit_simple_assign(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, uint32_t var, uint32_t var_info, zend_uchar val_type, znode_op val, uint32_t val_info) +static int zend_jit_simple_assign(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, uint32_t var, uint32_t var_info, zend_uchar val_type, znode_op val, uint32_t val_info, int inc_op) { if (val_info & (MAY_BE_UNDEF|MAY_BE_REF)) { /* TODO: Support for references ??? */ @@ -977,7 +985,9 @@ static int zend_jit_simple_assign(dasm_State **Dst, zend_op *opline, zend_op_arr || if (Z_REFCOUNTED_P(zv)) { | ADDREF_CONST zv, r0 || } - | INC_OPLINE + if (inc_op) { + | INC_OPLINE + } return 1; } @@ -985,7 +995,9 @@ static int zend_jit_simple_assign(dasm_State **Dst, zend_op *opline, zend_op_arr || if (val_type == IS_CV) { | TRY_ADDREF val_info, ah, r1 || } - | INC_OPLINE + if (inc_op) { + | INC_OPLINE + } return 1; @@ -994,7 +1006,7 @@ fallback: return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); } -static int zend_jit_qm_assign(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) +static int zend_jit_qm_assign(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, int inc_op) { uint32_t op1_info; @@ -1004,14 +1016,14 @@ static int zend_jit_qm_assign(dasm_State **Dst, zend_op *opline, zend_op_array * op1_info = OP1_INFO(); - return zend_jit_simple_assign(Dst, opline, op_array, ssa, opline->result.var, -1, opline->op1_type, opline->op1, op1_info); + return zend_jit_simple_assign(Dst, opline, op_array, ssa, opline->result.var, -1, opline->op1_type, opline->op1, op1_info, inc_op); fallback: /* fallback to subroutine threading */ return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); } -static int zend_jit_assign(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) +static int zend_jit_assign(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, int inc_op) { uint32_t op1_info, op2_info; @@ -1024,7 +1036,7 @@ static int zend_jit_assign(dasm_State **Dst, zend_op *opline, zend_op_array *op_ op2_info = OP2_INFO(); if (!(op1_info & (MAY_BE_STRING|MAY_BE_RESOURCE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_REF))) { - return zend_jit_simple_assign(Dst, opline, op_array, ssa, opline->op1.var, op1_info, opline->op2_type, opline->op2, op2_info); + return zend_jit_simple_assign(Dst, opline, op_array, ssa, opline->op1.var, op1_info, opline->op2_type, opline->op2, op2_info, inc_op); } fallback: @@ -1127,7 +1139,7 @@ static int zend_jit_init_fcall(dasm_State **Dst, zend_op *opline, zend_op_array return 1; } -static int zend_jit_send_val(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) +static int zend_jit_send_val(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, int inc_op) { uint32_t op1_info; @@ -1139,7 +1151,9 @@ static int zend_jit_send_val(dasm_State **Dst, zend_op *opline, zend_op_array *o || if (Z_REFCOUNTED_P(zv)) { | ADDREF_CONST zv, r1 || } - | INC_OPLINE + if (inc_op) { + | INC_OPLINE + } return 1; } @@ -1150,7 +1164,9 @@ static int zend_jit_send_val(dasm_State **Dst, zend_op *opline, zend_op_array *o op1_info = OP1_INFO(); | mov r0, EX->call | ZVAL_COPY_VALUE r0, opline->result.var, FP, opline->op1.var, op1_info, r1, ecx, r2 - | INC_OPLINE + if (inc_op) { + | INC_OPLINE + } return 1; fallback: @@ -1158,7 +1174,7 @@ fallback: return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); } -static int zend_jit_send_var(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) +static int zend_jit_send_var(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, int inc_op) { uint32_t op1_info; if (!ssa->ops || !ssa->var_info) { @@ -1177,7 +1193,9 @@ static int zend_jit_send_var(dasm_State **Dst, zend_op *opline, zend_op_array *o || if (opline->op1_type == IS_CV) { | TRY_ADDREF op1_info, ch, r2 || } - | INC_OPLINE + if (inc_op) { + | INC_OPLINE + } return 1; fallback: @@ -1185,7 +1203,7 @@ fallback: return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); } -static int zend_jit_type_check(dasm_State **Dst, zend_op *opline, int b, int *opnum, zend_op_array *op_array, zend_ssa *ssa) +static int zend_jit_type_check(dasm_State **Dst, zend_op *opline, int b, int *opnum, zend_op_array *op_array, zend_ssa *ssa, int inc_op) { uint32_t op1_info; unsigned int target_label; @@ -1213,7 +1231,9 @@ static int zend_jit_type_check(dasm_State **Dst, zend_op *opline, int b, int *op | jmp =>target_label } else { | mov dword [FP + opline->result.var + 8], IS_TRUE - | INC_OPLINE + if (inc_op) { + | INC_OPLINE + } } } else if (!(op1_info & (1<opcode == ZEND_JMPZ && @@ -1231,7 +1251,9 @@ static int zend_jit_type_check(dasm_State **Dst, zend_op *opline, int b, int *op } } else { | mov dword [FP + opline->result.var + 8], IS_FALSE - | INC_OPLINE + if (inc_op) { + | INC_OPLINE + } } } else { goto fallback; From ae45ea4e97d69773b28fec582f873faddd1b53cf Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Fri, 9 Sep 2016 13:27:55 +0800 Subject: [PATCH 146/569] Fixed func return type info if the function is disabled --- ext/opcache/Optimizer/zend_func_info.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ext/opcache/Optimizer/zend_func_info.c b/ext/opcache/Optimizer/zend_func_info.c index 0097e308641db..21ab50d22b26d 100644 --- a/ext/opcache/Optimizer/zend_func_info.c +++ b/ext/opcache/Optimizer/zend_func_info.c @@ -1212,7 +1212,9 @@ uint32_t zend_get_func_info(const zend_call_info *call_info, const zend_ssa *ssa func_info_t *info; if ((info = zend_hash_find_ptr(&func_info, Z_STR_P(CRT_CONSTANT_EX(call_info->caller_op_array, call_info->caller_init_opline->op2, ssa->rt_constants)))) != NULL) { - if (info->info_func) { + if (UNEXPECTED(zend_optimizer_is_disabled_func(info->name, info->name_len))) { + ret = MAY_BE_NULL; + } else if (info->info_func) { ret = info->info_func(call_info, ssa); } else { ret = info->info; From 73c64c595b192b37cbc87a569a4da9b59931e9de Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 9 Sep 2016 10:12:52 +0300 Subject: [PATCH 147/569] Improved JIT for PRE_INC/DEC and implemented JIT for POST_INC/DEC (MAY_BE_STRING and MAY_BE_ARRAY are unsupported yet) --- ext/opcache/jit/zend_jit.c | 2 + ext/opcache/jit/zend_jit_x86.dasc | 85 ++++++++++++++++++++++++++++++- 2 files changed, 86 insertions(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index d79a831d0e893..b97cede93bafc 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -777,6 +777,8 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) #if ZEND_JIT_LEVEL >= ZEND_JIT_LEVEL_OPT_FUNC case ZEND_PRE_INC: case ZEND_PRE_DEC: + case ZEND_POST_INC: + case ZEND_POST_DEC: if (!zend_jit_inc_dec(&dasm_state, opline, op_array, ssa, zend_need_inc_op(op_array, opline, b, ssa))) { goto jit_failure; diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 1bc8d2b181807..82f7fc89d0e35 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -370,6 +370,41 @@ static void* dasm_labels[zend_lb_MAX]; ||} |.endmacro +|.macro ZVAL_COPY_VALUE_REG, dst_base, dst_offset, src_reg, src_info, tmp_reg1, tmp_reg1d, tmp_reg2 +||if (src_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) { +|| if (!(src_info & MAY_BE_DOUBLE)) { +| mov tmp_reg2, aword [src_reg] +| mov aword [dst_base + dst_offset], tmp_reg2 +|| } else if ((src_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { +|.if X64 or SSE +| movsd xmm0, qword [src_reg] +| movsd qword [dst_base + dst_offset], xmm0 +|.else +| fld qword [src_reg] +| fstp qword [dst_base + dst_offset] +|.endif +|| } else { +|.if X64 +| mov tmp_reg2, aword [src_reg] +| mov aword [dst_base + dst_offset], tmp_reg2 +|.else +| mov tmp_reg2, dword [src_reg] +| mov tmp_reg1, dword [src_reg + 4] +| mov dword [dst_base + dst_offset], tmp_reg2 +| mov dword [dst_base + dst_offset + 4], tmp_reg1 +|.endif +|| } +||} +||if ((src_info & (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE)) && +|| has_concrete_type(src_info & MAY_BE_ANY)) { +|| zend_uchar type = concrete_type(src_info); +| mov dword [dst_base + dst_offset + 8], type +||} else { +| mov tmp_reg1d, dword [src_reg + 8] +| mov dword [dst_base + dst_offset + 8], tmp_reg1d +||} +|.endmacro + |.macro ADDREF_CONST, zv, tmp_reg |.if X64 || if (!IS_32BIT(Z_LVAL_P(zv))) { @@ -394,6 +429,16 @@ static void* dasm_labels[zend_lb_MAX]; ||} |.endmacro +|.macro ZVAL_DEREF, reg, info +|| if (info & MAY_BE_REF) { +| cmp byte [reg + 8], IS_REFERENCE +| jne >1 +| mov reg, [reg] +| add reg, 8 +|1: +|| } +|.endmacro + |.macro INC_OPLINE | add IP, sizeof(zend_op) |.endmacro @@ -659,11 +704,15 @@ static int zend_jit_inc_dec(dasm_State **Dst, zend_op *opline, zend_op_array *op { uint32_t op1_info, op1_def_info; - if (!ssa->ops || !ssa->var_info || opline->op1_type != IS_CV || opline->result_type != IS_UNUSED) { + if (!ssa->ops || !ssa->var_info || opline->op1_type != IS_CV) { goto fallback; } op1_info = OP1_INFO(); + if (op1_info & (MAY_BE_UNDEF|MAY_BE_STRING|MAY_BE_ARRAY)) { + // TODO: support for IS_STRING and IS_ARRAY ??? + goto fallback; + } if (!(op1_info & MAY_BE_LONG)) { goto fallback; } @@ -672,6 +721,10 @@ static int zend_jit_inc_dec(dasm_State **Dst, zend_op *opline, zend_op_array *op | cmp dword [FP + opline->op1.var + 8], IS_LONG | jne >2 } + || if ((opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) && + || opline->result_type != IS_UNUSED) { + | ZVAL_COPY_VALUE FP, opline->result.var, FP, opline->op1.var, MAY_BE_LONG, r0, eax, r1 + || } if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { | inc aword [FP + opline->op1.var] } else { @@ -680,6 +733,10 @@ static int zend_jit_inc_dec(dasm_State **Dst, zend_op *opline, zend_op_array *op op1_def_info = OP1_DEF_INFO(); if (op1_def_info & MAY_BE_DOUBLE) { | jo >1 + || if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && + || opline->result_type != IS_UNUSED) { + | ZVAL_COPY_VALUE FP, opline->result.var, FP, opline->op1.var, MAY_BE_LONG, r0, eax, r1 + || } |.cold_codes |1: if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { @@ -700,18 +757,44 @@ static int zend_jit_inc_dec(dasm_State **Dst, zend_op *opline, zend_op_array *op |.endif } | mov dword [FP + opline->op1.var + 8], IS_DOUBLE + || if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && + || opline->result_type != IS_UNUSED) { + | ZVAL_COPY_VALUE FP, opline->result.var, FP, opline->op1.var, MAY_BE_DOUBLE, r0, eax, r1 + || } | jmp >3 |.code + } else { + || if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && + || opline->result_type != IS_UNUSED) { + | ZVAL_COPY_VALUE FP, opline->result.var, FP, opline->op1.var, MAY_BE_LONG, r0, eax, r1 + || } } if (op1_info & (MAY_BE_ANY-MAY_BE_LONG)) { |.cold_codes |2: | lea FCARG1a, [FP + opline->op1.var] + | // ZVAL_DEREF(var_ptr); + | ZVAL_DEREF FCARG1a, op1_info + || if ((opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC)) { + || if (opline->result_type != IS_UNUSED) { + | ZVAL_COPY_VALUE_REG FP, opline->result.var, FCARG1a, op1_info, r0, eax, r2 + | // zval_opt_copy_ctor(var_ptr); + | // ??? + | TRY_ADDREF op1_info, ah, r2 + || } + || } else { + | // SEPARATE_ZVAL_NOREF(var_ptr); + | // ??? + || } || if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { | EXT_CALL increment_function, r0 || } else { | EXT_CALL decrement_function, r0 || } + || if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && + || opline->result_type != IS_UNUSED) { + | ZVAL_COPY_VALUE FP, opline->result.var, FP, opline->op1.var, op1_def_info, r0, eax, r1 + || } | jmp >3 |.code } From 5c89c1d6ee68524784260214cbabb7b5787a5aac Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 9 Sep 2016 12:30:57 +0300 Subject: [PATCH 148/569] Implemented JIT for INIT_FCALL handler --- ext/opcache/jit/Makefile.frag | 1 + ext/opcache/jit/zend_jit.c | 11 ++-- ext/opcache/jit/zend_jit_disasm_x86.c | 10 ++++ ext/opcache/jit/zend_jit_helpers.c | 50 ++++++++++++++++++ ext/opcache/jit/zend_jit_x86.dasc | 75 ++++++++++++++------------- 5 files changed, 107 insertions(+), 40 deletions(-) create mode 100644 ext/opcache/jit/zend_jit_helpers.c diff --git a/ext/opcache/jit/Makefile.frag b/ext/opcache/jit/Makefile.frag index 7f66dee0dcce4..3ef58205e9403 100644 --- a/ext/opcache/jit/Makefile.frag +++ b/ext/opcache/jit/Makefile.frag @@ -7,6 +7,7 @@ $(builddir)/jit/zend_jit_x86.c: $(srcdir)/jit/zend_jit_x86.dasc $(srcdir)/jit/dy $(builddir)/jit/zend_jit.lo: \ $(builddir)/jit/zend_jit_x86.c \ + $(srcdir)/jit/zend_jit_helpers.c \ $(srcdir)/jit/zend_jit_disasm_x86.c \ $(srcdir)/jit/zend_jit_gdb.c \ $(srcdir)/jit/zend_jit_perf_dump.c \ diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index b97cede93bafc..0eb904d298dae 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -56,6 +56,7 @@ static void *dasm_end = NULL; static int zend_may_throw(zend_op *opline, zend_op_array *op_array, zend_ssa *ssa); #include "dynasm/dasm_x86.h" +#include "jit/zend_jit_helpers.c" #include "jit/zend_jit_x86.c" #include "jit/zend_jit_disasm_x86.c" #include "jit/zend_jit_gdb.c" @@ -805,11 +806,11 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) goto jit_failure; } break; -// case ZEND_INIT_FCALL: -// if (!zend_jit_init_fcall(&dasm_state, opline, op_array)) { -// goto jit_failure; -// } -// break; + case ZEND_INIT_FCALL: + if (!zend_jit_init_fcall(&dasm_state, opline, op_array)) { + goto jit_failure; + } + break; case ZEND_SEND_VAL: if (!zend_jit_send_val(&dasm_state, opline, op_array, ssa, zend_need_inc_op(op_array, opline, b, ssa))) { diff --git a/ext/opcache/jit/zend_jit_disasm_x86.c b/ext/opcache/jit/zend_jit_disasm_x86.c index c2fb58102a41b..e147662f03cd6 100644 --- a/ext/opcache/jit/zend_jit_disasm_x86.c +++ b/ext/opcache/jit/zend_jit_disasm_x86.c @@ -368,6 +368,8 @@ static int zend_jit_disasm_init(void) REGISTER_EG(exception_op); REGISTER_EG(timed_out); REGISTER_EG(current_execute_data); + REGISTER_EG(vm_stack_top); + REGISTER_EG(vm_stack_end); #undef REGISTER_EG #define REGISTER_CG(n) \ zend_jit_disasm_add_symbol("CG("#n")", \ @@ -376,6 +378,14 @@ static int zend_jit_disasm_init(void) #undef REGISTER_CG #endif + /* Register JIT helper functions */ +#define REGISTER_HELPER(n) \ + zend_jit_disasm_add_symbol(#n, \ + (uint64_t)(uintptr_t)n, sizeof(void*)); + REGISTER_HELPER(zend_jit_find_func_helper); + REGISTER_HELPER(zend_jit_extend_stack_helper); +#undef REGISTER_HELPER + zend_elf_load_symbols(); return 1; diff --git a/ext/opcache/jit/zend_jit_helpers.c b/ext/opcache/jit/zend_jit_helpers.c new file mode 100644 index 0000000000000..9f0a342effc28 --- /dev/null +++ b/ext/opcache/jit/zend_jit_helpers.c @@ -0,0 +1,50 @@ +/* + +----------------------------------------------------------------------+ + | Zend JIT | + +----------------------------------------------------------------------+ + | Copyright (c) 1998-2016 The PHP Group | + +----------------------------------------------------------------------+ + | This source file is subject to version 3.01 of the PHP license, | + | that is bundled with this package in the file LICENSE, and is | + | available through the world-wide-web at the following url: | + | http://www.php.net/license/3_01.txt | + | If you did not receive a copy of the PHP license and are unable to | + | obtain it through the world-wide-web, please send a note to | + | license@php.net so we can mail you a copy immediately. | + +----------------------------------------------------------------------+ + | Authors: Dmitry Stogov | + +----------------------------------------------------------------------+ +*/ + +static zend_function* ZEND_FASTCALL zend_jit_find_func_helper(zend_string *name) +{ + zval *func = zend_hash_find(EG(function_table), name); + zend_function *fbc; + + if (UNEXPECTED(func == NULL)) { + zend_throw_error(NULL, "Call to undefined function %s()", ZSTR_VAL(name)); + return NULL; + } + fbc = Z_FUNC_P(func); + if (EXPECTED(fbc->type == ZEND_USER_FUNCTION) && UNEXPECTED(!fbc->op_array.run_time_cache)) { + fbc->op_array.run_time_cache = zend_arena_alloc(&CG(arena), fbc->op_array.cache_size); + memset(fbc->op_array.run_time_cache, 0, fbc->op_array.cache_size); + } + return fbc; +} + +static zend_execute_data* ZEND_FASTCALL zend_jit_extend_stack_helper(uint32_t used_stack, zend_function *fbc) +{ + zend_execute_data *call = (zend_execute_data*)zend_vm_stack_extend(used_stack); + call->func = fbc; + ZEND_SET_CALL_INFO(call, 0, ZEND_CALL_NESTED_FUNCTION|ZEND_CALL_ALLOCATED); + return call; +} + +/* + * Local variables: + * tab-width: 4 + * c-basic-offset: 4 + * indent-tabs-mode: t + * End: + */ diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 82f7fc89d0e35..99f26d7e8c56c 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -1127,7 +1127,7 @@ fallback: return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); } -static int zend_jit_push_stack_frame(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_function *func) +static int zend_jit_push_call_frame(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_function *func) { if (!func) { assert(0); @@ -1136,21 +1136,35 @@ static int zend_jit_push_stack_frame(dasm_State **Dst, zend_op *opline, zend_op_ uint32_t used_stack = zend_vm_calc_used_stack(opline->extended_value, func); | // if (UNEXPECTED(used_stack > (size_t)(((char*)EG(vm_stack_end)) - (char*)call))) { - | LOAD_ADDR r1, EG(vm_stack_top) - | LOAD_ADDR r2, EG(vm_stack_end) + | mov r1, aword [&EG(vm_stack_top)] + | mov r2, aword [&EG(vm_stack_end)] | sub r2,r1 | cmp r2, used_stack - | jz >1 // TODO proper condition - // EG(vm_stack_top) = (zval*)((char*)call + used_stack); - | // ??? + | jb >2 + | // EG(vm_stack_top) = (zval*)((char*)call + used_stack); + | add aword [&EG(vm_stack_top)], used_stack | // zend_vm_init_call_frame(call, call_info, func, num_args, called_scope, object); - | //??? - |.cold_codes + | // call->func = func; + | mov aword EX:r1->func, r0 + | // ZEND_SET_CALL_INFO(call, 0, call_info); + | mov dword EX:r1->This.u1.type_info, (IS_UNDEF | (ZEND_CALL_NESTED_FUNCTION << ZEND_CALL_INFO_SHIFT)) |1: - | //??? - | jmp >2 - |.code + | // Z_CE(call->This) = called_scope; + |.if X64 + | mov64 dword EX:r1->This.value.ptr, 0 + |.else + | mov dword EX:r1->This.value.ptr, 0 + |.endif + | // ZEND_CALL_NUM_ARGS(call) = num_args; + | mov dword EX:r1->This.u2.num_args, opline->extended_value + |.cold_codes |2: + | mov FCARG1d, used_stack + | mov FCARG2a, r0 + | EXT_CALL zend_jit_extend_stack_helper, r0 + | mov r1, r0 + | jmp <1 + |.code return 1; } @@ -1185,39 +1199,30 @@ static int zend_jit_init_fcall(dasm_State **Dst, zend_op *opline, zend_op_array | jz >1 |.cold_codes |1: - | // if ((func = zend_hash_find(EG(function_table), Z_STR_P(fname))) == NULL) { - | LOAD_ADDR FCARG1a, Z_STR_P(zv); - | EXT_CALL zend_hash_find, r0 - | test r0, r0 - | jz >2 - | mov r0, aword [r0] - | jmp >3 - |2: | // SAVE_OPLINE(); | mov EX->opline, IP - | // zend_throw_error(NULL, "Call to undefined function %s()", Z_STRVAL_P(fname)); - |.if X64 - | xor CARG1, CARG1 - | LOAD_ADDR CARG2, "Call to undefined function %s()" - | LOAD_ADDR CARG3, Z_STRVAL_P(zv) - |.else - | push 0 - | push "Call to undefined function %s()" - | push Z_STRVAL_P(zv) - |.endif - | EXT_CALL zend_throw_error, r0 - | jmp ->exception_handler + | LOAD_ADDR FCARG1a, Z_STR_P(zv); + | EXT_CALL zend_jit_find_func_helper, r0 + | // This opcode must not throw exception ??? + | // test r0, r0 + | // jz ->exception_handler */ + | // CACHE_PTR(Z_CACHE_SLOT_P(fname), fbc); + | mov r1, EX->run_time_cache + | mov aword [r1 + Z_CACHE_SLOT_P(zv)], r0 + | jmp >3 |.code |3: } - if (!zend_jit_push_stack_frame(Dst, opline, op_array, func)) { + if (!zend_jit_push_call_frame(Dst, opline, op_array, func)) { return 0; } | // call->prev_execute_data = EX(call); - | mov r1, EX->call - | mov EX:r0->prev_execute_data, r1 + | // TODO: optimize for first call level ??? + | mov r0, EX->call + | mov EX:r1->prev_execute_data, r0 | // EX(call) = call; - | mov EX->call, r0 + | mov EX->call, r1 + | INC_OPLINE return 1; } From bc0590ae62e2c88bd0db384922d7b6a6d0a236e0 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 9 Sep 2016 12:58:36 +0300 Subject: [PATCH 149/569] JIT optimization for the first call level --- ext/opcache/jit/zend_jit.c | 21 ++++++++++++++++++++- ext/opcache/jit/zend_jit_x86.dasc | 17 ++++++++--------- 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 0eb904d298dae..93f7374140b9f 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -694,6 +694,7 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) zend_op *opline; dasm_State* dasm_state = NULL; void *handler; + int call_level = 0; /* mark hidden branch targets */ for (b = 0; b < ssa->cfg.blocks_count; b++) { @@ -775,6 +776,17 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) for (i = ssa->cfg.blocks[b].start; i <= end; i++) { opline = op_array->opcodes + i; switch (opline->opcode) { + case ZEND_INIT_FCALL: + case ZEND_INIT_FCALL_BY_NAME: + case ZEND_INIT_NS_FCALL_BY_NAME: + case ZEND_INIT_METHOD_CALL: + case ZEND_INIT_DYNAMIC_CALL: + case ZEND_INIT_STATIC_METHOD_CALL: + case ZEND_INIT_USER_CALL: + case ZEND_NEW: + call_level++; + } + switch (opline->opcode) { #if ZEND_JIT_LEVEL >= ZEND_JIT_LEVEL_OPT_FUNC case ZEND_PRE_INC: case ZEND_PRE_DEC: @@ -807,7 +819,7 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) } break; case ZEND_INIT_FCALL: - if (!zend_jit_init_fcall(&dasm_state, opline, op_array)) { + if (!zend_jit_init_fcall(&dasm_state, opline, op_array, call_level)) { goto jit_failure; } break; @@ -979,6 +991,13 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) goto jit_failure; } } + switch (opline->opcode) { + case ZEND_DO_FCALL: + case ZEND_DO_ICALL: + case ZEND_DO_UCALL: + case ZEND_DO_FCALL_BY_NAME: + call_level--; + } } } diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 99f26d7e8c56c..a70e867451ebb 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -1150,11 +1150,7 @@ static int zend_jit_push_call_frame(dasm_State **Dst, zend_op *opline, zend_op_a | mov dword EX:r1->This.u1.type_info, (IS_UNDEF | (ZEND_CALL_NESTED_FUNCTION << ZEND_CALL_INFO_SHIFT)) |1: | // Z_CE(call->This) = called_scope; - |.if X64 - | mov64 dword EX:r1->This.value.ptr, 0 - |.else - | mov dword EX:r1->This.value.ptr, 0 - |.endif + | mov aword EX:r1->This.value.ptr, 0 | // ZEND_CALL_NUM_ARGS(call) = num_args; | mov dword EX:r1->This.u2.num_args, opline->extended_value |.cold_codes @@ -1169,7 +1165,7 @@ static int zend_jit_push_call_frame(dasm_State **Dst, zend_op *opline, zend_op_a return 1; } -static int zend_jit_init_fcall(dasm_State **Dst, zend_op *opline, zend_op_array *op_array) +static int zend_jit_init_fcall(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, int call_level) { zend_func_info *info = ZEND_FUNC_INFO(op_array); zend_function *func = NULL; @@ -1217,9 +1213,12 @@ static int zend_jit_init_fcall(dasm_State **Dst, zend_op *opline, zend_op_array return 0; } | // call->prev_execute_data = EX(call); - | // TODO: optimize for first call level ??? - | mov r0, EX->call - | mov EX:r1->prev_execute_data, r0 + || if (call_level == 1) { + | mov aword EX:r1->prev_execute_data, 0 + || } else { + | mov r0, EX->call + | mov EX:r1->prev_execute_data, r0 + || } | // EX(call) = call; | mov EX->call, r1 | INC_OPLINE From d4dca6c300c478d687d23824036d5daa4b5218fe Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Fri, 9 Sep 2016 18:38:29 +0800 Subject: [PATCH 150/569] Optimization OPLINE++ handling --- ext/opcache/jit/zend_jit.c | 6 +- ext/opcache/jit/zend_jit_x86.dasc | 119 ++++++++++++++++++++++-------- 2 files changed, 92 insertions(+), 33 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 93f7374140b9f..186f6e99bd199 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -874,7 +874,8 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) i++; skip++; } - if (!zend_jit_skip_handler(&dasm_state, skip)) { + if (!zend_jit_skip_handler(&dasm_state, + skip, zend_need_inc_op(op_array, opline + skip, b, ssa))) { goto jit_failure; } } @@ -966,7 +967,8 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) } break; case ZEND_NEW: - if (!zend_jit_new(&dasm_state, opline, &i, op_array, ssa)) { + if (!zend_jit_new(&dasm_state, + opline, &i, op_array, ssa, zend_need_inc_op(op_array, opline, b, ssa))) { goto jit_failure; } break; diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index a70e867451ebb..1ed1ad6f4e58a 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -439,10 +439,22 @@ static void* dasm_labels[zend_lb_MAX]; || } |.endmacro -|.macro INC_OPLINE -| add IP, sizeof(zend_op) +|.macro RESET_IP +||ip_count = 0; |.endmacro +|.macro INC_IP +||ip_count++; +|.endmacro + +|.macro SAVE_IP +||if (ip_count) { +| add IP, sizeof(zend_op) * ip_count +| RESET_IP +||} +|.endmacro + +static uint32_t ip_count = 0; #endif /* bit helpers */ @@ -581,7 +593,8 @@ static int zend_jit_check_exception(dasm_State **Dst) static int zend_jit_handler(dasm_State **Dst, zend_op *opline, int may_throw) { - | EXT_CALL opline->handler, r0 + | SAVE_IP + | EXT_CALL opline->handler, r0 if (may_throw) { zend_jit_check_exception(Dst); } @@ -590,14 +603,19 @@ static int zend_jit_handler(dasm_State **Dst, zend_op *opline, int may_throw) static int zend_jit_tail_handler(dasm_State **Dst, zend_op *opline) { + | SAVE_IP | add r4, SPAD // stack alignment | EXT_JMP opline->handler, r0 return 1; } -static int zend_jit_skip_handler(dasm_State **Dst, uint32_t skip) +static int zend_jit_skip_handler(dasm_State **Dst, uint32_t skip, int inc_op) { - | add IP, sizeof(zend_op) * skip + if (inc_op) { + while (skip--) { + | INC_IP + } + } return 1; } @@ -609,14 +627,17 @@ static int zend_jit_set_opline(dasm_State **Dst, zend_op *target_opline) static int zend_jit_jmp(dasm_State **Dst, unsigned int target_label) { + | RESET_IP | jmp =>target_label return 1; } static int zend_jit_cond_jmp(dasm_State **Dst, zend_op *next_opline, unsigned int target_label) { + | SAVE_IP | cmp IPl, next_opline | jnz =>target_label + return 1; } @@ -625,10 +646,12 @@ static int zend_jit_smart_branch(dasm_State **Dst, zend_op *opline, unsigned int zend_op *next_opline = opline + 1; zend_op *target_opline = OP_JMP_ADDR(opline, opline->op2); + | SAVE_IP | cmp IPl, next_opline | jz =>next_label | cmp IPl, target_opline | jz =>target_label + return 1; } @@ -661,7 +684,7 @@ static int zend_jit_call(dasm_State **Dst, zend_op *opline) #endif } -static int zend_jit_new(dasm_State **Dst, zend_op *opline, uint32_t *opnum, zend_op_array *op_array, zend_ssa *ssa) +static int zend_jit_new(dasm_State **Dst, zend_op *opline, uint32_t *opnum, zend_op_array *op_array, zend_ssa *ssa, int inc_op) { if (!zend_jit_handler(Dst, opline, 1)) { return 0; @@ -691,8 +714,10 @@ static int zend_jit_new(dasm_State **Dst, zend_op *opline, uint32_t *opnum, zend | jnz >1 zend_jit_call(Dst, next_opline); |1: + } else if (inc_op) { + | INC_IP } else { - //| INC_OPLINE + | RESET_IP } } return 1; @@ -800,7 +825,9 @@ static int zend_jit_inc_dec(dasm_State **Dst, zend_op *opline, zend_op_array *op } |3: if (inc_op) { - | INC_OPLINE + | INC_IP + } else { + | RESET_IP } return 1; @@ -859,9 +886,10 @@ static int zend_jit_math(dasm_State **Dst, zend_op *opline, zend_op_array *op_ar goto fallback; } if (inc_op) { - | INC_OPLINE + | INC_IP + } else { + | RESET_IP } - return 1; fallback: @@ -904,7 +932,10 @@ static int zend_jit_cmp(dasm_State **Dst, zend_op *opline, int b, int *opnum, ze break; } if (inc_op) { - | add IP, sizeof(zend_op) * 2 + | INC_IP + | INC_IP + } else { + | RESET_IP } } else if ((opline+1)->opcode == ZEND_JMPNZ && (opline+1)->op1_type == IS_TMP_VAR && @@ -925,8 +956,11 @@ static int zend_jit_cmp(dasm_State **Dst, zend_op *opline, int b, int *opnum, ze | jle => target_label break; } - if (!(ssa->cfg.blocks[ssa->cfg.blocks[b].successors[1]].flags & ZEND_BB_TARGET)) { - | add IP, sizeof(zend_op) * 2 + if (inc_op) { + | INC_IP + | INC_IP + } else { + | RESET_IP } } else { switch (opline->opcode) { @@ -948,7 +982,9 @@ static int zend_jit_cmp(dasm_State **Dst, zend_op *opline, int b, int *opnum, ze } | mov dword [FP + opline->result.var + 8], eax if (inc_op) { - | INC_OPLINE + | INC_IP + } else { + | RESET_IP } } } else if (((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) && @@ -985,8 +1021,11 @@ static int zend_jit_cmp(dasm_State **Dst, zend_op *opline, int b, int *opnum, ze | jae => target_label break; } - if (!(ssa->cfg.blocks[ssa->cfg.blocks[b].successors[1]].flags & ZEND_BB_TARGET)) { - | add IP, sizeof(zend_op) * 2 + if (inc_op) { + | INC_IP + | INC_IP + } else { + | RESET_IP } } else if ((opline+1)->opcode == ZEND_JMPNZ && (opline+1)->op1_type == IS_TMP_VAR && @@ -1011,8 +1050,9 @@ static int zend_jit_cmp(dasm_State **Dst, zend_op *opline, int b, int *opnum, ze | jnae => target_label break; } - if (!(ssa->cfg.blocks[ssa->cfg.blocks[b].successors[1]].flags & ZEND_BB_TARGET)) { - | add IP, sizeof(zend_op) * 2 + if (inc_op) { + | INC_IP + | INC_IP } } else { switch (opline->opcode) { @@ -1043,7 +1083,9 @@ static int zend_jit_cmp(dasm_State **Dst, zend_op *opline, int b, int *opnum, ze } | mov dword [FP + opline->result.var + 8], eax if (inc_op) { - | INC_OPLINE + | INC_IP + } else { + | RESET_IP } } } else { @@ -1069,7 +1111,9 @@ static int zend_jit_simple_assign(dasm_State **Dst, zend_op *opline, zend_op_arr | ADDREF_CONST zv, r0 || } if (inc_op) { - | INC_OPLINE + | INC_IP + } else { + | RESET_IP } return 1; } @@ -1079,7 +1123,9 @@ static int zend_jit_simple_assign(dasm_State **Dst, zend_op *opline, zend_op_arr | TRY_ADDREF val_info, ah, r1 || } if (inc_op) { - | INC_OPLINE + | INC_IP + } else { + | RESET_IP } return 1; @@ -1221,7 +1267,7 @@ static int zend_jit_init_fcall(dasm_State **Dst, zend_op *opline, zend_op_array || } | // EX(call) = call; | mov EX->call, r1 - | INC_OPLINE + | INC_IP return 1; } @@ -1239,7 +1285,9 @@ static int zend_jit_send_val(dasm_State **Dst, zend_op *opline, zend_op_array *o | ADDREF_CONST zv, r1 || } if (inc_op) { - | INC_OPLINE + | INC_IP + } else { + | RESET_IP } return 1; } @@ -1252,7 +1300,9 @@ static int zend_jit_send_val(dasm_State **Dst, zend_op *opline, zend_op_array *o | mov r0, EX->call | ZVAL_COPY_VALUE r0, opline->result.var, FP, opline->op1.var, op1_info, r1, ecx, r2 if (inc_op) { - | INC_OPLINE + | INC_IP + } else { + | RESET_IP } return 1; @@ -1281,7 +1331,9 @@ static int zend_jit_send_var(dasm_State **Dst, zend_op *opline, zend_op_array *o | TRY_ADDREF op1_info, ch, r2 || } if (inc_op) { - | INC_OPLINE + | INC_IP + } else { + | RESET_IP } return 1; @@ -1307,8 +1359,10 @@ static int zend_jit_type_check(dasm_State **Dst, zend_op *opline, int b, int *op (opline+1)->op1_type == IS_TMP_VAR && (opline+1)->op1.var == opline->result.var) { (*opnum)++; - if (!(ssa->cfg.blocks[ssa->cfg.blocks[b].successors[1]].flags & ZEND_BB_TARGET)) { - | add IP, sizeof(zend_op) * 2 + if (inc_op) { + | INC_IP + } else { + | RESET_IP } } else if ((opline+1)->opcode == ZEND_JMPNZ && (opline+1)->op1_type == IS_TMP_VAR && @@ -1319,7 +1373,9 @@ static int zend_jit_type_check(dasm_State **Dst, zend_op *opline, int b, int *op } else { | mov dword [FP + opline->result.var + 8], IS_TRUE if (inc_op) { - | INC_OPLINE + | INC_IP + } else { + | RESET_IP } } } else if (!(op1_info & (1<op1_type == IS_TMP_VAR && (opline+1)->op1.var == opline->result.var) { (*opnum)++; - if (!(ssa->cfg.blocks[ssa->cfg.blocks[b].successors[1]].flags & ZEND_BB_TARGET)) { - | add IP, sizeof(zend_op) * 2 + if (inc_op) { + | INC_IP + | INC_IP } } else { | mov dword [FP + opline->result.var + 8], IS_FALSE if (inc_op) { - | INC_OPLINE + | INC_IP } } } else { From 59737629acde946d3051fce8e36e9d2279fbc70d Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Fri, 9 Sep 2016 18:40:44 +0800 Subject: [PATCH 151/569] Fixed compiler warning --- ext/opcache/Optimizer/zend_func_info.c | 1 + ext/opcache/jit/zend_jit_x86.dasc | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ext/opcache/Optimizer/zend_func_info.c b/ext/opcache/Optimizer/zend_func_info.c index 21ab50d22b26d..0e4e4441eaf94 100644 --- a/ext/opcache/Optimizer/zend_func_info.c +++ b/ext/opcache/Optimizer/zend_func_info.c @@ -22,6 +22,7 @@ #include "zend_compile.h" #include "zend_extensions.h" #include "zend_ssa.h" +#include "zend_optimizer_internal.h" #include "zend_inference.h" #include "zend_call_graph.h" #include "zend_func_info.h" diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 1ed1ad6f4e58a..805f4dc1194af 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -684,7 +684,7 @@ static int zend_jit_call(dasm_State **Dst, zend_op *opline) #endif } -static int zend_jit_new(dasm_State **Dst, zend_op *opline, uint32_t *opnum, zend_op_array *op_array, zend_ssa *ssa, int inc_op) +static int zend_jit_new(dasm_State **Dst, zend_op *opline, int *opnum, zend_op_array *op_array, zend_ssa *ssa, int inc_op) { if (!zend_jit_handler(Dst, opline, 1)) { return 0; From 90f9575d51cab2cc239e6d567e168c9460203aaa Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Fri, 9 Sep 2016 18:42:14 +0800 Subject: [PATCH 152/569] Add myself into author list --- ext/opcache/jit/zend_jit_x86.dasc | 1 + 1 file changed, 1 insertion(+) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 805f4dc1194af..8be4e06849d49 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -13,6 +13,7 @@ * | license@php.net so we can mail you a copy immediately. | * +----------------------------------------------------------------------+ * | Authors: Dmitry Stogov | + * | Xinchen Hui | * +----------------------------------------------------------------------+ */ From d20b33d87851c62c2eae3973b1448c0b13329061 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Fri, 9 Sep 2016 19:25:15 +0800 Subject: [PATCH 153/569] Fixed segfault in bench --- ext/opcache/jit/zend_jit.c | 5 +++-- ext/opcache/jit/zend_jit_x86.dasc | 20 ++++++++++++++------ 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 186f6e99bd199..a5c9db64d0513 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -819,7 +819,8 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) } break; case ZEND_INIT_FCALL: - if (!zend_jit_init_fcall(&dasm_state, opline, op_array, call_level)) { + if (!zend_jit_init_fcall(&dasm_state, + opline, op_array, call_level, zend_need_inc_op(op_array, opline, b, ssa))) { goto jit_failure; } break; @@ -840,7 +841,7 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) case ZEND_IS_SMALLER: case ZEND_IS_SMALLER_OR_EQUAL: if (!zend_jit_cmp(&dasm_state, - opline, b, &i, op_array, ssa, zend_need_inc_op(op_array, opline, b, ssa))) { + opline, b, &i, op_array, ssa, zend_need_inc_op(op_array, opline + 1, b, ssa))) { goto jit_failure; } break; diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 8be4e06849d49..082997dd784bd 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -524,6 +524,7 @@ static int zend_jit_interrupt_handler_stub(dasm_State **Dst) | //} else if (zend_interrupt_function) { if (zend_interrupt_function) { | //SAVE_OPLINE(); + | SAVE_IP | mov EX->opline, IP | //zend_interrupt_function(execute_data); |.if X64 @@ -932,7 +933,7 @@ static int zend_jit_cmp(dasm_State **Dst, zend_op *opline, int b, int *opnum, ze | jge => target_label break; } - if (inc_op) { + if (!(ssa->cfg.blocks[ssa->cfg.blocks[b].successors[1]].flags & ZEND_BB_TARGET)) { | INC_IP | INC_IP } else { @@ -957,7 +958,7 @@ static int zend_jit_cmp(dasm_State **Dst, zend_op *opline, int b, int *opnum, ze | jle => target_label break; } - if (inc_op) { + if (!(ssa->cfg.blocks[ssa->cfg.blocks[b].successors[1]].flags & ZEND_BB_TARGET)) { | INC_IP | INC_IP } else { @@ -1022,7 +1023,7 @@ static int zend_jit_cmp(dasm_State **Dst, zend_op *opline, int b, int *opnum, ze | jae => target_label break; } - if (inc_op) { + if (!(ssa->cfg.blocks[ssa->cfg.blocks[b].successors[1]].flags & ZEND_BB_TARGET)) { | INC_IP | INC_IP } else { @@ -1051,9 +1052,11 @@ static int zend_jit_cmp(dasm_State **Dst, zend_op *opline, int b, int *opnum, ze | jnae => target_label break; } - if (inc_op) { + if (!(ssa->cfg.blocks[ssa->cfg.blocks[b].successors[1]].flags & ZEND_BB_TARGET)) { | INC_IP | INC_IP + } else { + | RESET_IP } } else { switch (opline->opcode) { @@ -1212,7 +1215,7 @@ static int zend_jit_push_call_frame(dasm_State **Dst, zend_op *opline, zend_op_a return 1; } -static int zend_jit_init_fcall(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, int call_level) +static int zend_jit_init_fcall(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, int call_level, int inc_op) { zend_func_info *info = ZEND_FUNC_INFO(op_array); zend_function *func = NULL; @@ -1243,6 +1246,7 @@ static int zend_jit_init_fcall(dasm_State **Dst, zend_op *opline, zend_op_array |.cold_codes |1: | // SAVE_OPLINE(); + | SAVE_IP | mov EX->opline, IP | LOAD_ADDR FCARG1a, Z_STR_P(zv); | EXT_CALL zend_jit_find_func_helper, r0 @@ -1268,7 +1272,11 @@ static int zend_jit_init_fcall(dasm_State **Dst, zend_op *opline, zend_op_array || } | // EX(call) = call; | mov EX->call, r1 - | INC_IP + if (inc_op) { + | INC_IP + } else { + | RESET_IP + } return 1; } From 0c9ab36e2ac0c39403c7e0d24fdad7d41cc6ee6f Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Fri, 9 Sep 2016 19:33:17 +0800 Subject: [PATCH 154/569] Fixed SAVE_IP --- ext/opcache/jit/zend_jit_x86.dasc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 082997dd784bd..23ccbad2d8313 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -1230,6 +1230,7 @@ static int zend_jit_init_fcall(dasm_State **Dst, zend_op *opline, zend_op_array func = call_info->callee_func; } } + | SAVE_IP if (func && func->type == ZEND_INTERNAL_FUNCTION) { | LOAD_ADDR r0, func } else if (func && op_array == &func->op_array) { @@ -1246,7 +1247,6 @@ static int zend_jit_init_fcall(dasm_State **Dst, zend_op *opline, zend_op_array |.cold_codes |1: | // SAVE_OPLINE(); - | SAVE_IP | mov EX->opline, IP | LOAD_ADDR FCARG1a, Z_STR_P(zv); | EXT_CALL zend_jit_find_func_helper, r0 From 2f9e73f429f6a4f882b5d57ae8f68dee4352279c Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Fri, 9 Sep 2016 19:37:48 +0800 Subject: [PATCH 155/569] cleanup --- ext/opcache/jit/zend_jit_x86.dasc | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 23ccbad2d8313..9c0519e1d1824 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -1230,7 +1230,6 @@ static int zend_jit_init_fcall(dasm_State **Dst, zend_op *opline, zend_op_array func = call_info->callee_func; } } - | SAVE_IP if (func && func->type == ZEND_INTERNAL_FUNCTION) { | LOAD_ADDR r0, func } else if (func && op_array == &func->op_array) { @@ -1239,7 +1238,8 @@ static int zend_jit_init_fcall(dasm_State **Dst, zend_op *opline, zend_op_array } else { zval *zv = RT_CONSTANT(op_array, opline->op2); - | // if (CACHED_PTR(Z_CACHE_SLOT_P(fname))) { + | SAVE_IP + | // if (CACHED_PTR(Z_CACHE_SLOT_P(fname))) | mov r0, EX->run_time_cache | mov r0, aword [r0 + Z_CACHE_SLOT_P(zv)] | test r0, r0 @@ -1264,14 +1264,14 @@ static int zend_jit_init_fcall(dasm_State **Dst, zend_op *opline, zend_op_array return 0; } | // call->prev_execute_data = EX(call); - || if (call_level == 1) { - | mov aword EX:r1->prev_execute_data, 0 - || } else { - | mov r0, EX->call - | mov EX:r1->prev_execute_data, r0 - || } - | // EX(call) = call; - | mov EX->call, r1 + if (call_level == 1) { + | mov aword EX:r1->prev_execute_data, 0 + } else { + | mov r0, EX->call + | mov EX:r1->prev_execute_data, r0 + } + | // EX(call) = call; + | mov EX->call, r1 if (inc_op) { | INC_IP } else { From 2b8bde6694e6eab21e7192e8ec32b387fea068f5 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Fri, 9 Sep 2016 20:06:59 +0800 Subject: [PATCH 156/569] Fixed silly bug on opline calculte --- ext/opcache/jit/zend_jit.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index a5c9db64d0513..8555e434d2460 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -876,7 +876,7 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) skip++; } if (!zend_jit_skip_handler(&dasm_state, - skip, zend_need_inc_op(op_array, opline + skip, b, ssa))) { + skip, zend_need_inc_op(op_array, opline + skip - 1, b, ssa))) { goto jit_failure; } } From bb9e9ea03792624774f02b38bfd8bf23e2cf135c Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 9 Sep 2016 16:19:11 +0300 Subject: [PATCH 157/569] Implemented JIT for DO_UCALL --- ext/opcache/jit/zend_jit.c | 7 +++ ext/opcache/jit/zend_jit_x86.dasc | 90 +++++++++++++++++++++++++++++++ 2 files changed, 97 insertions(+) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 8555e434d2460..77deebd17fc69 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -836,6 +836,11 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) goto jit_failure; } break; + case ZEND_DO_UCALL: + if (!zend_jit_do_fcall(&dasm_state, opline, op_array, ssa, call_level)) { + goto jit_failure; + } + break; case ZEND_IS_EQUAL: case ZEND_IS_NOT_EQUAL: case ZEND_IS_SMALLER: @@ -906,7 +911,9 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) /* stackless execution */ case ZEND_INCLUDE_OR_EVAL: case ZEND_DO_FCALL: +#if ZEND_JIT_LEVEL < ZEND_JIT_LEVEL_OPT_FUNC case ZEND_DO_UCALL: +#endif case ZEND_DO_FCALL_BY_NAME: if (!zend_jit_call(&dasm_state, opline)) { goto jit_failure; diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 9c0519e1d1824..7358a6d774abe 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -1281,6 +1281,96 @@ static int zend_jit_init_fcall(dasm_State **Dst, zend_op *opline, zend_op_array return 1; } +static int zend_jit_do_fcall(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, int call_level) +{ + zend_func_info *info = ZEND_FUNC_INFO(op_array); + zend_call_info *call_info = NULL; + zend_function *func = NULL; + uint32_t i; + + if (info) { + call_info = info->callee_info; + while (call_info && call_info->caller_call_opline != opline) { + call_info = call_info->next_callee; + } + if (call_info && call_info->callee_func) { + func = call_info->callee_func; + } + } + if (!func || + func->type != ZEND_USER_FUNCTION || + call_info->num_args > func->op_array.num_args || + (opline-1)->opcode == ZEND_SEND_UNPACK || + (func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) != 0) { + goto fallback; + } + + | // call = EX(call); + | mov r0, EX->call + | // fbc = call->func; + | // mov r1, EX:r0->func ??? + | // SAVE_OPLINE(); + | SAVE_IP + | mov EX->opline, IP + || if (call_level == 1) { + | mov aword EX->call, 0 + || } else { + | //EX(call) = call->prev_execute_data; + | mov r1, EX:r0->prev_execute_data + | mov EX->call, r1 + || } + | //call->prev_execute_data = execute_data; + | mov EX:r0->prev_execute_data, EX + | + | // EX(call) = NULL; + | mov aword EX:r0->call, 0 + || if (RETURN_VALUE_USED(opline)) { + | // ZVAL_NULL(EX_VAR(opline->result.var)); + | mov dword [FP + opline->result.var + 8], IS_NULL + | // EX(return_value) = EX_VAR(opline->result.var); + | lea r1, aword [FP + opline->result.var] + | mov aword EX:r0->return_value, r1 + || } else { + | // EX(return_value) = 0; + | mov aword EX:r0->return_value, 0 + || } + | + || for (i = call_info->num_args; i < func->op_array.last_var; i++) { + || uint32_t n = (uint32_t)(uintptr_t)ZEND_CALL_VAR_NUM(NULL, i); + | mov dword [r0 + n + 8], IS_UNDEF + || } + | + | //EX_LOAD_RUN_TIME_CACHE(op_array); + | mov r1, EX:r0->func + | mov r2, aword [r1 + offsetof(zend_op_array, run_time_cache)] + | mov EX:r0->run_time_cache, r2 + | //EX_LOAD_LITERALS(op_array); + |.if X64 + | LOAD_ADDR r2, func->op_array.literals + | mov EX:r0->literals, r2 + |.endif + | // EG(current_execute_data) = execute_data; + | mov aword [&EG(current_execute_data)], r0 + | mov FP, r0 + | // EX(opline) = op_array->opcodes; + | LOAD_ADDR IP, (func->op_array.opcodes + call_info->num_args) + | mov EX:r0->opline, IP + | add r4, SPAD // stack alignment + | jmp aword [IP] + + return 1; + +fallback: + /* fallback to subroutine threading */ + if (opline->opcode == ZEND_DO_FCALL || + opline->opcode == ZEND_DO_UCALL || + opline->opcode == ZEND_DO_FCALL_BY_NAME ){ + return zend_jit_call(Dst, opline); + } else { + return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); + } +} + static int zend_jit_send_val(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, int inc_op) { uint32_t op1_info; From 2fbeba39ae0518a3adaf0e92e1389e6f482d06f5 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Mon, 12 Sep 2016 12:51:15 +0800 Subject: [PATCH 158/569] Fixed may_throw for ZEND_ASSIGN_DIM(benc.php ary3) --- ext/opcache/jit/zend_jit.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 77deebd17fc69..9b44e0d888c43 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -326,6 +326,7 @@ static int zend_may_throw(zend_op *opline, zend_op_array *op_array, zend_ssa *ss case ZEND_ISSET_ISEMPTY_DIM_OBJ: case ZEND_ISSET_ISEMPTY_PROP_OBJ: case ZEND_ASSIGN: + case ZEND_ASSIGN_DIM: case ZEND_ASSIGN_REF: case ZEND_BIND_GLOBAL: case ZEND_FETCH_DIM_IS: @@ -555,6 +556,9 @@ static int zend_may_throw(zend_op *opline, zend_op_array *op_array, zend_ssa *ss (t2 & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT)); case ZEND_ASSIGN: return (t1 & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_ARRAY)); + case ZEND_ASSIGN_DIM: + return (t1 & (MAY_BE_OBJECT|MAY_BE_LONG|MAY_BE_DOUBLE)) || opline->op2_type == IS_UNUSED || + (t2 & (MAY_BE_UNDEF| MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)); case ZEND_ROPE_INIT: case ZEND_ROPE_ADD: case ZEND_ROPE_END: From dc84ee2bae9375d64c0a137607df2e3e09a27fdf Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Mon, 12 Sep 2016 14:01:16 +0800 Subject: [PATCH 159/569] Explictly reset ip count --- ext/opcache/jit/zend_jit_x86.dasc | 1 + 1 file changed, 1 insertion(+) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 7358a6d774abe..bfbfd72396abe 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -563,6 +563,7 @@ static const zend_jit_stub zend_jit_stubs[] = { static int zend_jit_align_func(dasm_State **Dst) { + | RESET_IP |.align 16 return 1; } From 7b3c90f2cd0adb71d28c2b0a38152fe9cdd4d7f8 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 12 Sep 2016 09:49:58 +0300 Subject: [PATCH 160/569] ASSIGN_ADD may throw if op1 MAY_BE_RESOURCE|MAY_BE_TRUE|MAY_BE_STRING. --- ext/opcache/jit/zend_jit.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 9b44e0d888c43..d2f2c2285d1d9 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -557,8 +557,8 @@ static int zend_may_throw(zend_op *opline, zend_op_array *op_array, zend_ssa *ss case ZEND_ASSIGN: return (t1 & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_ARRAY)); case ZEND_ASSIGN_DIM: - return (t1 & (MAY_BE_OBJECT|MAY_BE_LONG|MAY_BE_DOUBLE)) || opline->op2_type == IS_UNUSED || - (t2 & (MAY_BE_UNDEF| MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)); + return (t1 & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_TRUE|MAY_BE_STRING|MAY_BE_LONG|MAY_BE_DOUBLE)) || opline->op2_type == IS_UNUSED || + (t2 & (MAY_BE_UNDEF|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)); case ZEND_ROPE_INIT: case ZEND_ROPE_ADD: case ZEND_ROPE_END: From a30cdc19faf4bb21cc77c760de8c5c7e31a1dc34 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 12 Sep 2016 15:48:40 +0300 Subject: [PATCH 161/569] Improved JIT for ADD, SUB and MUL (only MAY_BE_UNDEF triggers fallback) --- ext/opcache/jit/zend_jit_x86.dasc | 418 +++++++++++++++++++++++++++--- 1 file changed, 379 insertions(+), 39 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index bfbfd72396abe..eb6d87b50e7c5 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -109,6 +109,34 @@ static void* dasm_labels[zend_lb_MAX]; #if ZEND_JIT_LEVEL >= ZEND_JIT_LEVEL_OT_FUNC +|.macro LOAD_ZVAL_ADDR, reg, op_type, op +||if (op_type == IS_CONST) { +| .if X64 +| mov reg, aword EX->literals +| add reg, op.constant +| .else +| mov reg, op.zv +| .endif +||} else { +| lea reg, qword [FP + op.var] +||} +|.endmacro + +|.macro PUSH_ZVAL_ADDR, op_type, op, tmp_reg +||if (op_type == IS_CONST) { +| .if X64 +| mov reg, aword EX->literals +| add reg, op.constant +| push reg +| .else +| push op.zv +| .endif +||} else { +| lea tmp_reg, qword [FP + op.var] +| push tmp_reg +||} +|.endmacro + |.macro FP_OP, fp_ins, op_type, op ||if (op_type == IS_CONST) { | .if X64 @@ -161,6 +189,23 @@ static void* dasm_labels[zend_lb_MAX]; ||} |.endmacro +|.macro SSE_LOAD_LONG, op_type, op, reg +||if (op_type == IS_CONST) { +| .if X64 +| mov r0, aword EX->literals +| cvtsi2sd reg, qword [r0 + op.constant] +| .else +| cvtsi2sd reg, dword [op.zv] +| .endif +||} else { +| .if X64 +| cvtsi2sd reg, qword [FP + op.var] +| .else +| cvtsi2sd reg, dword [FP + op.var] +| .endif +||} +|.endmacro + |.macro SSE_LOAD, op_type, op, reg | SSE_OP movsd, op_type, op, reg |.endmacro @@ -254,6 +299,20 @@ static void* dasm_labels[zend_lb_MAX]; ||} |.endmacro +|.macro LONG_MATH2, opcode, reg1, reg2 +||switch (opcode) { +|| case ZEND_ADD: +| add reg2, reg1 +|| break; +|| case ZEND_SUB: +| sub reg2, reg1 +|| break; +|| case ZEND_MUL: +| imul reg2, reg1 +|| break; +||} +|.endmacro + |.macro LONGF_OP, fp_ins, op_type, op ||if (op_type == IS_CONST) { | .if X64 @@ -274,16 +333,16 @@ static void* dasm_labels[zend_lb_MAX]; |.macro LONGF_MATH, opcode ||switch (opcode) { || case ZEND_ADD: -| fadd st0, st1 +| fadd st1 || break; || case ZEND_SUB: -| fsub st0, st1 +| fsub st1 || break; || case ZEND_MUL: -| fmul st0, st1 +| fmul st1 || break; || case ZEND_DIV: -| fdiv st0, st1 +| fdiv st1 || break; ||} |.endmacro @@ -455,6 +514,12 @@ static void* dasm_labels[zend_lb_MAX]; ||} |.endmacro + +|.macro JNE_SLOW +| jne >9 +|| has_slow = 1; +|.endmacro + static uint32_t ip_count = 0; #endif @@ -839,9 +904,111 @@ fallback: return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); } +static int zend_jit_math_long_long(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) +{ + uint32_t res_info = RES_INFO(); + zend_bool same_ops = (opline->op1_type == opline->op2_type) && (opline->op1.var == opline->op2.var); + + | LONG_LOAD opline->op1_type, opline->op1, r0 + || if (same_ops && opline->opcode != ZEND_DIV) { + | LONG_MATH2 opline->opcode, r0, r0 + || } else { + | LONG_MATH opline->opcode, opline->op2_type, opline->op2, r0 + || } + if ((res_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) { + | LONG_STORE opline->result, r0 + } else { + | jo >1 + |.cold_codes + |1: + |.if X64 or SSE + | SSE_LOAD_LONG opline->op1_type, opline->op1, xmm0 + | SSE_LOAD_LONG opline->op2_type, opline->op2, xmm1 + | SSE_MATH2 opline->opcode, xmm1, xmm0 + | SSE_STORE opline->result, xmm0 + |.else + | LONGF_LOAD opline->op2_type, opline->op2 + | LONGF_LOAD opline->op1_type, opline->op1 + | LONGF_MATH opline->opcode + | FP_STORE opline->result + |.endif + | jmp >2 + |.code + | LONG_STORE opline->result, r0 + |2: + } + + return 1; +} + +static int zend_jit_math_long_double(dasm_State **Dst, zend_op *opline) +{ + |.if X64 or SSE + | SSE_LOAD_LONG opline->op1_type, opline->op1, xmm0 + | SSE_MATH opline->opcode, opline->op2_type, opline->op2, xmm0 + | SSE_STORE opline->result, xmm0 + |.else + | LONGF_LOAD opline->op1_type, opline->op1 + | FP_MATH opline->opcode, opline->op2_type, opline->op2 + | FP_STORE opline->result + |.endif + + return 1; +} + +static int zend_jit_math_double_long(dasm_State **Dst, zend_op *opline) +{ + |.if X64 or SSE + || if (opline->opcode == ZEND_ADD || opline->opcode == ZEND_MUL) { + | SSE_LOAD_LONG opline->op2_type, opline->op2, xmm0 + | SSE_MATH opline->opcode, opline->op1_type, opline->op1, xmm0 + | SSE_STORE opline->result, xmm0 + || } else { + | SSE_LOAD opline->op1_type, opline->op1, xmm0 + | SSE_LOAD_LONG opline->op2_type, opline->op2, xmm1 + | SSE_MATH2 opline->opcode, xmm1, xmm0 + | SSE_STORE opline->result, xmm0 + || } + |.else + | LONGF_LOAD opline->op2_type, opline->op2 + || if (opline->opcode == ZEND_ADD || opline->opcode == ZEND_MUL) { + | FP_MATH opline->opcode, opline->op1_type, opline->op1 + || } else { + | FP_LOAD opline->op1_type, opline->op1 + | LONGF_MATH opline->opcode + || } + | FP_STORE opline->result + |.endif + + return 1; +} + +static int zend_jit_math_double_double(dasm_State **Dst, zend_op *opline) +{ + zend_bool same_ops = (opline->op1_type == opline->op2_type) && (opline->op1.var == opline->op2.var); + + |.if X64 or SSE + | SSE_LOAD opline->op1_type, opline->op1, xmm0 + if (same_ops) { + | SSE_MATH2 opline->opcode, xmm0, xmm0 + } else { + | SSE_MATH opline->opcode, opline->op2_type, opline->op2, xmm0 + } + | SSE_STORE opline->result, xmm0 + |.else + | FP_LOAD opline->op1_type, opline->op1 + | FP_MATH opline->opcode, opline->op2_type, opline->op2 + | FP_STORE opline->result + |.endif + + return 1; +} + static int zend_jit_math(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, int inc_op) { - uint32_t op1_info, op2_info, res_info; + uint32_t op1_info, op2_info; + zend_bool same_ops = (opline->op1_type == opline->op2_type) && (opline->op1.var == opline->op2.var); + zend_bool has_slow = 0; if (!ssa->ops || !ssa->var_info) { goto fallback; @@ -850,49 +1017,222 @@ static int zend_jit_math(dasm_State **Dst, zend_op *opline, zend_op_array *op_ar op1_info = OP1_INFO(); op2_info = OP2_INFO(); - if (((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) && - ((op2_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG)) { - res_info = RES_INFO(); - | LONG_LOAD opline->op1_type, opline->op1, r0 - | LONG_MATH opline->opcode, opline->op2_type, opline->op2, r0 - if ((res_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) { - | LONG_STORE opline->result, r0 - } else { - | jo >1 + if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) { + goto fallback; + } + + if (!(op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) || + !(op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { + goto fallback; + } + + if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)) { + if (op1_info & (MAY_BE_ANY-MAY_BE_LONG)) { + | cmp dword [FP + opline->op1.var + 8], IS_LONG + if (op1_info & MAY_BE_DOUBLE) { + | jne >4 + } else { + | JNE_SLOW + } + } + if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_LONG))) { + | cmp dword [FP + opline->op2.var + 8], IS_LONG + if (op2_info & MAY_BE_DOUBLE) { + | jne >3 + |.cold_codes + |3: + if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { + | cmp dword [FP + opline->op2.var + 8], IS_DOUBLE + | JNE_SLOW + } + if (!zend_jit_math_long_double(Dst, opline)) { + return 0; + } + | jmp >6 + |.code + } else { + | JNE_SLOW + } + } + if (!zend_jit_math_long_long(Dst, opline, op_array, ssa)) { + return 0; + } + if (op1_info & MAY_BE_DOUBLE) { |.cold_codes - |1: - | LONGF_LOAD opline->op2_type, opline->op2 - | LONGF_LOAD opline->op1_type, opline->op1 - | LONGF_MATH opline->opcode - | FP_STORE opline->result - | jmp >2 + |4: + if (op1_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { + | cmp dword [FP + opline->op1.var + 8], IS_DOUBLE + | JNE_SLOW + } + if (op2_info & MAY_BE_DOUBLE) { + if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { + | cmp dword [FP + opline->op2.var + 8], IS_DOUBLE + if (!same_ops) { + | jne >5 + } else { + | JNE_SLOW + } + } + if (!zend_jit_math_double_double(Dst, opline)) { + return 0; + } + | jmp >6 + } + if (!same_ops) { + |5: + if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { + | cmp dword [FP + opline->op2.var + 8], IS_LONG + | JNE_SLOW + } + if (!zend_jit_math_double_long(Dst, opline)) { + return 0; + } + | jmp >6 + } |.code - | LONG_STORE opline->result, r0 - |2: } - } else if (((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) && - ((op2_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE)) { - |.if X64 or SSE - | SSE_LOAD opline->op1_type, opline->op1, xmm0 - if (opline->op1_type == opline->op2_type && opline->op1.var == opline->op2.var) { - | SSE_MATH2 opline->opcode, xmm0, xmm0 - } else { - | SSE_MATH opline->opcode, opline->op2_type, opline->op2, xmm0 + } else if ((op1_info & MAY_BE_DOUBLE) && + !(op1_info & MAY_BE_LONG) && + (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { + if (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) { + | cmp dword [FP + opline->op1.var + 8], IS_DOUBLE + | JNE_SLOW } - | SSE_STORE opline->result, xmm0 - |.else - | FP_LOAD opline->op1_type, opline->op1 - | FP_MATH opline->opcode, opline->op2_type, opline->op2 - | FP_STORE opline->result - |.endif - } else { - goto fallback; - } + if (op2_info & MAY_BE_DOUBLE) { + if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { + | cmp dword [FP + opline->op2.var + 8], IS_DOUBLE + if (!same_ops && (op2_info & MAY_BE_LONG)) { + | jne >3 + } else { + | JNE_SLOW + } + } + if (!zend_jit_math_double_double(Dst, opline)) { + return 0; + } + } + if (!same_ops && (op2_info & MAY_BE_LONG)) { + if (op2_info & MAY_BE_DOUBLE) { + |.cold_codes + } + |3: + if (op2_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) { + | cmp dword [FP + opline->op2.var + 8], IS_LONG + | JNE_SLOW + } + if (!zend_jit_math_double_long(Dst, opline)) { + return 0; + } + if (op2_info & MAY_BE_DOUBLE) { + | jmp >6 + |.code + } + } + } else if ((op2_info & MAY_BE_DOUBLE) && + !(op2_info & MAY_BE_LONG) && + (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { + if (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) { + | cmp dword [FP + opline->op2.var + 8], IS_DOUBLE + | JNE_SLOW + } + if (op1_info & MAY_BE_DOUBLE) { + if (!same_ops && (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { + | cmp dword [FP + opline->op1.var + 8], IS_DOUBLE + if (!same_ops && (op1_info & MAY_BE_LONG)) { + | jne >3 + } else { + | JNE_SLOW + } + } + if (!zend_jit_math_double_double(Dst, opline)) { + return 0; + } + } + if (!same_ops && (op1_info & MAY_BE_LONG)) { + if (op1_info & MAY_BE_DOUBLE) { + |.cold_codes + } + |3: + if (op1_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) { + | cmp dword [FP + opline->op1.var + 8], IS_LONG + | JNE_SLOW + } + if (!zend_jit_math_long_double(Dst, opline)) { + return 0; + } + if (op1_info & MAY_BE_DOUBLE) { + | jmp >6 + |.code + } + } + } + + |6: if (inc_op) { | INC_IP } else { | RESET_IP } + + if (has_slow) { + |.cold_codes + |9: + | lea FCARG1a, [FP + opline->result.var] + | LOAD_ZVAL_ADDR FCARG2a, opline->op1_type, opline->op1 + |.if X64 + | LOAD_ZVAL_ADDR CARG3, opline->op2_type, opline->op2 + |.else + | PUSH_ZVAL_ADDR opline->op2_type, opline->op2, r0 + |.endif + || if (opline->opcode == ZEND_ADD) { + | EXT_CALL add_function, r0 + || } else if (opline->opcode == ZEND_SUB) { + | EXT_CALL sub_function, r0 + || } else if (opline->opcode == ZEND_MUL) { + | EXT_CALL mul_function, r0 + || } else if (opline->opcode == ZEND_DIV) { + | EXT_CALL div_function, r0 + || } else { + || ZEND_ASSERT(0); + || } + || if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && + || (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { + | and byte [FP + opline->op1.var + 9], IS_TYPE_REFCOUNTED + | je >1 + | mov FCARG1a, aword [FP + opline->op1.var] + | dec dword [FCARG1a] + | jnz >1 + || if (ZEND_DEBUG) { + || const char *filename = op_array->filename ? op_array->filename->val : NULL; + | LOAD_ADDR FCARG2a, filename + | push opline->lineno + || } + | EXT_CALL _zval_dtor_func, r0 + |1: + || } + || if ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) && + || (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { + | and byte [FP + opline->op2.var + 9], IS_TYPE_REFCOUNTED + | je >1 + | mov FCARG1a, aword [FP + opline->op2.var] + | dec dword [FCARG1a] + | dec dword [FP + opline->op2.var] + | jnz >1 + || if (ZEND_DEBUG) { + || const char *filename = op_array->filename ? op_array->filename->val : NULL; + | LOAD_ADDR FCARG2a, filename + | push opline->lineno + || } + | EXT_CALL _zval_dtor_func, r0 + |1: + || } + || if (zend_may_throw(opline, op_array, ssa)) { + || zend_jit_check_exception(Dst); + || } + | jmp <6 + |.code + } + return 1; fallback: From 9a50c5eacfeada826ac652ac776bd0d73578c3ab Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 12 Sep 2016 19:32:45 +0300 Subject: [PATCH 162/569] Fixed incorrect IP calculation --- ext/opcache/jit/zend_jit.c | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index d2f2c2285d1d9..51726dd4ffd49 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -678,8 +678,22 @@ static int zend_jit_op_array_analyze2(zend_op_array *op_array, zend_script *scri static int zend_need_inc_op(zend_op_array *op_array, zend_op *opline, int b, zend_ssa *ssa) { zend_op *next_opline = opline + 1; - if (next_opline->opcode == ZEND_JMP || - next_opline->opcode == ZEND_JMPZNZ) { + + switch (opline->opcode) { + case ZEND_IS_EQUAL: + case ZEND_IS_NOT_EQUAL: + case ZEND_IS_SMALLER: + case ZEND_IS_SMALLER_OR_EQUAL: + if ((next_opline->opcode == ZEND_JMPZ || + next_opline->opcode == ZEND_JMPNZ) && + next_opline->op1_type == IS_TMP_VAR && + next_opline->op1.var == opline->result.var) { + opline++; + } + } + next_opline = opline + 1; + if (next_opline->opcode == ZEND_JMP /*|| + next_opline->opcode == ZEND_JMPZNZ*/) { return 0; } else if (opline == (op_array->opcodes + ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len - 1)) { if ((ssa->cfg.blocks[b].successors[0] >= 0 && @@ -850,7 +864,7 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) case ZEND_IS_SMALLER: case ZEND_IS_SMALLER_OR_EQUAL: if (!zend_jit_cmp(&dasm_state, - opline, b, &i, op_array, ssa, zend_need_inc_op(op_array, opline + 1, b, ssa))) { + opline, b, &i, op_array, ssa, zend_need_inc_op(op_array, opline, b, ssa))) { goto jit_failure; } break; From 5e00936c42b26c377bef2188199aa3f57d533781 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 12 Sep 2016 19:33:22 +0300 Subject: [PATCH 163/569] Improved JIT for comparion opcodes (only MAY_BE_UNDEF triggers fallback) --- ext/opcache/jit/zend_jit_x86.dasc | 641 ++++++++++++++++++++++-------- 1 file changed, 470 insertions(+), 171 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index eb6d87b50e7c5..d326786a0f9f9 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -1177,6 +1177,7 @@ static int zend_jit_math(dasm_State **Dst, zend_op *opline, zend_op_array *op_ar if (has_slow) { |.cold_codes |9: + | mov EX->opline, IP | lea FCARG1a, [FP + opline->result.var] | LOAD_ZVAL_ADDR FCARG2a, opline->op1_type, opline->op1 |.if X64 @@ -1240,202 +1241,500 @@ fallback: return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); } +static int zend_jit_cmp_long_long(dasm_State **Dst, zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa) +{ + unsigned int target_label; + + | LONG_LOAD opline->op1_type, opline->op1, r0 + | LONG_OP cmp, opline->op2_type, opline->op2, r0 + if ((opline+1)->opcode == ZEND_JMPZ && + (opline+1)->op1_type == IS_TMP_VAR && + (opline+1)->op1.var == opline->result.var) { + target_label = ssa->cfg.blocks[b].successors[0]; + switch (opline->opcode) { + case ZEND_IS_EQUAL: + | jne => target_label + break; + case ZEND_IS_NOT_EQUAL: + | je => target_label + break; + case ZEND_IS_SMALLER: + | jge => target_label + break; + case ZEND_IS_SMALLER_OR_EQUAL: + | jg => target_label + break; + } + } else if ((opline+1)->opcode == ZEND_JMPNZ && + (opline+1)->op1_type == IS_TMP_VAR && + (opline+1)->op1.var == opline->result.var) { + target_label = ssa->cfg.blocks[b].successors[0]; + switch (opline->opcode) { + case ZEND_IS_EQUAL: + | je => target_label + break; + case ZEND_IS_NOT_EQUAL: + | jne => target_label + break; + case ZEND_IS_SMALLER: + | jl => target_label + break; + case ZEND_IS_SMALLER_OR_EQUAL: + | jle => target_label + break; + } + } else { + switch (opline->opcode) { + case ZEND_IS_EQUAL: + | sete al + | add eax, 2 + break; + case ZEND_IS_NOT_EQUAL: + | setne al + | add eax, 2 + break; + case ZEND_IS_SMALLER: + | setl al + | add eax, 2 + break; + case ZEND_IS_SMALLER_OR_EQUAL: + | setle al + | add eax, 2 + break; + } + | mov dword [FP + opline->result.var + 8], eax + } + + return 1; +} + +static int zend_jit_cmp_double_common(dasm_State **Dst, zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa) +{ + unsigned int target_label; + + if ((opline+1)->opcode == ZEND_JMPZ && + (opline+1)->op1_type == IS_TMP_VAR && + (opline+1)->op1.var == opline->result.var) { + target_label = ssa->cfg.blocks[b].successors[0]; + switch (opline->opcode) { + case ZEND_IS_EQUAL: + | jp >1 + | jne => target_label + |1: + break; + case ZEND_IS_NOT_EQUAL: + | jp >1 + | je => target_label + |1: + break; + case ZEND_IS_SMALLER: + | jae => target_label + break; + case ZEND_IS_SMALLER_OR_EQUAL: + | ja => target_label + break; + } + } else if ((opline+1)->opcode == ZEND_JMPNZ && + (opline+1)->op1_type == IS_TMP_VAR && + (opline+1)->op1.var == opline->result.var) { + target_label = ssa->cfg.blocks[b].successors[0]; + switch (opline->opcode) { + case ZEND_IS_EQUAL: + | jp >1 + | je => target_label + |1: + break; + case ZEND_IS_NOT_EQUAL: + | jp >1 + | jne => target_label + |1: + break; + case ZEND_IS_SMALLER: + | jnae => target_label + break; + case ZEND_IS_SMALLER_OR_EQUAL: + | jna => target_label + break; + } + } else { + switch (opline->opcode) { + case ZEND_IS_EQUAL: + | jp >1 + | mov eax, IS_TRUE + | je >2 + |1: + | mov eax, IS_FALSE + |2: + break; + case ZEND_IS_NOT_EQUAL: + | jp >1 + | mov eax, IS_FALSE + | je >2 + |1: + | mov eax, IS_TRUE + |2: + break; + case ZEND_IS_SMALLER: + | seta al + | add eax, 2 + break; + case ZEND_IS_SMALLER_OR_EQUAL: + | setae al + | add eax, 2 + break; + } + | mov dword [FP + opline->result.var + 8], eax + } + + return 1; +} + +static int zend_jit_cmp_long_double(dasm_State **Dst, zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa) +{ + |.if X64 or SSE + | SSE_LOAD_LONG opline->op1_type, opline->op1, xmm0 + | SSE_OP ucomisd, opline->op2_type, opline->op2, xmm0 + |.else + | FP_LOAD opline->op2_type, opline->op2 + | LONGF_LOAD opline->op1_type, opline->op1 + | fucomip st1 + | fstp st0 + |.endif + + return zend_jit_cmp_double_common(Dst, opline, b, op_array, ssa); +} + +static int zend_jit_cmp_double_long(dasm_State **Dst, zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa) +{ + |.if X64 or SSE + | SSE_LOAD opline->op1_type, opline->op1, xmm0 + | SSE_LOAD_LONG opline->op2_type, opline->op2, xmm1 + | ucomisd xmm0, xmm1 + |.else + | LONGF_LOAD opline->op2_type, opline->op2 + | FP_LOAD opline->op1_type, opline->op1 + | fucomip st1 + | fstp st0 + |.endif + + return zend_jit_cmp_double_common(Dst, opline, b, op_array, ssa); +} + +static int zend_jit_cmp_double_double(dasm_State **Dst, zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa) +{ + |.if X64 or SSE + | SSE_LOAD opline->op1_type, opline->op1, xmm0 + | SSE_OP ucomisd, opline->op2_type, opline->op2, xmm0 + |.else + | FP_LOAD opline->op2_type, opline->op2 + | FP_LOAD opline->op1_type, opline->op1 + | fucomip st1 + | fstp st0 + |.endif + + return zend_jit_cmp_double_common(Dst, opline, b, op_array, ssa); +} + +static int zend_jit_cmp_slow(dasm_State **Dst, zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa) +{ + unsigned int target_label; + + | cmp aword [FP + opline->result.var], 0 + if ((opline+1)->opcode == ZEND_JMPZ && + (opline+1)->op1_type == IS_TMP_VAR && + (opline+1)->op1.var == opline->result.var) { + target_label = ssa->cfg.blocks[b].successors[0]; + switch (opline->opcode) { + case ZEND_IS_EQUAL: + | jne => target_label + break; + case ZEND_IS_NOT_EQUAL: + | je => target_label + break; + case ZEND_IS_SMALLER: + | jge => target_label + break; + case ZEND_IS_SMALLER_OR_EQUAL: + | jg => target_label + break; + } + } else if ((opline+1)->opcode == ZEND_JMPNZ && + (opline+1)->op1_type == IS_TMP_VAR && + (opline+1)->op1.var == opline->result.var) { + target_label = ssa->cfg.blocks[b].successors[0]; + switch (opline->opcode) { + case ZEND_IS_EQUAL: + | je => target_label + break; + case ZEND_IS_NOT_EQUAL: + | jne => target_label + break; + case ZEND_IS_SMALLER: + | jl => target_label + break; + case ZEND_IS_SMALLER_OR_EQUAL: + | jle => target_label + break; + } + } else { + switch (opline->opcode) { + case ZEND_IS_EQUAL: + | sete al + | add eax, 2 + break; + case ZEND_IS_NOT_EQUAL: + | setne al + | add eax, 2 + break; + case ZEND_IS_SMALLER: + | setl al + | add eax, 2 + break; + case ZEND_IS_SMALLER_OR_EQUAL: + | setle al + | add eax, 2 + break; + } + | mov dword [FP + opline->result.var + 8], eax + } + + return 1; +} + static int zend_jit_cmp(dasm_State **Dst, zend_op *opline, int b, int *opnum, zend_op_array *op_array, zend_ssa *ssa, int inc_op) { uint32_t op1_info, op2_info; - unsigned int target_label; + zend_bool same_ops = (opline->op1_type == opline->op2_type) && (opline->op1.var == opline->op2.var); + zend_bool has_slow = 0; if (!ssa->ops || !ssa->var_info) { goto fallback; } + op1_info = OP1_INFO(); op2_info = OP2_INFO(); - if (((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) && - ((op2_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG)) { - | LONG_LOAD opline->op1_type, opline->op1, r0 - | LONG_OP cmp, opline->op2_type, opline->op2, r0 - if ((opline+1)->opcode == ZEND_JMPZ && - (opline+1)->op1_type == IS_TMP_VAR && - (opline+1)->op1.var == opline->result.var) { - (*opnum)++; - target_label = ssa->cfg.blocks[b].successors[0]; - switch (opline->opcode) { - case ZEND_IS_EQUAL: - | jne => target_label - break; - case ZEND_IS_NOT_EQUAL: - | je => target_label - break; - case ZEND_IS_SMALLER: - | jg => target_label - break; - case ZEND_IS_SMALLER_OR_EQUAL: - | jge => target_label - break; + if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) { + goto fallback; + } + + if (!(op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) || + !(op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { + goto fallback; + } + + if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)) { + if (op1_info & (MAY_BE_ANY-MAY_BE_LONG)) { + | cmp dword [FP + opline->op1.var + 8], IS_LONG + if (op1_info & MAY_BE_DOUBLE) { + | jne >4 + } else { + | JNE_SLOW } - if (!(ssa->cfg.blocks[ssa->cfg.blocks[b].successors[1]].flags & ZEND_BB_TARGET)) { - | INC_IP - | INC_IP + } + if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_LONG))) { + | cmp dword [FP + opline->op2.var + 8], IS_LONG + if (op2_info & MAY_BE_DOUBLE) { + | jne >3 + |.cold_codes + |3: + if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { + | cmp dword [FP + opline->op2.var + 8], IS_DOUBLE + | JNE_SLOW + } + if (!zend_jit_cmp_long_double(Dst, opline, b, op_array, ssa)) { + return 0; + } + | jmp >6 + |.code } else { - | RESET_IP + | JNE_SLOW } - } else if ((opline+1)->opcode == ZEND_JMPNZ && - (opline+1)->op1_type == IS_TMP_VAR && - (opline+1)->op1.var == opline->result.var) { - (*opnum)++; - target_label = ssa->cfg.blocks[b].successors[0]; - switch (opline->opcode) { - case ZEND_IS_EQUAL: - | je => target_label - break; - case ZEND_IS_NOT_EQUAL: - | jne => target_label - break; - case ZEND_IS_SMALLER: - | jl => target_label - break; - case ZEND_IS_SMALLER_OR_EQUAL: - | jle => target_label - break; + } + if (!zend_jit_cmp_long_long(Dst, opline, b, op_array, ssa)) { + return 0; + } + if (op1_info & MAY_BE_DOUBLE) { + |.cold_codes + |4: + if (op1_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { + | cmp dword [FP + opline->op1.var + 8], IS_DOUBLE + | JNE_SLOW } - if (!(ssa->cfg.blocks[ssa->cfg.blocks[b].successors[1]].flags & ZEND_BB_TARGET)) { - | INC_IP - | INC_IP - } else { - | RESET_IP + if (op2_info & MAY_BE_DOUBLE) { + if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { + | cmp dword [FP + opline->op2.var + 8], IS_DOUBLE + if (!same_ops) { + | jne >5 + } else { + | JNE_SLOW + } + } + if (!zend_jit_cmp_double_double(Dst, opline, b, op_array, ssa)) { + return 0; + } + | jmp >6 } - } else { - switch (opline->opcode) { - case ZEND_IS_EQUAL: - | sete al - | add eax, 2 - case ZEND_IS_NOT_EQUAL: - | setne al - | add eax, 2 - break; - case ZEND_IS_SMALLER: - | setl al - | add eax, 2 - break; - case ZEND_IS_SMALLER_OR_EQUAL: - | setle al - | add eax, 2 - break; + if (!same_ops) { + |5: + if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { + | cmp dword [FP + opline->op2.var + 8], IS_LONG + | JNE_SLOW + } + if (!zend_jit_cmp_double_long(Dst, opline, b, op_array, ssa)) { + return 0; + } + | jmp >6 } - | mov dword [FP + opline->result.var + 8], eax - if (inc_op) { - | INC_IP - } else { - | RESET_IP + |.code + } + } else if ((op1_info & MAY_BE_DOUBLE) && + !(op1_info & MAY_BE_LONG) && + (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { + if (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) { + | cmp dword [FP + opline->op1.var + 8], IS_DOUBLE + | JNE_SLOW + } + if (op2_info & MAY_BE_DOUBLE) { + if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { + | cmp dword [FP + opline->op2.var + 8], IS_DOUBLE + if (!same_ops && (op2_info & MAY_BE_LONG)) { + | jne >3 + } else { + | JNE_SLOW + } + } + if (!zend_jit_cmp_double_double(Dst, opline, b, op_array, ssa)) { + return 0; } } - } else if (((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) && - ((op2_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE)) { - |.if X64 or SSE - | SSE_LOAD opline->op1_type, opline->op1, xmm0 - | SSE_OP ucomisd, opline->op2_type, opline->op2, xmm0 - |.else - | FP_LOAD opline->op2_type, opline->op2 - | FP_LOAD opline->op1_type, opline->op1 - | fucomip st1 - | fstp st0 - |.endif - if ((opline+1)->opcode == ZEND_JMPZ && - (opline+1)->op1_type == IS_TMP_VAR && - (opline+1)->op1.var == opline->result.var) { - (*opnum)++; - target_label = ssa->cfg.blocks[b].successors[0]; - switch (opline->opcode) { - case ZEND_IS_EQUAL: - | jp >1 - | je => target_label - |1: - break; - case ZEND_IS_NOT_EQUAL: - | jp >1 - | jne => target_label - |1: - break; - case ZEND_IS_SMALLER: - | ja => target_label - break; - case ZEND_IS_SMALLER_OR_EQUAL: - | jae => target_label - break; + if (!same_ops && (op2_info & MAY_BE_LONG)) { + if (op2_info & MAY_BE_DOUBLE) { + |.cold_codes } - if (!(ssa->cfg.blocks[ssa->cfg.blocks[b].successors[1]].flags & ZEND_BB_TARGET)) { - | INC_IP - | INC_IP - } else { - | RESET_IP + |3: + if (op2_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) { + | cmp dword [FP + opline->op2.var + 8], IS_LONG + | JNE_SLOW } - } else if ((opline+1)->opcode == ZEND_JMPNZ && - (opline+1)->op1_type == IS_TMP_VAR && - (opline+1)->op1.var == opline->result.var) { - (*opnum)++; - target_label = ssa->cfg.blocks[b].successors[0]; - switch (opline->opcode) { - case ZEND_IS_EQUAL: - | jp >1 - | jne => target_label - |1: - break; - case ZEND_IS_NOT_EQUAL: - | jp >1 - | je => target_label - |1: - break; - case ZEND_IS_SMALLER: - | jna => target_label - break; - case ZEND_IS_SMALLER_OR_EQUAL: - | jnae => target_label - break; + if (!zend_jit_cmp_double_long(Dst, opline, b, op_array, ssa)) { + return 0; } - if (!(ssa->cfg.blocks[ssa->cfg.blocks[b].successors[1]].flags & ZEND_BB_TARGET)) { - | INC_IP - | INC_IP - } else { - | RESET_IP + if (op2_info & MAY_BE_DOUBLE) { + | jmp >6 + |.code } - } else { - switch (opline->opcode) { - case ZEND_IS_EQUAL: - | jp >1 - | mov eax, IS_TRUE - | je >2 - |1: - | mov eax, IS_FALSE - |2: - break; - case ZEND_IS_NOT_EQUAL: - | jp >1 - | mov eax, IS_TRUE - | jne >2 - |1: - | mov eax, IS_FALSE - |2: - break; - case ZEND_IS_SMALLER: - | seta al - | add eax, 2 - break; - case ZEND_IS_SMALLER_OR_EQUAL: - | setae al - | add eax, 2 - break; + } + } else if ((op2_info & MAY_BE_DOUBLE) && + !(op2_info & MAY_BE_LONG) && + (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { + if (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) { + | cmp dword [FP + opline->op2.var + 8], IS_DOUBLE + | JNE_SLOW + } + if (op1_info & MAY_BE_DOUBLE) { + if (!same_ops && (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { + | cmp dword [FP + opline->op1.var + 8], IS_DOUBLE + if (!same_ops && (op1_info & MAY_BE_LONG)) { + | jne >3 + } else { + | JNE_SLOW + } } - | mov dword [FP + opline->result.var + 8], eax - if (inc_op) { - | INC_IP - } else { - | RESET_IP + if (!zend_jit_cmp_double_double(Dst, opline, b, op_array, ssa)) { + return 0; } } - } else { - goto fallback; - } + if (!same_ops && (op1_info & MAY_BE_LONG)) { + if (op1_info & MAY_BE_DOUBLE) { + |.cold_codes + } + |3: + if (op1_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) { + | cmp dword [FP + opline->op1.var + 8], IS_LONG + | JNE_SLOW + } + if (!zend_jit_cmp_long_double(Dst, opline, b, op_array, ssa)) { + return 0; + } + if (op1_info & MAY_BE_DOUBLE) { + | jmp >6 + |.code + } + } + } + + |6: + if (((opline+1)->opcode == ZEND_JMPZ || (opline+1)->opcode == ZEND_JMPNZ) && + (opline+1)->op1_type == IS_TMP_VAR && + (opline+1)->op1.var == opline->result.var) { + (*opnum)++; + if (inc_op) { + | INC_IP + | INC_IP + } else { + | RESET_IP + } + } else { + if (inc_op) { + | INC_IP + } else { + | RESET_IP + } + } + + if (has_slow) { + |.cold_codes + |9: + | mov EX->opline, IP + | lea FCARG1a, [FP + opline->result.var] + | LOAD_ZVAL_ADDR FCARG2a, opline->op1_type, opline->op1 + |.if X64 + | LOAD_ZVAL_ADDR CARG3, opline->op2_type, opline->op2 + |.else + | PUSH_ZVAL_ADDR opline->op2_type, opline->op2, r0 + |.endif + | EXT_CALL compare_function, r0 + || if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && + || (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { + | and byte [FP + opline->op1.var + 9], IS_TYPE_REFCOUNTED + | je >1 + | mov FCARG1a, aword [FP + opline->op1.var] + | dec dword [FCARG1a] + | jnz >1 + || if (ZEND_DEBUG) { + || const char *filename = op_array->filename ? op_array->filename->val : NULL; + | LOAD_ADDR FCARG2a, filename + | push opline->lineno + || } + | EXT_CALL _zval_dtor_func, r0 + |1: + || } + || if ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) && + || (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { + | and byte [FP + opline->op2.var + 9], IS_TYPE_REFCOUNTED + | je >1 + | mov FCARG1a, aword [FP + opline->op2.var] + | dec dword [FCARG1a] + | dec dword [FP + opline->op2.var] + | jnz >1 + || if (ZEND_DEBUG) { + || const char *filename = op_array->filename ? op_array->filename->val : NULL; + | LOAD_ADDR FCARG2a, filename + | push opline->lineno + || } + | EXT_CALL _zval_dtor_func, r0 + |1: + || } + || if (zend_may_throw(opline, op_array, ssa)) { + || zend_jit_check_exception(Dst); + || } + if (!zend_jit_cmp_slow(Dst, opline, b, op_array, ssa)) { + return 0; + } + | jmp <6 + |.code + } return 1; From 07df93a2fb65a208b00517f5d9e51deb18e0be0f Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 12 Sep 2016 20:09:25 +0300 Subject: [PATCH 164/569] Optimized DO_UCALL for recursive calls (use direct jmp insted of indirect) --- ext/opcache/jit/zend_jit_x86.dasc | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index d326786a0f9f9..fd1be18c71584 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -1995,8 +1995,13 @@ static int zend_jit_do_fcall(dasm_State **Dst, zend_op *opline, zend_op_array *o | // EX(opline) = op_array->opcodes; | LOAD_ADDR IP, (func->op_array.opcodes + call_info->num_args) | mov EX:r0->opline, IP - | add r4, SPAD // stack alignment - | jmp aword [IP] + || if (func && op_array == &func->op_array) { + || /* recursive call */ + | jmp =>call_info->num_args + || } else { + | add r4, SPAD // stack alignment + | jmp aword [IP] + || } return 1; From 79f97d6897c1dac2fd33abe99caaccd589e35bb3 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 12 Sep 2016 21:11:10 +0300 Subject: [PATCH 165/569] Renamed section ".cold_codes" into ".cold_code" --- ext/opcache/jit/zend_jit_x86.dasc | 32 +++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index fd1be18c71584..4830840e43e1c 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -65,7 +65,7 @@ |.globals zend_lb static void* dasm_labels[zend_lb_MAX]; -|.section code, cold_codes +|.section code, cold_code |.macro LOAD_ADDR, reg, addr |.if X64 @@ -830,7 +830,7 @@ static int zend_jit_inc_dec(dasm_State **Dst, zend_op *opline, zend_op_array *op || opline->result_type != IS_UNUSED) { | ZVAL_COPY_VALUE FP, opline->result.var, FP, opline->op1.var, MAY_BE_LONG, r0, eax, r1 || } - |.cold_codes + |.cold_code |1: if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { |.if X64 @@ -863,7 +863,7 @@ static int zend_jit_inc_dec(dasm_State **Dst, zend_op *opline, zend_op_array *op || } } if (op1_info & (MAY_BE_ANY-MAY_BE_LONG)) { - |.cold_codes + |.cold_code |2: | lea FCARG1a, [FP + opline->op1.var] | // ZVAL_DEREF(var_ptr); @@ -919,7 +919,7 @@ static int zend_jit_math_long_long(dasm_State **Dst, zend_op *opline, zend_op_ar | LONG_STORE opline->result, r0 } else { | jo >1 - |.cold_codes + |.cold_code |1: |.if X64 or SSE | SSE_LOAD_LONG opline->op1_type, opline->op1, xmm0 @@ -1039,7 +1039,7 @@ static int zend_jit_math(dasm_State **Dst, zend_op *opline, zend_op_array *op_ar | cmp dword [FP + opline->op2.var + 8], IS_LONG if (op2_info & MAY_BE_DOUBLE) { | jne >3 - |.cold_codes + |.cold_code |3: if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { | cmp dword [FP + opline->op2.var + 8], IS_DOUBLE @@ -1058,7 +1058,7 @@ static int zend_jit_math(dasm_State **Dst, zend_op *opline, zend_op_array *op_ar return 0; } if (op1_info & MAY_BE_DOUBLE) { - |.cold_codes + |.cold_code |4: if (op1_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { | cmp dword [FP + opline->op1.var + 8], IS_DOUBLE @@ -1113,7 +1113,7 @@ static int zend_jit_math(dasm_State **Dst, zend_op *opline, zend_op_array *op_ar } if (!same_ops && (op2_info & MAY_BE_LONG)) { if (op2_info & MAY_BE_DOUBLE) { - |.cold_codes + |.cold_code } |3: if (op2_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) { @@ -1150,7 +1150,7 @@ static int zend_jit_math(dasm_State **Dst, zend_op *opline, zend_op_array *op_ar } if (!same_ops && (op1_info & MAY_BE_LONG)) { if (op1_info & MAY_BE_DOUBLE) { - |.cold_codes + |.cold_code } |3: if (op1_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) { @@ -1175,7 +1175,7 @@ static int zend_jit_math(dasm_State **Dst, zend_op *opline, zend_op_array *op_ar } if (has_slow) { - |.cold_codes + |.cold_code |9: | mov EX->opline, IP | lea FCARG1a, [FP + opline->result.var] @@ -1536,7 +1536,7 @@ static int zend_jit_cmp(dasm_State **Dst, zend_op *opline, int b, int *opnum, ze | cmp dword [FP + opline->op2.var + 8], IS_LONG if (op2_info & MAY_BE_DOUBLE) { | jne >3 - |.cold_codes + |.cold_code |3: if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { | cmp dword [FP + opline->op2.var + 8], IS_DOUBLE @@ -1555,7 +1555,7 @@ static int zend_jit_cmp(dasm_State **Dst, zend_op *opline, int b, int *opnum, ze return 0; } if (op1_info & MAY_BE_DOUBLE) { - |.cold_codes + |.cold_code |4: if (op1_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { | cmp dword [FP + opline->op1.var + 8], IS_DOUBLE @@ -1610,7 +1610,7 @@ static int zend_jit_cmp(dasm_State **Dst, zend_op *opline, int b, int *opnum, ze } if (!same_ops && (op2_info & MAY_BE_LONG)) { if (op2_info & MAY_BE_DOUBLE) { - |.cold_codes + |.cold_code } |3: if (op2_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) { @@ -1647,7 +1647,7 @@ static int zend_jit_cmp(dasm_State **Dst, zend_op *opline, int b, int *opnum, ze } if (!same_ops && (op1_info & MAY_BE_LONG)) { if (op1_info & MAY_BE_DOUBLE) { - |.cold_codes + |.cold_code } |3: if (op1_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) { @@ -1684,7 +1684,7 @@ static int zend_jit_cmp(dasm_State **Dst, zend_op *opline, int b, int *opnum, ze } if (has_slow) { - |.cold_codes + |.cold_code |9: | mov EX->opline, IP | lea FCARG1a, [FP + opline->result.var] @@ -1843,7 +1843,7 @@ static int zend_jit_push_call_frame(dasm_State **Dst, zend_op *opline, zend_op_a | mov aword EX:r1->This.value.ptr, 0 | // ZEND_CALL_NUM_ARGS(call) = num_args; | mov dword EX:r1->This.u2.num_args, opline->extended_value - |.cold_codes + |.cold_code |2: | mov FCARG1d, used_stack | mov FCARG2a, r0 @@ -1884,7 +1884,7 @@ static int zend_jit_init_fcall(dasm_State **Dst, zend_op *opline, zend_op_array | mov r0, aword [r0 + Z_CACHE_SLOT_P(zv)] | test r0, r0 | jz >1 - |.cold_codes + |.cold_code |1: | // SAVE_OPLINE(); | mov EX->opline, IP From 5479a42154e98cefa11b34dcf78635d91aefb292 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 12 Sep 2016 21:55:02 +0300 Subject: [PATCH 166/569] Simplified delayed IP update --- ext/opcache/jit/zend_jit.c | 72 +++---------- ext/opcache/jit/zend_jit_x86.dasc | 172 ++++++++++-------------------- 2 files changed, 66 insertions(+), 178 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 51726dd4ffd49..571d5599f9492 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -676,36 +676,6 @@ static int zend_jit_op_array_analyze2(zend_op_array *op_array, zend_script *scri return SUCCESS; } -static int zend_need_inc_op(zend_op_array *op_array, zend_op *opline, int b, zend_ssa *ssa) { - zend_op *next_opline = opline + 1; - - switch (opline->opcode) { - case ZEND_IS_EQUAL: - case ZEND_IS_NOT_EQUAL: - case ZEND_IS_SMALLER: - case ZEND_IS_SMALLER_OR_EQUAL: - if ((next_opline->opcode == ZEND_JMPZ || - next_opline->opcode == ZEND_JMPNZ) && - next_opline->op1_type == IS_TMP_VAR && - next_opline->op1.var == opline->result.var) { - opline++; - } - } - next_opline = opline + 1; - if (next_opline->opcode == ZEND_JMP /*|| - next_opline->opcode == ZEND_JMPZNZ*/) { - return 0; - } else if (opline == (op_array->opcodes + ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len - 1)) { - if ((ssa->cfg.blocks[b].successors[0] >= 0 && - (ssa->cfg.blocks[ssa->cfg.blocks[b].successors[0]].flags & ZEND_BB_TARGET)) && - (ssa->cfg.blocks[b].successors[1] < 0 || - (ssa->cfg.blocks[ssa->cfg.blocks[b].successors[1]].flags & ZEND_BB_TARGET))) { - return 0; - } - } - return 1; -} - static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) { int b, i, end; @@ -810,8 +780,7 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) case ZEND_PRE_DEC: case ZEND_POST_INC: case ZEND_POST_DEC: - if (!zend_jit_inc_dec(&dasm_state, - opline, op_array, ssa, zend_need_inc_op(op_array, opline, b, ssa))) { + if (!zend_jit_inc_dec(&dasm_state, opline, op_array, ssa)) { goto jit_failure; } break; @@ -819,38 +788,32 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) case ZEND_SUB: case ZEND_MUL: // case ZEND_DIV: // TODO: check for division by zero ??? - if (!zend_jit_math(&dasm_state, - opline, op_array, ssa, zend_need_inc_op(op_array, opline, b, ssa))) { + if (!zend_jit_math(&dasm_state, opline, op_array, ssa)) { goto jit_failure; } break; case ZEND_ASSIGN: - if (!zend_jit_assign(&dasm_state, - opline, op_array, ssa, zend_need_inc_op(op_array, opline, b, ssa))) { + if (!zend_jit_assign(&dasm_state, opline, op_array, ssa)) { goto jit_failure; } break; case ZEND_QM_ASSIGN: - if (!zend_jit_qm_assign(&dasm_state, - opline, op_array, ssa, zend_need_inc_op(op_array, opline, b, ssa))) { + if (!zend_jit_qm_assign(&dasm_state, opline, op_array, ssa)) { goto jit_failure; } break; case ZEND_INIT_FCALL: - if (!zend_jit_init_fcall(&dasm_state, - opline, op_array, call_level, zend_need_inc_op(op_array, opline, b, ssa))) { + if (!zend_jit_init_fcall(&dasm_state, opline, op_array, call_level)) { goto jit_failure; } break; case ZEND_SEND_VAL: - if (!zend_jit_send_val(&dasm_state, - opline, op_array, ssa, zend_need_inc_op(op_array, opline, b, ssa))) { + if (!zend_jit_send_val(&dasm_state, opline, op_array, ssa)) { goto jit_failure; } break; case ZEND_SEND_VAR: - if (!zend_jit_send_var(&dasm_state, - opline, op_array, ssa, zend_need_inc_op(op_array, opline, b, ssa))) { + if (!zend_jit_send_var(&dasm_state, opline, op_array, ssa)) { goto jit_failure; } break; @@ -863,14 +826,12 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) case ZEND_IS_NOT_EQUAL: case ZEND_IS_SMALLER: case ZEND_IS_SMALLER_OR_EQUAL: - if (!zend_jit_cmp(&dasm_state, - opline, b, &i, op_array, ssa, zend_need_inc_op(op_array, opline, b, ssa))) { + if (!zend_jit_cmp(&dasm_state, opline, b, &i, op_array, ssa)) { goto jit_failure; } break; case ZEND_TYPE_CHECK: - if (!zend_jit_type_check(&dasm_state, - opline, b, &i, op_array, ssa, zend_need_inc_op(op_array, opline, b, ssa))) { + if (!zend_jit_type_check(&dasm_state, opline, b, &i, op_array, ssa)) { goto jit_failure; } break; @@ -892,16 +853,8 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) } break; case ZEND_NOP: - { - uint32_t skip = 1; - while (i < end && (opline + skip)->opcode == ZEND_NOP) { - i++; - skip++; - } - if (!zend_jit_skip_handler(&dasm_state, - skip, zend_need_inc_op(op_array, opline + skip - 1, b, ssa))) { - goto jit_failure; - } + if (!zend_jit_skip_handler(&dasm_state)) { + goto jit_failure; } break; case ZEND_OP_DATA: @@ -993,8 +946,7 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) } break; case ZEND_NEW: - if (!zend_jit_new(&dasm_state, - opline, &i, op_array, ssa, zend_need_inc_op(op_array, opline, b, ssa))) { + if (!zend_jit_new(&dasm_state, opline, &i, op_array, ssa)) { goto jit_failure; } break; diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 4830840e43e1c..0ce31e7490cc0 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -499,28 +499,12 @@ static void* dasm_labels[zend_lb_MAX]; || } |.endmacro -|.macro RESET_IP -||ip_count = 0; -|.endmacro - -|.macro INC_IP -||ip_count++; -|.endmacro - -|.macro SAVE_IP -||if (ip_count) { -| add IP, sizeof(zend_op) * ip_count -| RESET_IP -||} -|.endmacro - - |.macro JNE_SLOW | jne >9 || has_slow = 1; |.endmacro -static uint32_t ip_count = 0; +static uint32_t valid_opline_offset; #endif /* bit helpers */ @@ -589,7 +573,6 @@ static int zend_jit_interrupt_handler_stub(dasm_State **Dst) | //} else if (zend_interrupt_function) { if (zend_interrupt_function) { | //SAVE_OPLINE(); - | SAVE_IP | mov EX->opline, IP | //zend_interrupt_function(execute_data); |.if X64 @@ -628,7 +611,7 @@ static const zend_jit_stub zend_jit_stubs[] = { static int zend_jit_align_func(dasm_State **Dst) { - | RESET_IP + valid_opline_offset = 0; |.align 16 return 1; } @@ -659,9 +642,20 @@ static int zend_jit_check_exception(dasm_State **Dst) return 1; } +static int zend_jit_set_valid_ip(dasm_State **Dst) +{ + if (valid_opline_offset) { + | add IP, sizeof(zend_op) * valid_opline_offset + valid_opline_offset = 0; + } + return 1; +} + static int zend_jit_handler(dasm_State **Dst, zend_op *opline, int may_throw) { - | SAVE_IP + if (!zend_jit_set_valid_ip(Dst)) { + return 0; + } | EXT_CALL opline->handler, r0 if (may_throw) { zend_jit_check_exception(Dst); @@ -671,38 +665,36 @@ static int zend_jit_handler(dasm_State **Dst, zend_op *opline, int may_throw) static int zend_jit_tail_handler(dasm_State **Dst, zend_op *opline) { - | SAVE_IP + if (!zend_jit_set_valid_ip(Dst)) { + return 0; + } | add r4, SPAD // stack alignment | EXT_JMP opline->handler, r0 return 1; } -static int zend_jit_skip_handler(dasm_State **Dst, uint32_t skip, int inc_op) +static int zend_jit_skip_handler(dasm_State **Dst) { - if (inc_op) { - while (skip--) { - | INC_IP - } - } + valid_opline_offset++; return 1; } static int zend_jit_set_opline(dasm_State **Dst, zend_op *target_opline) { + valid_opline_offset = 0; | LOAD_ADDR IP, target_opline return 1; } static int zend_jit_jmp(dasm_State **Dst, unsigned int target_label) { - | RESET_IP + valid_opline_offset = 0; | jmp =>target_label return 1; } static int zend_jit_cond_jmp(dasm_State **Dst, zend_op *next_opline, unsigned int target_label) { - | SAVE_IP | cmp IPl, next_opline | jnz =>target_label @@ -714,7 +706,6 @@ static int zend_jit_smart_branch(dasm_State **Dst, zend_op *opline, unsigned int zend_op *next_opline = opline + 1; zend_op *target_opline = OP_JMP_ADDR(opline, opline->op2); - | SAVE_IP | cmp IPl, next_opline | jz =>next_label | cmp IPl, target_opline @@ -752,7 +743,7 @@ static int zend_jit_call(dasm_State **Dst, zend_op *opline) #endif } -static int zend_jit_new(dasm_State **Dst, zend_op *opline, int *opnum, zend_op_array *op_array, zend_ssa *ssa, int inc_op) +static int zend_jit_new(dasm_State **Dst, zend_op *opline, int *opnum, zend_op_array *op_array, zend_ssa *ssa) { if (!zend_jit_handler(Dst, opline, 1)) { return 0; @@ -782,10 +773,8 @@ static int zend_jit_new(dasm_State **Dst, zend_op *opline, int *opnum, zend_op_a | jnz >1 zend_jit_call(Dst, next_opline); |1: - } else if (inc_op) { - | INC_IP } else { - | RESET_IP + valid_opline_offset++; } } return 1; @@ -793,7 +782,7 @@ static int zend_jit_new(dasm_State **Dst, zend_op *opline, int *opnum, zend_op_a #if ZEND_JIT_LEVEL >= ZEND_JIT_LEVEL_OPT_FUNC -static int zend_jit_inc_dec(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, int inc_op) +static int zend_jit_inc_dec(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) { uint32_t op1_info, op1_def_info; @@ -892,11 +881,7 @@ static int zend_jit_inc_dec(dasm_State **Dst, zend_op *opline, zend_op_array *op |.code } |3: - if (inc_op) { - | INC_IP - } else { - | RESET_IP - } + valid_opline_offset++; return 1; fallback: @@ -1004,7 +989,7 @@ static int zend_jit_math_double_double(dasm_State **Dst, zend_op *opline) return 1; } -static int zend_jit_math(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, int inc_op) +static int zend_jit_math(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) { uint32_t op1_info, op2_info; zend_bool same_ops = (opline->op1_type == opline->op2_type) && (opline->op1.var == opline->op2.var); @@ -1168,11 +1153,7 @@ static int zend_jit_math(dasm_State **Dst, zend_op *opline, zend_op_array *op_ar } |6: - if (inc_op) { - | INC_IP - } else { - | RESET_IP - } + valid_opline_offset++; if (has_slow) { |.cold_code @@ -1501,7 +1482,7 @@ static int zend_jit_cmp_slow(dasm_State **Dst, zend_op *opline, int b, zend_op_a return 1; } -static int zend_jit_cmp(dasm_State **Dst, zend_op *opline, int b, int *opnum, zend_op_array *op_array, zend_ssa *ssa, int inc_op) +static int zend_jit_cmp(dasm_State **Dst, zend_op *opline, int b, int *opnum, zend_op_array *op_array, zend_ssa *ssa) { uint32_t op1_info, op2_info; zend_bool same_ops = (opline->op1_type == opline->op2_type) && (opline->op1.var == opline->op2.var); @@ -1669,18 +1650,9 @@ static int zend_jit_cmp(dasm_State **Dst, zend_op *opline, int b, int *opnum, ze (opline+1)->op1_type == IS_TMP_VAR && (opline+1)->op1.var == opline->result.var) { (*opnum)++; - if (inc_op) { - | INC_IP - | INC_IP - } else { - | RESET_IP - } + valid_opline_offset += 2; } else { - if (inc_op) { - | INC_IP - } else { - | RESET_IP - } + valid_opline_offset++; } if (has_slow) { @@ -1743,7 +1715,7 @@ fallback: return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); } -static int zend_jit_simple_assign(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, uint32_t var, uint32_t var_info, zend_uchar val_type, znode_op val, uint32_t val_info, int inc_op) +static int zend_jit_simple_assign(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, uint32_t var, uint32_t var_info, zend_uchar val_type, znode_op val, uint32_t val_info) { if (val_info & (MAY_BE_UNDEF|MAY_BE_REF)) { /* TODO: Support for references ??? */ @@ -1754,11 +1726,7 @@ static int zend_jit_simple_assign(dasm_State **Dst, zend_op *opline, zend_op_arr || if (Z_REFCOUNTED_P(zv)) { | ADDREF_CONST zv, r0 || } - if (inc_op) { - | INC_IP - } else { - | RESET_IP - } + valid_opline_offset++; return 1; } @@ -1766,11 +1734,7 @@ static int zend_jit_simple_assign(dasm_State **Dst, zend_op *opline, zend_op_arr || if (val_type == IS_CV) { | TRY_ADDREF val_info, ah, r1 || } - if (inc_op) { - | INC_IP - } else { - | RESET_IP - } + valid_opline_offset++; return 1; @@ -1779,7 +1743,7 @@ fallback: return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); } -static int zend_jit_qm_assign(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, int inc_op) +static int zend_jit_qm_assign(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) { uint32_t op1_info; @@ -1789,14 +1753,14 @@ static int zend_jit_qm_assign(dasm_State **Dst, zend_op *opline, zend_op_array * op1_info = OP1_INFO(); - return zend_jit_simple_assign(Dst, opline, op_array, ssa, opline->result.var, -1, opline->op1_type, opline->op1, op1_info, inc_op); + return zend_jit_simple_assign(Dst, opline, op_array, ssa, opline->result.var, -1, opline->op1_type, opline->op1, op1_info); fallback: /* fallback to subroutine threading */ return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); } -static int zend_jit_assign(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, int inc_op) +static int zend_jit_assign(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) { uint32_t op1_info, op2_info; @@ -1809,7 +1773,7 @@ static int zend_jit_assign(dasm_State **Dst, zend_op *opline, zend_op_array *op_ op2_info = OP2_INFO(); if (!(op1_info & (MAY_BE_STRING|MAY_BE_RESOURCE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_REF))) { - return zend_jit_simple_assign(Dst, opline, op_array, ssa, opline->op1.var, op1_info, opline->op2_type, opline->op2, op2_info, inc_op); + return zend_jit_simple_assign(Dst, opline, op_array, ssa, opline->op1.var, op1_info, opline->op2_type, opline->op2, op2_info); } fallback: @@ -1855,7 +1819,7 @@ static int zend_jit_push_call_frame(dasm_State **Dst, zend_op *opline, zend_op_a return 1; } -static int zend_jit_init_fcall(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, int call_level, int inc_op) +static int zend_jit_init_fcall(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, int call_level) { zend_func_info *info = ZEND_FUNC_INFO(op_array); zend_function *func = NULL; @@ -1878,7 +1842,6 @@ static int zend_jit_init_fcall(dasm_State **Dst, zend_op *opline, zend_op_array } else { zval *zv = RT_CONSTANT(op_array, opline->op2); - | SAVE_IP | // if (CACHED_PTR(Z_CACHE_SLOT_P(fname))) | mov r0, EX->run_time_cache | mov r0, aword [r0 + Z_CACHE_SLOT_P(zv)] @@ -1910,13 +1873,9 @@ static int zend_jit_init_fcall(dasm_State **Dst, zend_op *opline, zend_op_array | mov r0, EX->call | mov EX:r1->prev_execute_data, r0 } - | // EX(call) = call; - | mov EX->call, r1 - if (inc_op) { - | INC_IP - } else { - | RESET_IP - } + | // EX(call) = call; + | mov EX->call, r1 + valid_opline_offset++; return 1; } @@ -1950,7 +1909,9 @@ static int zend_jit_do_fcall(dasm_State **Dst, zend_op *opline, zend_op_array *o | // fbc = call->func; | // mov r1, EX:r0->func ??? | // SAVE_OPLINE(); - | SAVE_IP + if (!zend_jit_set_valid_ip(Dst)) { + return 0; + } | mov EX->opline, IP || if (call_level == 1) { | mov aword EX->call, 0 @@ -2016,7 +1977,7 @@ fallback: } } -static int zend_jit_send_val(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, int inc_op) +static int zend_jit_send_val(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) { uint32_t op1_info; @@ -2028,11 +1989,7 @@ static int zend_jit_send_val(dasm_State **Dst, zend_op *opline, zend_op_array *o || if (Z_REFCOUNTED_P(zv)) { | ADDREF_CONST zv, r1 || } - if (inc_op) { - | INC_IP - } else { - | RESET_IP - } + valid_opline_offset++; return 1; } @@ -2043,11 +2000,7 @@ static int zend_jit_send_val(dasm_State **Dst, zend_op *opline, zend_op_array *o op1_info = OP1_INFO(); | mov r0, EX->call | ZVAL_COPY_VALUE r0, opline->result.var, FP, opline->op1.var, op1_info, r1, ecx, r2 - if (inc_op) { - | INC_IP - } else { - | RESET_IP - } + valid_opline_offset++; return 1; fallback: @@ -2055,7 +2008,7 @@ fallback: return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); } -static int zend_jit_send_var(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, int inc_op) +static int zend_jit_send_var(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) { uint32_t op1_info; if (!ssa->ops || !ssa->var_info) { @@ -2074,11 +2027,7 @@ static int zend_jit_send_var(dasm_State **Dst, zend_op *opline, zend_op_array *o || if (opline->op1_type == IS_CV) { | TRY_ADDREF op1_info, ch, r2 || } - if (inc_op) { - | INC_IP - } else { - | RESET_IP - } + valid_opline_offset++; return 1; fallback: @@ -2086,7 +2035,7 @@ fallback: return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); } -static int zend_jit_type_check(dasm_State **Dst, zend_op *opline, int b, int *opnum, zend_op_array *op_array, zend_ssa *ssa, int inc_op) +static int zend_jit_type_check(dasm_State **Dst, zend_op *opline, int b, int *opnum, zend_op_array *op_array, zend_ssa *ssa) { uint32_t op1_info; unsigned int target_label; @@ -2103,11 +2052,7 @@ static int zend_jit_type_check(dasm_State **Dst, zend_op *opline, int b, int *op (opline+1)->op1_type == IS_TMP_VAR && (opline+1)->op1.var == opline->result.var) { (*opnum)++; - if (inc_op) { - | INC_IP - } else { - | RESET_IP - } + valid_opline_offset++; } else if ((opline+1)->opcode == ZEND_JMPNZ && (opline+1)->op1_type == IS_TMP_VAR && (opline+1)->op1.var == opline->result.var) { @@ -2116,11 +2061,7 @@ static int zend_jit_type_check(dasm_State **Dst, zend_op *opline, int b, int *op | jmp =>target_label } else { | mov dword [FP + opline->result.var + 8], IS_TRUE - if (inc_op) { - | INC_IP - } else { - | RESET_IP - } + valid_opline_offset++; } } else if (!(op1_info & (1<opcode == ZEND_JMPZ && @@ -2133,15 +2074,10 @@ static int zend_jit_type_check(dasm_State **Dst, zend_op *opline, int b, int *op (opline+1)->op1_type == IS_TMP_VAR && (opline+1)->op1.var == opline->result.var) { (*opnum)++; - if (inc_op) { - | INC_IP - | INC_IP - } + valid_opline_offset += 2; } else { | mov dword [FP + opline->result.var + 8], IS_FALSE - if (inc_op) { - | INC_IP - } + valid_opline_offset++; } } else { goto fallback; From 2a5d2b3ca00428881fd907fb83cd1243b6b98343 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 12 Sep 2016 22:37:30 +0300 Subject: [PATCH 167/569] Reset high bits --- ext/opcache/jit/zend_jit_x86.dasc | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 0ce31e7490cc0..f23be8a20814f 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -1268,18 +1268,22 @@ static int zend_jit_cmp_long_long(dasm_State **Dst, zend_op *opline, int b, zend switch (opline->opcode) { case ZEND_IS_EQUAL: | sete al + | movzx eax, al | add eax, 2 break; case ZEND_IS_NOT_EQUAL: | setne al + | movzx eax, al | add eax, 2 break; case ZEND_IS_SMALLER: | setl al + | movzx eax, al | add eax, 2 break; case ZEND_IS_SMALLER_OR_EQUAL: | setle al + | movzx eax, al | add eax, 2 break; } @@ -1357,10 +1361,12 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, zend_op *opline, int b, break; case ZEND_IS_SMALLER: | seta al + | movzx eax, al | add eax, 2 break; case ZEND_IS_SMALLER_OR_EQUAL: | setae al + | movzx eax, al | add eax, 2 break; } @@ -1461,18 +1467,22 @@ static int zend_jit_cmp_slow(dasm_State **Dst, zend_op *opline, int b, zend_op_a switch (opline->opcode) { case ZEND_IS_EQUAL: | sete al + | movzx eax, al | add eax, 2 break; case ZEND_IS_NOT_EQUAL: | setne al + | movzx eax, al | add eax, 2 break; case ZEND_IS_SMALLER: | setl al + | movzx eax, al | add eax, 2 break; case ZEND_IS_SMALLER_OR_EQUAL: | setle al + | movzx eax, al | add eax, 2 break; } From 0b9e580ab79efafa37fcce2d2d1c875363a29b59 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 12 Sep 2016 23:36:27 +0300 Subject: [PATCH 168/569] Delayed IP initialization --- ext/opcache/jit/zend_jit_x86.dasc | 35 +++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index f23be8a20814f..1a063403ebee5 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -504,6 +504,7 @@ static void* dasm_labels[zend_lb_MAX]; || has_slow = 1; |.endmacro +const zend_op *valid_opline; static uint32_t valid_opline_offset; #endif @@ -611,6 +612,7 @@ static const zend_jit_stub zend_jit_stubs[] = { static int zend_jit_align_func(dasm_State **Dst) { + valid_opline = NULL; valid_opline_offset = 0; |.align 16 return 1; @@ -628,8 +630,29 @@ static int zend_jit_label(dasm_State **Dst, unsigned int label) return 1; } +static int zend_jit_set_valid_ip(dasm_State **Dst) +{ + if (valid_opline || valid_opline_offset) { + if (!valid_opline) { + | add IP, sizeof(zend_op) * valid_opline_offset + } else { + const zend_op *target_opline = valid_opline + valid_opline_offset; + + valid_opline = NULL; + | LOAD_ADDR IP, target_opline + } + valid_opline_offset = 0; + } + return 1; +} + static int zend_jit_check_timeout(dasm_State **Dst) { + if (zend_interrupt_function) { + if (!zend_jit_set_valid_ip(Dst)) { + return 0; + } + } | cmp byte [&EG(vm_interrupt)], 0 | jne ->interrupt_handler return 1; @@ -642,15 +665,6 @@ static int zend_jit_check_exception(dasm_State **Dst) return 1; } -static int zend_jit_set_valid_ip(dasm_State **Dst) -{ - if (valid_opline_offset) { - | add IP, sizeof(zend_op) * valid_opline_offset - valid_opline_offset = 0; - } - return 1; -} - static int zend_jit_handler(dasm_State **Dst, zend_op *opline, int may_throw) { if (!zend_jit_set_valid_ip(Dst)) { @@ -681,14 +695,13 @@ static int zend_jit_skip_handler(dasm_State **Dst) static int zend_jit_set_opline(dasm_State **Dst, zend_op *target_opline) { + valid_opline = target_opline; valid_opline_offset = 0; - | LOAD_ADDR IP, target_opline return 1; } static int zend_jit_jmp(dasm_State **Dst, unsigned int target_label) { - valid_opline_offset = 0; | jmp =>target_label return 1; } From 567a59d34a144b4c4b8483a323d6cc01887d3a49 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 13 Sep 2016 01:06:57 +0300 Subject: [PATCH 169/569] Eliminated one memory load on recursive DO_UCALL. --- ext/opcache/jit/zend_jit_x86.dasc | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 1a063403ebee5..57881958a2c3c 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -1965,8 +1965,13 @@ static int zend_jit_do_fcall(dasm_State **Dst, zend_op *opline, zend_op_array *o || } | | //EX_LOAD_RUN_TIME_CACHE(op_array); - | mov r1, EX:r0->func - | mov r2, aword [r1 + offsetof(zend_op_array, run_time_cache)] + || if (func && op_array == &func->op_array) { + || /* recursive call */ + | mov r2, EX->run_time_cache + || } else { + | mov r1, EX:r0->func + | mov r2, aword [r1 + offsetof(zend_op_array, run_time_cache)] + || } | mov EX:r0->run_time_cache, r2 | //EX_LOAD_LITERALS(op_array); |.if X64 From b6e79a6f003c2d12d7fd3c8740d310e1d92f761a Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 13 Sep 2016 01:29:19 +0300 Subject: [PATCH 170/569] Use the same register for EX(call). (this may simplify dead memory loads) --- ext/opcache/jit/zend_jit_x86.dasc | 54 +++++++++++++++---------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 57881958a2c3c..2a3b14d3528dc 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -1928,9 +1928,9 @@ static int zend_jit_do_fcall(dasm_State **Dst, zend_op *opline, zend_op_array *o } | // call = EX(call); - | mov r0, EX->call + | mov r1, EX->call | // fbc = call->func; - | // mov r1, EX:r0->func ??? + | // mov r2, EX:r1->func ??? | // SAVE_OPLINE(); if (!zend_jit_set_valid_ip(Dst)) { return 0; @@ -1940,50 +1940,50 @@ static int zend_jit_do_fcall(dasm_State **Dst, zend_op *opline, zend_op_array *o | mov aword EX->call, 0 || } else { | //EX(call) = call->prev_execute_data; - | mov r1, EX:r0->prev_execute_data - | mov EX->call, r1 + | mov r0, EX:r1->prev_execute_data + | mov EX->call, r0 || } | //call->prev_execute_data = execute_data; - | mov EX:r0->prev_execute_data, EX + | mov EX:r1->prev_execute_data, EX | | // EX(call) = NULL; - | mov aword EX:r0->call, 0 + | mov aword EX:r1->call, 0 || if (RETURN_VALUE_USED(opline)) { | // ZVAL_NULL(EX_VAR(opline->result.var)); | mov dword [FP + opline->result.var + 8], IS_NULL | // EX(return_value) = EX_VAR(opline->result.var); - | lea r1, aword [FP + opline->result.var] - | mov aword EX:r0->return_value, r1 + | lea r0, aword [FP + opline->result.var] + | mov aword EX:r1->return_value, r0 || } else { | // EX(return_value) = 0; - | mov aword EX:r0->return_value, 0 + | mov aword EX:r1->return_value, 0 || } | || for (i = call_info->num_args; i < func->op_array.last_var; i++) { || uint32_t n = (uint32_t)(uintptr_t)ZEND_CALL_VAR_NUM(NULL, i); - | mov dword [r0 + n + 8], IS_UNDEF + | mov dword [r1 + n + 8], IS_UNDEF || } | | //EX_LOAD_RUN_TIME_CACHE(op_array); || if (func && op_array == &func->op_array) { || /* recursive call */ - | mov r2, EX->run_time_cache + | mov r0, EX->run_time_cache || } else { - | mov r1, EX:r0->func - | mov r2, aword [r1 + offsetof(zend_op_array, run_time_cache)] + | mov r2, EX:r1->func + | mov r0, aword [r2 + offsetof(zend_op_array, run_time_cache)] || } - | mov EX:r0->run_time_cache, r2 + | mov EX:r1->run_time_cache, r0 | //EX_LOAD_LITERALS(op_array); |.if X64 - | LOAD_ADDR r2, func->op_array.literals - | mov EX:r0->literals, r2 + | LOAD_ADDR r0, func->op_array.literals + | mov EX:r1->literals, r0 |.endif | // EG(current_execute_data) = execute_data; - | mov aword [&EG(current_execute_data)], r0 - | mov FP, r0 + | mov aword [&EG(current_execute_data)], r1 + | mov FP, r1 | // EX(opline) = op_array->opcodes; | LOAD_ADDR IP, (func->op_array.opcodes + call_info->num_args) - | mov EX:r0->opline, IP + | mov EX:r1->opline, IP || if (func && op_array == &func->op_array) { || /* recursive call */ | jmp =>call_info->num_args @@ -2012,10 +2012,10 @@ static int zend_jit_send_val(dasm_State **Dst, zend_op *opline, zend_op_array *o if (opline->op1_type == IS_CONST) { zval *zv = RT_CONSTANT(op_array, opline->op1); - | mov r0, EX->call - | ZVAL_COPY_CONST r0, opline->result.var, -1, zv, r1 + | mov r1, EX->call + | ZVAL_COPY_CONST r1, opline->result.var, -1, zv, r0 || if (Z_REFCOUNTED_P(zv)) { - | ADDREF_CONST zv, r1 + | ADDREF_CONST zv, r0 || } valid_opline_offset++; return 1; @@ -2026,8 +2026,8 @@ static int zend_jit_send_val(dasm_State **Dst, zend_op *opline, zend_op_array *o } op1_info = OP1_INFO(); - | mov r0, EX->call - | ZVAL_COPY_VALUE r0, opline->result.var, FP, opline->op1.var, op1_info, r1, ecx, r2 + | mov r1, EX->call + | ZVAL_COPY_VALUE r1, opline->result.var, FP, opline->op1.var, op1_info, r0, eax, r2 valid_opline_offset++; return 1; @@ -2050,10 +2050,10 @@ static int zend_jit_send_var(dasm_State **Dst, zend_op *opline, zend_op_array *o } op1_info = OP1_INFO(); - | mov r0, EX->call - | ZVAL_COPY_VALUE r0, opline->result.var, FP, opline->op1.var, op1_info, r1, ecx, r2 + | mov r1, EX->call + | ZVAL_COPY_VALUE r1, opline->result.var, FP, opline->op1.var, op1_info, r0, eax, r2 || if (opline->op1_type == IS_CV) { - | TRY_ADDREF op1_info, ch, r2 + | TRY_ADDREF op1_info, ah, r2 || } valid_opline_offset++; return 1; From 086166429cd46371f470ccfe2260e06afe02cde2 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Tue, 13 Sep 2016 11:13:08 +0800 Subject: [PATCH 171/569] Fixed typo(IS_UNUSED) --- ext/opcache/jit/zend_jit_x86.dasc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 2a3b14d3528dc..79b4fed89d87c 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -1787,8 +1787,7 @@ static int zend_jit_assign(dasm_State **Dst, zend_op *opline, zend_op_array *op_ { uint32_t op1_info, op2_info; - if (opline->op1_type != IS_CV || opline->result_type != IS_UNDEF || - !ssa->ops || !ssa->var_info) { + if (opline->op1_type != IS_CV || opline->result_type != IS_UNUSED || !ssa->ops || !ssa->var_info) { goto fallback; } From e1ac8096cf97b6d3a0f5d9f2d9823f2459f2dd6a Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 13 Sep 2016 08:09:48 +0300 Subject: [PATCH 172/569] Implemeted JIT for CASE. --- ext/opcache/jit/zend_jit.c | 1 + ext/opcache/jit/zend_jit_x86.dasc | 17 +++++++++++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 571d5599f9492..c3a1b25b43cad 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -826,6 +826,7 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) case ZEND_IS_NOT_EQUAL: case ZEND_IS_SMALLER: case ZEND_IS_SMALLER_OR_EQUAL: + case ZEND_CASE: if (!zend_jit_cmp(&dasm_state, opline, b, &i, op_array, ssa)) { goto jit_failure; } diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 2a3b14d3528dc..0accd222fbc99 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -294,7 +294,7 @@ static void* dasm_labels[zend_lb_MAX]; | LONG_OP imul, op_type, op, reg || break; || case ZEND_DIV: -| idiv aword [FP + op.var] +| idiv aword [FP + op.var] // (reg == r0) || break; ||} |.endmacro @@ -310,6 +310,9 @@ static void* dasm_labels[zend_lb_MAX]; || case ZEND_MUL: | imul reg2, reg1 || break; +|| case ZEND_DIV: +| idiv reg2 // (reg1 == r0) +|| break; ||} |.endmacro @@ -1247,6 +1250,7 @@ static int zend_jit_cmp_long_long(dasm_State **Dst, zend_op *opline, int b, zend target_label = ssa->cfg.blocks[b].successors[0]; switch (opline->opcode) { case ZEND_IS_EQUAL: + case ZEND_CASE: | jne => target_label break; case ZEND_IS_NOT_EQUAL: @@ -1265,6 +1269,7 @@ static int zend_jit_cmp_long_long(dasm_State **Dst, zend_op *opline, int b, zend target_label = ssa->cfg.blocks[b].successors[0]; switch (opline->opcode) { case ZEND_IS_EQUAL: + case ZEND_CASE: | je => target_label break; case ZEND_IS_NOT_EQUAL: @@ -1280,6 +1285,7 @@ static int zend_jit_cmp_long_long(dasm_State **Dst, zend_op *opline, int b, zend } else { switch (opline->opcode) { case ZEND_IS_EQUAL: + case ZEND_CASE: | sete al | movzx eax, al | add eax, 2 @@ -1316,6 +1322,7 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, zend_op *opline, int b, target_label = ssa->cfg.blocks[b].successors[0]; switch (opline->opcode) { case ZEND_IS_EQUAL: + case ZEND_CASE: | jp >1 | jne => target_label |1: @@ -1338,6 +1345,7 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, zend_op *opline, int b, target_label = ssa->cfg.blocks[b].successors[0]; switch (opline->opcode) { case ZEND_IS_EQUAL: + case ZEND_CASE: | jp >1 | je => target_label |1: @@ -1357,6 +1365,7 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, zend_op *opline, int b, } else { switch (opline->opcode) { case ZEND_IS_EQUAL: + case ZEND_CASE: | jp >1 | mov eax, IS_TRUE | je >2 @@ -1446,6 +1455,7 @@ static int zend_jit_cmp_slow(dasm_State **Dst, zend_op *opline, int b, zend_op_a target_label = ssa->cfg.blocks[b].successors[0]; switch (opline->opcode) { case ZEND_IS_EQUAL: + case ZEND_CASE: | jne => target_label break; case ZEND_IS_NOT_EQUAL: @@ -1464,6 +1474,7 @@ static int zend_jit_cmp_slow(dasm_State **Dst, zend_op *opline, int b, zend_op_a target_label = ssa->cfg.blocks[b].successors[0]; switch (opline->opcode) { case ZEND_IS_EQUAL: + case ZEND_CASE: | je => target_label break; case ZEND_IS_NOT_EQUAL: @@ -1479,6 +1490,7 @@ static int zend_jit_cmp_slow(dasm_State **Dst, zend_op *opline, int b, zend_op_a } else { switch (opline->opcode) { case ZEND_IS_EQUAL: + case ZEND_CASE: | sete al | movzx eax, al | add eax, 2 @@ -1690,7 +1702,8 @@ static int zend_jit_cmp(dasm_State **Dst, zend_op *opline, int b, int *opnum, ze | PUSH_ZVAL_ADDR opline->op2_type, opline->op2, r0 |.endif | EXT_CALL compare_function, r0 - || if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && + || if (opline->opcode != ZEND_CASE && + || (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && || (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { | and byte [FP + opline->op1.var + 9], IS_TYPE_REFCOUNTED | je >1 From 3d3495cc2fdf70ef721bdcf747ca05b30a83ac62 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 13 Sep 2016 08:42:36 +0300 Subject: [PATCH 173/569] Improved JIT for comparison opcodes. --- ext/opcache/jit/zend_jit_x86.dasc | 48 ++++++++++++++++++------------- 1 file changed, 28 insertions(+), 20 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 0accd222fbc99..761d7ef8f9a76 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -1534,11 +1534,6 @@ static int zend_jit_cmp(dasm_State **Dst, zend_op *opline, int b, int *opnum, ze goto fallback; } - if (!(op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) || - !(op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { - goto fallback; - } - if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)) { if (op1_info & (MAY_BE_ANY-MAY_BE_LONG)) { | cmp dword [FP + opline->op1.var + 8], IS_LONG @@ -1680,20 +1675,21 @@ static int zend_jit_cmp(dasm_State **Dst, zend_op *opline, int b, int *opnum, ze } } - |6: - if (((opline+1)->opcode == ZEND_JMPZ || (opline+1)->opcode == ZEND_JMPNZ) && - (opline+1)->op1_type == IS_TMP_VAR && - (opline+1)->op1.var == opline->result.var) { - (*opnum)++; - valid_opline_offset += 2; - } else { - valid_opline_offset++; - } - - if (has_slow) { - |.cold_code - |9: + if (has_slow || + !(op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) || + !(op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { + if (has_slow) { + |.cold_code + |9: + } | mov EX->opline, IP + if (opline->op1_type == IS_CV && (op1_info & MAY_BE_UNDEF)) { + | cmp dword [FP + opline->op1.var + 8], 0 + | jnz >1 + |1: + } + if (opline->op2_type == IS_CV && (op2_info & MAY_BE_UNDEF)) { + } | lea FCARG1a, [FP + opline->result.var] | LOAD_ZVAL_ADDR FCARG2a, opline->op1_type, opline->op1 |.if X64 @@ -1740,8 +1736,20 @@ static int zend_jit_cmp(dasm_State **Dst, zend_op *opline, int b, int *opnum, ze if (!zend_jit_cmp_slow(Dst, opline, b, op_array, ssa)) { return 0; } - | jmp <6 - |.code + if (has_slow) { + | jmp >6 + |.code + } + } + + |6: + if (((opline+1)->opcode == ZEND_JMPZ || (opline+1)->opcode == ZEND_JMPNZ) && + (opline+1)->op1_type == IS_TMP_VAR && + (opline+1)->op1.var == opline->result.var) { + (*opnum)++; + valid_opline_offset += 2; + } else { + valid_opline_offset++; } return 1; From 9d9ecf65032ae1125895f17a00455fb45bfd0033 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 13 Sep 2016 11:17:45 +0300 Subject: [PATCH 174/569] Fixed 64-bit debug build --- ext/opcache/jit/zend_jit_x86.dasc | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 761d7ef8f9a76..bfdd30f9cfcd7 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -1203,7 +1203,11 @@ static int zend_jit_math(dasm_State **Dst, zend_op *opline, zend_op_array *op_ar || if (ZEND_DEBUG) { || const char *filename = op_array->filename ? op_array->filename->val : NULL; | LOAD_ADDR FCARG2a, filename + |.if X64 + | mov CARG3d, opline->lineno + |.else | push opline->lineno + |.endif || } | EXT_CALL _zval_dtor_func, r0 |1: @@ -1219,7 +1223,11 @@ static int zend_jit_math(dasm_State **Dst, zend_op *opline, zend_op_array *op_ar || if (ZEND_DEBUG) { || const char *filename = op_array->filename ? op_array->filename->val : NULL; | LOAD_ADDR FCARG2a, filename + |.if X64 + | mov CARG3d, opline->lineno + |.else | push opline->lineno + |.endif || } | EXT_CALL _zval_dtor_func, r0 |1: @@ -1709,7 +1717,11 @@ static int zend_jit_cmp(dasm_State **Dst, zend_op *opline, int b, int *opnum, ze || if (ZEND_DEBUG) { || const char *filename = op_array->filename ? op_array->filename->val : NULL; | LOAD_ADDR FCARG2a, filename + |.if X64 + | mov CARG3d, opline->lineno + |.else | push opline->lineno + |.endif || } | EXT_CALL _zval_dtor_func, r0 |1: @@ -1725,7 +1737,11 @@ static int zend_jit_cmp(dasm_State **Dst, zend_op *opline, int b, int *opnum, ze || if (ZEND_DEBUG) { || const char *filename = op_array->filename ? op_array->filename->val : NULL; | LOAD_ADDR FCARG2a, filename + |.if X64 + | mov CARG3d, opline->lineno + |.else | push opline->lineno + |.endif || } | EXT_CALL _zval_dtor_func, r0 |1: From cf7cd756f4cdcbf41906c304dd30aa127301bf7e Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Tue, 13 Sep 2016 16:31:56 +0800 Subject: [PATCH 175/569] Implementing ZEND_RETURN(incomplete) --- ext/opcache/jit/zend_jit_disasm_x86.c | 1 + ext/opcache/jit/zend_jit_helpers.c | 134 +++++++++++++++++++ ext/opcache/jit/zend_jit_x86.dasc | 177 +++++++++++++++++--------- 3 files changed, 250 insertions(+), 62 deletions(-) diff --git a/ext/opcache/jit/zend_jit_disasm_x86.c b/ext/opcache/jit/zend_jit_disasm_x86.c index e147662f03cd6..b1628eccaa6e1 100644 --- a/ext/opcache/jit/zend_jit_disasm_x86.c +++ b/ext/opcache/jit/zend_jit_disasm_x86.c @@ -384,6 +384,7 @@ static int zend_jit_disasm_init(void) (uint64_t)(uintptr_t)n, sizeof(void*)); REGISTER_HELPER(zend_jit_find_func_helper); REGISTER_HELPER(zend_jit_extend_stack_helper); + REGISTER_HELPER(zend_jit_leave_helper); #undef REGISTER_HELPER zend_elf_load_symbols(); diff --git a/ext/opcache/jit/zend_jit_helpers.c b/ext/opcache/jit/zend_jit_helpers.c index 9f0a342effc28..c573a33b4662d 100644 --- a/ext/opcache/jit/zend_jit_helpers.c +++ b/ext/opcache/jit/zend_jit_helpers.c @@ -16,6 +16,8 @@ +----------------------------------------------------------------------+ */ +#include "Zend/zend_API.h" + static zend_function* ZEND_FASTCALL zend_jit_find_func_helper(zend_string *name) { zval *func = zend_hash_find(EG(function_table), name); @@ -41,6 +43,138 @@ static zend_execute_data* ZEND_FASTCALL zend_jit_extend_stack_helper(uint32_t us return call; } +static zend_always_inline void i_free_compiled_variables(zend_execute_data *execute_data) +{ + zval *cv = EX_VAR_NUM(0); + zval *end = cv + EX(func)->op_array.last_var; + while (EXPECTED(cv != end)) { + if (Z_REFCOUNTED_P(cv)) { + if (!Z_DELREF_P(cv)) { + zend_refcounted *r = Z_COUNTED_P(cv); + ZVAL_NULL(cv); + zval_dtor_func(r); + } else { + GC_ZVAL_CHECK_POSSIBLE_ROOT(cv); + } + } + cv++; + } +} + +static int ZEND_FASTCALL zend_jit_leave_helper(zend_execute_data *execute_data) { + zend_execute_data *old_execute_data; + uint32_t call_info = EX_CALL_INFO(); + + if (EXPECTED((call_info & (ZEND_CALL_CODE|ZEND_CALL_TOP|ZEND_CALL_HAS_SYMBOL_TABLE|ZEND_CALL_FREE_EXTRA_ARGS|ZEND_CALL_ALLOCATED)) == 0)) { + i_free_compiled_variables(execute_data); + + EG(current_execute_data) = EX(prev_execute_data); + if (UNEXPECTED(call_info & ZEND_CALL_RELEASE_THIS)) { + zend_object *object = Z_OBJ(execute_data->This); + if (UNEXPECTED(EG(exception) != NULL) && (call_info & ZEND_CALL_CTOR)) { + GC_REFCOUNT(object)--; + zend_object_store_ctor_failed(object); + } + OBJ_RELEASE(object); + } else if (UNEXPECTED(call_info & ZEND_CALL_CLOSURE)) { + OBJ_RELEASE((zend_object*)execute_data->func->op_array.prototype); + } + EG(vm_stack_top) = (zval*)execute_data; + execute_data = EX(prev_execute_data); + + if (UNEXPECTED(EG(exception) != NULL)) { + const zend_op *old_opline = EX(opline); + zend_throw_exception_internal(NULL); + if (RETURN_VALUE_USED(old_opline)) { + zval_ptr_dtor(EX_VAR(old_opline->result.var)); + } + return 2; + } + EX(opline)++; + return 0; + } else if (EXPECTED((call_info & (ZEND_CALL_CODE|ZEND_CALL_TOP)) == 0)) { + i_free_compiled_variables(execute_data); + + if (UNEXPECTED(call_info & ZEND_CALL_HAS_SYMBOL_TABLE)) { + zend_clean_and_cache_symbol_table(EX(symbol_table)); + } + EG(current_execute_data) = EX(prev_execute_data); + if (UNEXPECTED(call_info & ZEND_CALL_RELEASE_THIS)) { + zend_object *object = Z_OBJ(execute_data->This); + if (UNEXPECTED(EG(exception) != NULL) && (call_info & ZEND_CALL_CTOR)) { + GC_REFCOUNT(object)--; + zend_object_store_ctor_failed(object); + } + OBJ_RELEASE(object); + } else if (UNEXPECTED(call_info & ZEND_CALL_CLOSURE)) { + OBJ_RELEASE((zend_object*)execute_data->func->op_array.prototype); + } + + zend_vm_stack_free_extra_args_ex(call_info, execute_data); + old_execute_data = execute_data; + execute_data = EX(prev_execute_data); + zend_vm_stack_free_call_frame_ex(call_info, old_execute_data); + + if (UNEXPECTED(EG(exception) != NULL)) { + const zend_op *old_opline = EX(opline); + zend_throw_exception_internal(NULL); + if (RETURN_VALUE_USED(old_opline)) { + zval_ptr_dtor(EX_VAR(old_opline->result.var)); + } + return 2; + } + EX(opline)++; + return 0; + } else if (EXPECTED((call_info & ZEND_CALL_TOP) == 0)) { + zend_detach_symbol_table(execute_data); + destroy_op_array(&EX(func)->op_array); + efree_size(EX(func), sizeof(zend_op_array)); + old_execute_data = execute_data; + execute_data = EG(current_execute_data) = EX(prev_execute_data); + zend_vm_stack_free_call_frame_ex(call_info, old_execute_data); + + zend_attach_symbol_table(execute_data); + if (UNEXPECTED(EG(exception) != NULL)) { + zend_throw_exception_internal(NULL); + return 2; + } + EX(opline)++; + return 0; + } else { + if (EXPECTED((call_info & ZEND_CALL_CODE) == 0)) { + i_free_compiled_variables(execute_data); + i_free_compiled_variables(execute_data); + if (UNEXPECTED(call_info & (ZEND_CALL_HAS_SYMBOL_TABLE|ZEND_CALL_FREE_EXTRA_ARGS))) { + if (UNEXPECTED(call_info & ZEND_CALL_HAS_SYMBOL_TABLE)) { + zend_clean_and_cache_symbol_table(EX(symbol_table)); + } + zend_vm_stack_free_extra_args_ex(call_info, execute_data); + } + EG(current_execute_data) = EX(prev_execute_data); + if (UNEXPECTED(call_info & ZEND_CALL_CLOSURE)) { + OBJ_RELEASE((zend_object*)EX(func)->op_array.prototype); + } + return -1; + } else /* if (call_kind == ZEND_CALL_TOP_CODE) */ { + zend_array *symbol_table = EX(symbol_table); + + zend_detach_symbol_table(execute_data); + old_execute_data = EX(prev_execute_data); + while (old_execute_data) { + if (old_execute_data->func && (ZEND_CALL_INFO(old_execute_data) & ZEND_CALL_HAS_SYMBOL_TABLE)) { + if (old_execute_data->symbol_table == symbol_table) { + zend_attach_symbol_table(old_execute_data); + } + break; + } + old_execute_data = old_execute_data->prev_execute_data; + } + EG(current_execute_data) = EX(prev_execute_data); + return -1; + } + } +} + /* * Local variables: * tab-width: 4 diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 79b4fed89d87c..b1a675dd92a82 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -504,6 +504,27 @@ static void* dasm_labels[zend_lb_MAX]; || has_slow = 1; |.endmacro +|.macro FREE_OP, op_type, op, op_info, op_array, lineno +||if ((op_type & (IS_VAR|IS_TMP_VAR)) && +|| (op_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { +| and byte [FP + op.var + 9], IS_TYPE_REFCOUNTED +| je >1 +| mov FCARG1a, aword [FP + op.var] +| dec dword [FCARG1a] +| jnz >1 +|| if (ZEND_DEBUG) { +|| const char *filename = op_array->filename ? op_array->filename->val : NULL; +| LOAD_ADDR FCARG2a, filename +| push lineno +|| } +| EXT_CALL _zval_dtor_func, r0 +|| if (ZEND_DEBUG) { +| add r4, sizeof(ptrdiff_t) +|| } +| 1: +||} +|.endmacro + const zend_op *valid_opline; static uint32_t valid_opline_offset; #endif @@ -1190,37 +1211,8 @@ static int zend_jit_math(dasm_State **Dst, zend_op *opline, zend_op_array *op_ar || } else { || ZEND_ASSERT(0); || } - || if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && - || (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { - | and byte [FP + opline->op1.var + 9], IS_TYPE_REFCOUNTED - | je >1 - | mov FCARG1a, aword [FP + opline->op1.var] - | dec dword [FCARG1a] - | jnz >1 - || if (ZEND_DEBUG) { - || const char *filename = op_array->filename ? op_array->filename->val : NULL; - | LOAD_ADDR FCARG2a, filename - | push opline->lineno - || } - | EXT_CALL _zval_dtor_func, r0 - |1: - || } - || if ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) && - || (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { - | and byte [FP + opline->op2.var + 9], IS_TYPE_REFCOUNTED - | je >1 - | mov FCARG1a, aword [FP + opline->op2.var] - | dec dword [FCARG1a] - | dec dword [FP + opline->op2.var] - | jnz >1 - || if (ZEND_DEBUG) { - || const char *filename = op_array->filename ? op_array->filename->val : NULL; - | LOAD_ADDR FCARG2a, filename - | push opline->lineno - || } - | EXT_CALL _zval_dtor_func, r0 - |1: - || } + | FREE_OP opline->op1_type, opline->op1, op1_info, op_array, opline->lineno + | FREE_OP opline->op2_type, opline->op2, op2_info, op_array, opline->lineno || if (zend_may_throw(opline, op_array, ssa)) { || zend_jit_check_exception(Dst); || } @@ -1690,37 +1682,8 @@ static int zend_jit_cmp(dasm_State **Dst, zend_op *opline, int b, int *opnum, ze | PUSH_ZVAL_ADDR opline->op2_type, opline->op2, r0 |.endif | EXT_CALL compare_function, r0 - || if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && - || (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { - | and byte [FP + opline->op1.var + 9], IS_TYPE_REFCOUNTED - | je >1 - | mov FCARG1a, aword [FP + opline->op1.var] - | dec dword [FCARG1a] - | jnz >1 - || if (ZEND_DEBUG) { - || const char *filename = op_array->filename ? op_array->filename->val : NULL; - | LOAD_ADDR FCARG2a, filename - | push opline->lineno - || } - | EXT_CALL _zval_dtor_func, r0 - |1: - || } - || if ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) && - || (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { - | and byte [FP + opline->op2.var + 9], IS_TYPE_REFCOUNTED - | je >1 - | mov FCARG1a, aword [FP + opline->op2.var] - | dec dword [FCARG1a] - | dec dword [FP + opline->op2.var] - | jnz >1 - || if (ZEND_DEBUG) { - || const char *filename = op_array->filename ? op_array->filename->val : NULL; - | LOAD_ADDR FCARG2a, filename - | push opline->lineno - || } - | EXT_CALL _zval_dtor_func, r0 - |1: - || } + | FREE_OP opline->op1_type, opline->op1, op1_info, op_array, opline->lineno + | FREE_OP opline->op2_type, opline->op2, op2_info, op_array, opline->lineno || if (zend_may_throw(opline, op_array, ssa)) { || zend_jit_check_exception(Dst); || } @@ -2117,6 +2080,96 @@ fallback: return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); } +static int zend_jit_return(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) +{ + uint32_t op1_info; + + if (op_array->type == ZEND_EVAL_CODE || !op_array->function_name || !ssa->ops || !ssa->var_info) { + return zend_jit_tail_handler(Dst, opline); + } + + op1_info = OP1_INFO(); + if (opline->op1_type == IS_CV && (op1_info & MAY_BE_UNDEF)) { + return zend_jit_tail_handler(Dst, opline); + } + + // if (!EX(return_value)) + | mov r1, EX->return_value + | test r1, r1 + | jnz >2 + | FREE_OP opline->op1_type, opline->op1, op1_info, op_array, opline->lineno + | jmp >9 + |2: + if (opline->op1_type == IS_CONST) { + zval *zv = RT_CONSTANT(op_array, opline->op1); + | ZVAL_COPY_CONST r1, 0, -1, zv, r0 + if (Z_REFCOUNTED_P(zv)) { + | ADDREF_CONST zv, r0 + } + } else if (opline->op1_type == IS_TMP_VAR) { + | ZVAL_COPY_VALUE r1, 0, FP, opline->op1.var, op1_info, r0, eax, r2 + } else if (opline->op1_type == IS_CV) { + if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { + | lea r0, [FP + opline->op1.var] + | ZVAL_DEREF r0, op1_info + | ZVAL_COPY_VALUE_REG r1, 0, r0, op1_info, r0, eax, r2 + } else { + | ZVAL_COPY_VALUE r1, 0, FP, opline->op1.var, op1_info, r0, eax, r2 + } + | TRY_ADDREF op1_info, ah, r2 + } else { + if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { + if (op1_info & MAY_BE_REF) { + | lea FCARG1a, [FP + opline->op1.var] + | cmp byte [FCARG1a + 8], IS_REFERENCE + | jne >4 + | mov FCARG1a, [FCARG1a] + | add FCARG1a, 8 + | ZVAL_COPY_VALUE_REG r1, 0, FCARG1a, op1_info, r1, ebx, r2 + | sub FCARG1a, 8 + | dec dword [r0] + | jne >3 + if (ZEND_DEBUG) { + const char *filename = op_array->filename ? op_array->filename->val : NULL; + | LOAD_ADDR FCARG1a, filename + | push opline->lineno + } + | EXT_CALL _efree, r0 + if (ZEND_DEBUG) { + | add r4, sizeof(ptrdiff_t) + } + | jmp >9 + |3: + | TRY_ADDREF op1_info, bh, r2 + | jmp >9 + |4: + | ZVAL_COPY_VALUE r1, 0, FP, opline->op1.var, op1_info, r0, eax, r2 + } else { + | ZVAL_COPY_VALUE r1, 0, FP, opline->op1.var, op1_info, r0, eax, r2 + } + } else { + | ZVAL_COPY_VALUE r1, 0, FP, opline->op1.var, op1_info, r0, eax, r2 + } + } + |9: + | mov FCARG1a, FP + if (!zend_jit_set_valid_ip(Dst)) { + return 0; + } + | mov [FP], IP + | EXT_CALL zend_jit_leave_helper, r0 + zend_jit_check_exception(Dst); + | cmp al, 0 + | jg >1 + | LOAD_ADDR FP, &EG(current_execute_data) + | mov FP, [FP] + | mov IP, EX->opline + |1: + | add r4, SPAD + | ret + return 1; +} + #endif /* From b1596ea1a52bf712266eb06de6c32b5c33e3b7ba Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 13 Sep 2016 12:21:12 +0300 Subject: [PATCH 176/569] Fixes --- ext/opcache/jit/zend_jit_x86.dasc | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 16e451fb8970e..dd6ee05864c79 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -525,11 +525,6 @@ static void* dasm_labels[zend_lb_MAX]; |.endif || } | EXT_CALL _zval_dtor_func, r0 -|| if (ZEND_DEBUG) { -|.if not X64 -| add r4, sizeof(ptrdiff_t) -|.endif -|| } | 1: ||} |.endmacro @@ -1686,8 +1681,6 @@ static int zend_jit_cmp(dasm_State **Dst, zend_op *opline, int b, int *opnum, ze | jnz >1 |1: } - if (opline->op2_type == IS_CV && (op2_info & MAY_BE_UNDEF)) { - } | lea FCARG1a, [FP + opline->result.var] | LOAD_ZVAL_ADDR FCARG2a, opline->op1_type, opline->op1 |.if X64 @@ -1696,7 +1689,9 @@ static int zend_jit_cmp(dasm_State **Dst, zend_op *opline, int b, int *opnum, ze | PUSH_ZVAL_ADDR opline->op2_type, opline->op2, r0 |.endif | EXT_CALL compare_function, r0 - | FREE_OP opline->op1_type, opline->op1, op1_info, op_array, opline->lineno + || if (opline->opcode != ZEND_CASE) { + | FREE_OP opline->op1_type, opline->op1, op1_info, op_array, opline->lineno + || } | FREE_OP opline->op2_type, opline->op2, op2_info, op_array, opline->lineno || if (zend_may_throw(opline, op_array, ssa)) { || zend_jit_check_exception(Dst); From 8465c5bb96a07d1420bb818de074c5b307123610 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 13 Sep 2016 12:30:24 +0300 Subject: [PATCH 177/569] Implemented JIT for ZEND_SEND_VAL_EX. --- ext/opcache/jit/zend_jit.c | 2 ++ ext/opcache/jit/zend_jit_x86.dasc | 56 +++++++++++++++++++++++++------ 2 files changed, 47 insertions(+), 11 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index c3a1b25b43cad..ae431196dfef3 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -19,6 +19,7 @@ #include #include "Zend/zend_execute.h" #include "Zend/zend_vm.h" +#include "Zend/zend_exceptions.h" #include "zend_smart_str.h" #include "jit/zend_jit.h" @@ -808,6 +809,7 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) } break; case ZEND_SEND_VAL: + case ZEND_SEND_VAL_EX: if (!zend_jit_send_val(&dasm_state, opline, op_array, ssa)) { goto jit_failure; } diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index dd6ee05864c79..da16bdfa986ca 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -1991,27 +1991,61 @@ fallback: static int zend_jit_send_val(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) { uint32_t op1_info; + uint32_t arg_num = opline->op2.num; + + if (opline->opcode == ZEND_SEND_VAL_EX && arg_num > MAX_ARG_FLAG_NUM) { + goto fallback; + } + + op1_info = OP1_INFO(); + + | mov r1, EX->call + + if (opline->opcode == ZEND_SEND_VAL_EX) { + uint32_t mask = ZEND_SEND_BY_REF << ((arg_num + 3) * 2); + + | mov r0, EX:r1->func + if (arg_num <= MAX_ARG_FLAG_NUM) { + | mov eax, dword [r0 + offsetof(zend_function, quick_arg_flags)] + | and eax, mask + | jnz >1 + } else { + ZEND_ASSERT(0); + } + |.cold_code + |1: + | mov dword [r1 + opline->result.var + 8], IS_UNDEF + | mov EX->opline, IP + |.if X64 + | mov CARG1, 0 + | LOAD_ADDR CARG2, "Cannot pass parameter %d by reference" + | mov CARG3d, arg_num + | EXT_CALL zend_throw_error, r0 + |.else + | sub r4, 4 + | push arg_num + | push "Cannot pass parameter %d by reference" + | push 0 + | EXT_CALL zend_throw_error, r0 + | add r4, 16 + |.endif + | FREE_OP opline->op1_type, opline->op1, op1_info, op_array, opline->lineno + | jmp ->exception_handler + |.code + } if (opline->op1_type == IS_CONST) { zval *zv = RT_CONSTANT(op_array, opline->op1); - | mov r1, EX->call | ZVAL_COPY_CONST r1, opline->result.var, -1, zv, r0 || if (Z_REFCOUNTED_P(zv)) { | ADDREF_CONST zv, r0 || } - valid_opline_offset++; - return 1; - } - - if (!ssa->ops || !ssa->var_info) { - goto fallback; + } else { + | ZVAL_COPY_VALUE r1, opline->result.var, FP, opline->op1.var, op1_info, r0, eax, r2 } - - op1_info = OP1_INFO(); - | mov r1, EX->call - | ZVAL_COPY_VALUE r1, opline->result.var, FP, opline->op1.var, op1_info, r0, eax, r2 valid_opline_offset++; + return 1; fallback: From 79737aa824e775b7b1bc900b17110fd3d5d81232 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 13 Sep 2016 12:38:36 +0300 Subject: [PATCH 178/569] Fixed Zend/tests/entry_block_with_predecessors.phpt --- ext/opcache/jit/zend_jit.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index ae431196dfef3..8baa1380b1b4a 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -616,7 +616,7 @@ static int zend_may_throw(zend_op *opline, zend_op_array *op_array, zend_ssa *ss static int zend_jit_op_array_analyze1(zend_op_array *op_array, zend_script *script, zend_ssa *ssa, uint32_t *flags) { - if (zend_build_cfg(&CG(arena), op_array, ZEND_CFG_STACKLESS | ZEND_CFG_RECV_ENTRY | ZEND_RT_CONSTANTS | ZEND_SSA_RC_INFERENCE, &ssa->cfg, flags) != SUCCESS) { + if (zend_build_cfg(&CG(arena), op_array, ZEND_CFG_STACKLESS | ZEND_CFG_RECV_ENTRY | ZEND_RT_CONSTANTS | ZEND_CFG_NO_ENTRY_PREDECESSORS | ZEND_SSA_RC_INFERENCE, &ssa->cfg, flags) != SUCCESS) { return FAILURE; } From 7c299088fc6e92570b497db46068f08c44f8c459 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 13 Sep 2016 13:18:32 +0300 Subject: [PATCH 179/569] Separate zval type setting from value setting (type setting may be eliminated in some cases) --- ext/opcache/jit/zend_jit_x86.dasc | 50 +++++++++++++++++-------------- 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index da16bdfa986ca..9f0ac893014d9 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -137,6 +137,10 @@ static void* dasm_labels[zend_lb_MAX]; ||} |.endmacro +|.macro SET_TYPE_INFO, dst_base, dst_offset, type +| mov dword [dst_base + dst_offset + 8], type +|.endmacro + |.macro FP_OP, fp_ins, op_type, op ||if (op_type == IS_CONST) { | .if X64 @@ -172,8 +176,7 @@ static void* dasm_labels[zend_lb_MAX]; |.endmacro |.macro FP_STORE, op -| fstp qword [FP + op.var] -| mov dword [FP + op.var + 8], IS_DOUBLE +| fstp qword [FP + op.var] |.endmacro |.macro SSE_OP, sse_ins, op_type, op, reg @@ -245,8 +248,7 @@ static void* dasm_labels[zend_lb_MAX]; |.endmacro |.macro SSE_STORE, op, reg -| movsd qword [FP + op.var], reg -| mov dword [FP + op.var + 8], IS_DOUBLE +| movsd qword [FP + op.var], reg |.endmacro |.macro LONG_OP, long_ins, op_type, op, reg @@ -351,8 +353,7 @@ static void* dasm_labels[zend_lb_MAX]; |.endmacro |.macro LONG_STORE, op, reg -| mov aword [FP + op.var], reg -| mov dword [FP + op.var + 8], IS_LONG +| mov aword [FP + op.var], reg |.endmacro |.macro ZVAL_COPY_CONST, dst_base, dst_offset, dst_info, zv, tmp_reg @@ -394,7 +395,7 @@ static void* dasm_labels[zend_lb_MAX]; || } ||} ||if ((dst_info & MAY_BE_ANY) != (1<op1.var + 4], 0xc1e00000 |.endif } - | mov dword [FP + opline->op1.var + 8], IS_DOUBLE + | SET_TYPE_INFO FP, opline->op1.var, IS_DOUBLE || if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && || opline->result_type != IS_UNUSED) { | ZVAL_COPY_VALUE FP, opline->result.var, FP, opline->op1.var, MAY_BE_DOUBLE, r0, eax, r1 @@ -940,6 +941,7 @@ static int zend_jit_math_long_long(dasm_State **Dst, zend_op *opline, zend_op_ar || } if ((res_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) { | LONG_STORE opline->result, r0 + | SET_TYPE_INFO FP, opline->result.var, IS_LONG } else { | jo >1 |.cold_code @@ -955,9 +957,11 @@ static int zend_jit_math_long_long(dasm_State **Dst, zend_op *opline, zend_op_ar | LONGF_MATH opline->opcode | FP_STORE opline->result |.endif + | SET_TYPE_INFO FP, opline->result.var, IS_DOUBLE | jmp >2 |.code | LONG_STORE opline->result, r0 + | SET_TYPE_INFO FP, opline->result.var, IS_LONG |2: } @@ -975,6 +979,7 @@ static int zend_jit_math_long_double(dasm_State **Dst, zend_op *opline) | FP_MATH opline->opcode, opline->op2_type, opline->op2 | FP_STORE opline->result |.endif + | SET_TYPE_INFO FP, opline->result.var, IS_DOUBLE return 1; } @@ -985,13 +990,12 @@ static int zend_jit_math_double_long(dasm_State **Dst, zend_op *opline) || if (opline->opcode == ZEND_ADD || opline->opcode == ZEND_MUL) { | SSE_LOAD_LONG opline->op2_type, opline->op2, xmm0 | SSE_MATH opline->opcode, opline->op1_type, opline->op1, xmm0 - | SSE_STORE opline->result, xmm0 || } else { | SSE_LOAD opline->op1_type, opline->op1, xmm0 | SSE_LOAD_LONG opline->op2_type, opline->op2, xmm1 | SSE_MATH2 opline->opcode, xmm1, xmm0 - | SSE_STORE opline->result, xmm0 || } + | SSE_STORE opline->result, xmm0 |.else | LONGF_LOAD opline->op2_type, opline->op2 || if (opline->opcode == ZEND_ADD || opline->opcode == ZEND_MUL) { @@ -1002,6 +1006,7 @@ static int zend_jit_math_double_long(dasm_State **Dst, zend_op *opline) || } | FP_STORE opline->result |.endif + | SET_TYPE_INFO FP, opline->result.var, IS_DOUBLE return 1; } @@ -1023,6 +1028,7 @@ static int zend_jit_math_double_double(dasm_State **Dst, zend_op *opline) | FP_MATH opline->opcode, opline->op2_type, opline->op2 | FP_STORE opline->result |.endif + | SET_TYPE_INFO FP, opline->result.var, IS_DOUBLE return 1; } @@ -1299,7 +1305,7 @@ static int zend_jit_cmp_long_long(dasm_State **Dst, zend_op *opline, int b, zend | add eax, 2 break; } - | mov dword [FP + opline->result.var + 8], eax + | SET_TYPE_INFO FP, opline->result.var, eax } return 1; @@ -1385,7 +1391,7 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, zend_op *opline, int b, | add eax, 2 break; } - | mov dword [FP + opline->result.var + 8], eax + | SET_TYPE_INFO FP, opline->result.var, eax } return 1; @@ -1504,7 +1510,7 @@ static int zend_jit_cmp_slow(dasm_State **Dst, zend_op *opline, int b, zend_op_a | add eax, 2 break; } - | mov dword [FP + opline->result.var + 8], eax + | SET_TYPE_INFO FP, opline->result.var, eax } return 1; @@ -1933,7 +1939,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, zend_op *opline, zend_op_array *o | mov aword EX:r1->call, 0 || if (RETURN_VALUE_USED(opline)) { | // ZVAL_NULL(EX_VAR(opline->result.var)); - | mov dword [FP + opline->result.var + 8], IS_NULL + | SET_TYPE_INFO FP, opline->result.var, IS_NULL | // EX(return_value) = EX_VAR(opline->result.var); | lea r0, aword [FP + opline->result.var] | mov aword EX:r1->return_value, r0 @@ -1944,7 +1950,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, zend_op *opline, zend_op_array *o | || for (i = call_info->num_args; i < func->op_array.last_var; i++) { || uint32_t n = (uint32_t)(uintptr_t)ZEND_CALL_VAR_NUM(NULL, i); - | mov dword [r1 + n + 8], IS_UNDEF + | SET_TYPE_INFO r1, n, IS_UNDEF || } | | //EX_LOAD_RUN_TIME_CACHE(op_array); @@ -2014,7 +2020,7 @@ static int zend_jit_send_val(dasm_State **Dst, zend_op *opline, zend_op_array *o } |.cold_code |1: - | mov dword [r1 + opline->result.var + 8], IS_UNDEF + | SET_TYPE_INFO r1, opline->result.var, IS_UNDEF | mov EX->opline, IP |.if X64 | mov CARG1, 0 @@ -2105,7 +2111,7 @@ static int zend_jit_type_check(dasm_State **Dst, zend_op *opline, int b, int *op target_label = ssa->cfg.blocks[b].successors[0]; | jmp =>target_label } else { - | mov dword [FP + opline->result.var + 8], IS_TRUE + | SET_TYPE_INFO FP, opline->result.var, IS_TRUE valid_opline_offset++; } } else if (!(op1_info & (1<result.var + 8], IS_FALSE + | SET_TYPE_INFO FP, opline->result.var, IS_FALSE valid_opline_offset++; } } else { From 900f647a9d10cea65f57145dbfae49bd78d98f7a Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 13 Sep 2016 14:06:25 +0300 Subject: [PATCH 180/569] Renamed macros --- ext/opcache/jit/zend_jit_x86.dasc | 86 ++++++++++++++++--------------- 1 file changed, 45 insertions(+), 41 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 9f0ac893014d9..b10803657df11 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -141,7 +141,7 @@ static void* dasm_labels[zend_lb_MAX]; | mov dword [dst_base + dst_offset + 8], type |.endmacro -|.macro FP_OP, fp_ins, op_type, op +|.macro FPU_OP, fp_ins, op_type, op ||if (op_type == IS_CONST) { | .if X64 | mov r0, aword EX->literals @@ -154,29 +154,29 @@ static void* dasm_labels[zend_lb_MAX]; ||} |.endmacro -|.macro FP_LOAD, op_type, op -| FP_OP fld, op_type, op +|.macro FPU_LOAD, op_type, op +| FPU_OP fld, op_type, op |.endmacro -|.macro FP_MATH, opcode, op_type, op +|.macro FPU_MATH, opcode, op_type, op ||switch (opcode) { || case ZEND_ADD: -| FP_OP fadd, op_type, op +| FPU_OP fadd, op_type, op || break; || case ZEND_SUB: -| FP_OP fsub, op_type, op +| FPU_OP fsub, op_type, op || break; || case ZEND_MUL: -| FP_OP fmul, op_type, op +| FPU_OP fmul, op_type, op || break; || case ZEND_DIV: -| FP_OP fdiv, op_type, op +| FPU_OP fdiv, op_type, op || break; ||} |.endmacro -|.macro FP_STORE, op -| fstp qword [FP + op.var] +|.macro FPU_STORE, dst_base, dst_offset +| fstp qword [dst_base + dst_offset] |.endmacro |.macro SSE_OP, sse_ins, op_type, op, reg @@ -247,8 +247,16 @@ static void* dasm_labels[zend_lb_MAX]; ||} |.endmacro -|.macro SSE_STORE, op, reg -| movsd qword [FP + op.var], reg +|.macro SSE_STORE, dst_base, dst_offset, reg +| movsd qword [dst_base + dst_offset], reg +|.endmacro + +|.macro DOUBLE_STORE, dst_base, dst_offset, reg +|.if X64 or SSE +| SSE_STORE dst_base, dst_offset, reg +|.else +| FPU_STORE dst_base, dst_offset +|.endif |.endmacro |.macro LONG_OP, long_ins, op_type, op, reg @@ -318,7 +326,7 @@ static void* dasm_labels[zend_lb_MAX]; ||} |.endmacro -|.macro LONGF_OP, fp_ins, op_type, op +|.macro FPU_LONG_OP, fp_ins, op_type, op ||if (op_type == IS_CONST) { | .if X64 | mov r0, aword EX->literals @@ -331,11 +339,11 @@ static void* dasm_labels[zend_lb_MAX]; ||} |.endmacro -|.macro LONGF_LOAD, op_type, op -| LONGF_OP fild, op_type, op +|.macro FPU_LONG_LOAD, op_type, op +| FPU_LONG_OP fild, op_type, op |.endmacro -|.macro LONGF_MATH, opcode +|.macro FPU_MATH2, opcode ||switch (opcode) { || case ZEND_ADD: | fadd st1 @@ -950,13 +958,12 @@ static int zend_jit_math_long_long(dasm_State **Dst, zend_op *opline, zend_op_ar | SSE_LOAD_LONG opline->op1_type, opline->op1, xmm0 | SSE_LOAD_LONG opline->op2_type, opline->op2, xmm1 | SSE_MATH2 opline->opcode, xmm1, xmm0 - | SSE_STORE opline->result, xmm0 |.else - | LONGF_LOAD opline->op2_type, opline->op2 - | LONGF_LOAD opline->op1_type, opline->op1 - | LONGF_MATH opline->opcode - | FP_STORE opline->result + | FPU_LONG_LOAD opline->op2_type, opline->op2 + | FPU_LONG_LOAD opline->op1_type, opline->op1 + | FPU_MATH2 opline->opcode |.endif + | DOUBLE_STORE FP, opline->result.var, xmm0 | SET_TYPE_INFO FP, opline->result.var, IS_DOUBLE | jmp >2 |.code @@ -973,12 +980,11 @@ static int zend_jit_math_long_double(dasm_State **Dst, zend_op *opline) |.if X64 or SSE | SSE_LOAD_LONG opline->op1_type, opline->op1, xmm0 | SSE_MATH opline->opcode, opline->op2_type, opline->op2, xmm0 - | SSE_STORE opline->result, xmm0 |.else - | LONGF_LOAD opline->op1_type, opline->op1 - | FP_MATH opline->opcode, opline->op2_type, opline->op2 - | FP_STORE opline->result + | FPU_LONG_LOAD opline->op1_type, opline->op1 + | FPU_MATH opline->opcode, opline->op2_type, opline->op2 |.endif + | DOUBLE_STORE FP, opline->result.var, xmm0 | SET_TYPE_INFO FP, opline->result.var, IS_DOUBLE return 1; @@ -995,17 +1001,16 @@ static int zend_jit_math_double_long(dasm_State **Dst, zend_op *opline) | SSE_LOAD_LONG opline->op2_type, opline->op2, xmm1 | SSE_MATH2 opline->opcode, xmm1, xmm0 || } - | SSE_STORE opline->result, xmm0 |.else - | LONGF_LOAD opline->op2_type, opline->op2 + | FPU_LONG_LOAD opline->op2_type, opline->op2 || if (opline->opcode == ZEND_ADD || opline->opcode == ZEND_MUL) { - | FP_MATH opline->opcode, opline->op1_type, opline->op1 + | FPU_MATH opline->opcode, opline->op1_type, opline->op1 || } else { - | FP_LOAD opline->op1_type, opline->op1 - | LONGF_MATH opline->opcode + | FPU_LOAD opline->op1_type, opline->op1 + | FPU_MATH2 opline->opcode || } - | FP_STORE opline->result |.endif + | DOUBLE_STORE FP, opline->result.var, xmm0 | SET_TYPE_INFO FP, opline->result.var, IS_DOUBLE return 1; @@ -1022,12 +1027,11 @@ static int zend_jit_math_double_double(dasm_State **Dst, zend_op *opline) } else { | SSE_MATH opline->opcode, opline->op2_type, opline->op2, xmm0 } - | SSE_STORE opline->result, xmm0 |.else - | FP_LOAD opline->op1_type, opline->op1 - | FP_MATH opline->opcode, opline->op2_type, opline->op2 - | FP_STORE opline->result + | FPU_LOAD opline->op1_type, opline->op1 + | FPU_MATH opline->opcode, opline->op2_type, opline->op2 |.endif + | DOUBLE_STORE FP, opline->result.var, xmm0 | SET_TYPE_INFO FP, opline->result.var, IS_DOUBLE return 1; @@ -1403,8 +1407,8 @@ static int zend_jit_cmp_long_double(dasm_State **Dst, zend_op *opline, int b, ze | SSE_LOAD_LONG opline->op1_type, opline->op1, xmm0 | SSE_OP ucomisd, opline->op2_type, opline->op2, xmm0 |.else - | FP_LOAD opline->op2_type, opline->op2 - | LONGF_LOAD opline->op1_type, opline->op1 + | FPU_LOAD opline->op2_type, opline->op2 + | FPU_LONG_LOAD opline->op1_type, opline->op1 | fucomip st1 | fstp st0 |.endif @@ -1419,8 +1423,8 @@ static int zend_jit_cmp_double_long(dasm_State **Dst, zend_op *opline, int b, ze | SSE_LOAD_LONG opline->op2_type, opline->op2, xmm1 | ucomisd xmm0, xmm1 |.else - | LONGF_LOAD opline->op2_type, opline->op2 - | FP_LOAD opline->op1_type, opline->op1 + | FPU_LONG_LOAD opline->op2_type, opline->op2 + | FPU_LOAD opline->op1_type, opline->op1 | fucomip st1 | fstp st0 |.endif @@ -1434,8 +1438,8 @@ static int zend_jit_cmp_double_double(dasm_State **Dst, zend_op *opline, int b, | SSE_LOAD opline->op1_type, opline->op1, xmm0 | SSE_OP ucomisd, opline->op2_type, opline->op2, xmm0 |.else - | FP_LOAD opline->op2_type, opline->op2 - | FP_LOAD opline->op1_type, opline->op1 + | FPU_LOAD opline->op2_type, opline->op2 + | FPU_LOAD opline->op1_type, opline->op1 | fucomip st1 | fstp st0 |.endif From 672298f09f5780514b36059f350eebc734e82c8b Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 13 Sep 2016 14:48:58 +0300 Subject: [PATCH 181/569] Produce better code for MATH + SEND_VAL. --- ext/opcache/jit/zend_jit.c | 2 +- ext/opcache/jit/zend_jit_x86.dasc | 107 ++++++++++++++++++++++-------- 2 files changed, 79 insertions(+), 30 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 8baa1380b1b4a..6a504c9185b99 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -789,7 +789,7 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) case ZEND_SUB: case ZEND_MUL: // case ZEND_DIV: // TODO: check for division by zero ??? - if (!zend_jit_math(&dasm_state, opline, op_array, ssa)) { + if (!zend_jit_math(&dasm_state, opline, &i, op_array, ssa)) { goto jit_failure; } break; diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index b10803657df11..30b69e8343117 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -360,8 +360,8 @@ static void* dasm_labels[zend_lb_MAX]; ||} |.endmacro -|.macro LONG_STORE, op, reg -| mov aword [FP + op.var], reg +|.macro LONG_STORE, dst_base, dst_offset, reg +| mov aword [dst_base + dst_offset], reg |.endmacro |.macro ZVAL_COPY_CONST, dst_base, dst_offset, dst_info, zv, tmp_reg @@ -936,7 +936,7 @@ fallback: return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); } -static int zend_jit_math_long_long(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) +static int zend_jit_math_long_long(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, uint32_t res_forward) { uint32_t res_info = RES_INFO(); zend_bool same_ops = (opline->op1_type == opline->op2_type) && (opline->op1.var == opline->op2.var); @@ -948,8 +948,13 @@ static int zend_jit_math_long_long(dasm_State **Dst, zend_op *opline, zend_op_ar | LONG_MATH opline->opcode, opline->op2_type, opline->op2, r0 || } if ((res_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) { - | LONG_STORE opline->result, r0 - | SET_TYPE_INFO FP, opline->result.var, IS_LONG + if (res_forward) { + | LONG_STORE r1, res_forward, r0 + | SET_TYPE_INFO r1, res_forward, IS_LONG + } else { + | LONG_STORE FP, opline->result.var, r0 + | SET_TYPE_INFO FP, opline->result.var, IS_LONG + } } else { | jo >1 |.cold_code @@ -963,19 +968,29 @@ static int zend_jit_math_long_long(dasm_State **Dst, zend_op *opline, zend_op_ar | FPU_LONG_LOAD opline->op1_type, opline->op1 | FPU_MATH2 opline->opcode |.endif - | DOUBLE_STORE FP, opline->result.var, xmm0 - | SET_TYPE_INFO FP, opline->result.var, IS_DOUBLE + if (res_forward) { + | DOUBLE_STORE r1, res_forward, xmm0 + | SET_TYPE_INFO r1, res_forward, IS_DOUBLE + } else { + | DOUBLE_STORE FP, opline->result.var, xmm0 + | SET_TYPE_INFO FP, opline->result.var, IS_DOUBLE + } | jmp >2 |.code - | LONG_STORE opline->result, r0 - | SET_TYPE_INFO FP, opline->result.var, IS_LONG + if (res_forward) { + | LONG_STORE r1, res_forward, r0 + | SET_TYPE_INFO r1, res_forward, IS_LONG + } else { + | LONG_STORE FP, opline->result.var, r0 + | SET_TYPE_INFO FP, opline->result.var, IS_LONG + } |2: } return 1; } -static int zend_jit_math_long_double(dasm_State **Dst, zend_op *opline) +static int zend_jit_math_long_double(dasm_State **Dst, zend_op *opline, uint32_t res_forward) { |.if X64 or SSE | SSE_LOAD_LONG opline->op1_type, opline->op1, xmm0 @@ -984,13 +999,18 @@ static int zend_jit_math_long_double(dasm_State **Dst, zend_op *opline) | FPU_LONG_LOAD opline->op1_type, opline->op1 | FPU_MATH opline->opcode, opline->op2_type, opline->op2 |.endif - | DOUBLE_STORE FP, opline->result.var, xmm0 - | SET_TYPE_INFO FP, opline->result.var, IS_DOUBLE + if (res_forward) { + | DOUBLE_STORE r1, res_forward, xmm0 + | SET_TYPE_INFO r1, res_forward, IS_DOUBLE + } else { + | DOUBLE_STORE FP, opline->result.var, xmm0 + | SET_TYPE_INFO FP, opline->result.var, IS_DOUBLE + } return 1; } -static int zend_jit_math_double_long(dasm_State **Dst, zend_op *opline) +static int zend_jit_math_double_long(dasm_State **Dst, zend_op *opline, uint32_t res_forward) { |.if X64 or SSE || if (opline->opcode == ZEND_ADD || opline->opcode == ZEND_MUL) { @@ -1010,13 +1030,18 @@ static int zend_jit_math_double_long(dasm_State **Dst, zend_op *opline) | FPU_MATH2 opline->opcode || } |.endif - | DOUBLE_STORE FP, opline->result.var, xmm0 - | SET_TYPE_INFO FP, opline->result.var, IS_DOUBLE + if (res_forward) { + | DOUBLE_STORE r1, res_forward, xmm0 + | SET_TYPE_INFO r1, res_forward, IS_DOUBLE + } else { + | DOUBLE_STORE FP, opline->result.var, xmm0 + | SET_TYPE_INFO FP, opline->result.var, IS_DOUBLE + } return 1; } -static int zend_jit_math_double_double(dasm_State **Dst, zend_op *opline) +static int zend_jit_math_double_double(dasm_State **Dst, zend_op *opline, uint32_t res_forward) { zend_bool same_ops = (opline->op1_type == opline->op2_type) && (opline->op1.var == opline->op2.var); @@ -1031,17 +1056,23 @@ static int zend_jit_math_double_double(dasm_State **Dst, zend_op *opline) | FPU_LOAD opline->op1_type, opline->op1 | FPU_MATH opline->opcode, opline->op2_type, opline->op2 |.endif - | DOUBLE_STORE FP, opline->result.var, xmm0 - | SET_TYPE_INFO FP, opline->result.var, IS_DOUBLE + if (res_forward) { + | DOUBLE_STORE r1, res_forward, xmm0 + | SET_TYPE_INFO r1, res_forward, IS_DOUBLE + } else { + | DOUBLE_STORE FP, opline->result.var, xmm0 + | SET_TYPE_INFO FP, opline->result.var, IS_DOUBLE + } return 1; } -static int zend_jit_math(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) +static int zend_jit_math(dasm_State **Dst, zend_op *opline, int *opnum, zend_op_array *op_array, zend_ssa *ssa) { uint32_t op1_info, op2_info; zend_bool same_ops = (opline->op1_type == opline->op2_type) && (opline->op1.var == opline->op2.var); zend_bool has_slow = 0; + uint32_t res_forward = 0; if (!ssa->ops || !ssa->var_info) { goto fallback; @@ -1059,6 +1090,16 @@ static int zend_jit_math(dasm_State **Dst, zend_op *opline, zend_op_array *op_ar goto fallback; } + if (opline->result_type == IS_TMP_VAR && + (opline+1)->opcode == ZEND_SEND_VAL && + (opline+1)->op1_type == IS_TMP_VAR && + (opline+1)->op1.var == opline->result.var) { + /* Eliminate the following SEND_VAL */ + (*opnum)++; + res_forward = (opline+1)->result.var; + | mov r1, EX->call + } + if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)) { if (op1_info & (MAY_BE_ANY-MAY_BE_LONG)) { | cmp dword [FP + opline->op1.var + 8], IS_LONG @@ -1078,7 +1119,7 @@ static int zend_jit_math(dasm_State **Dst, zend_op *opline, zend_op_array *op_ar | cmp dword [FP + opline->op2.var + 8], IS_DOUBLE | JNE_SLOW } - if (!zend_jit_math_long_double(Dst, opline)) { + if (!zend_jit_math_long_double(Dst, opline, res_forward)) { return 0; } | jmp >6 @@ -1087,7 +1128,7 @@ static int zend_jit_math(dasm_State **Dst, zend_op *opline, zend_op_array *op_ar | JNE_SLOW } } - if (!zend_jit_math_long_long(Dst, opline, op_array, ssa)) { + if (!zend_jit_math_long_long(Dst, opline, op_array, ssa, res_forward)) { return 0; } if (op1_info & MAY_BE_DOUBLE) { @@ -1106,7 +1147,7 @@ static int zend_jit_math(dasm_State **Dst, zend_op *opline, zend_op_array *op_ar | JNE_SLOW } } - if (!zend_jit_math_double_double(Dst, opline)) { + if (!zend_jit_math_double_double(Dst, opline, res_forward)) { return 0; } | jmp >6 @@ -1117,7 +1158,7 @@ static int zend_jit_math(dasm_State **Dst, zend_op *opline, zend_op_array *op_ar | cmp dword [FP + opline->op2.var + 8], IS_LONG | JNE_SLOW } - if (!zend_jit_math_double_long(Dst, opline)) { + if (!zend_jit_math_double_long(Dst, opline, res_forward)) { return 0; } | jmp >6 @@ -1140,7 +1181,7 @@ static int zend_jit_math(dasm_State **Dst, zend_op *opline, zend_op_array *op_ar | JNE_SLOW } } - if (!zend_jit_math_double_double(Dst, opline)) { + if (!zend_jit_math_double_double(Dst, opline, res_forward)) { return 0; } } @@ -1153,7 +1194,7 @@ static int zend_jit_math(dasm_State **Dst, zend_op *opline, zend_op_array *op_ar | cmp dword [FP + opline->op2.var + 8], IS_LONG | JNE_SLOW } - if (!zend_jit_math_double_long(Dst, opline)) { + if (!zend_jit_math_double_long(Dst, opline, res_forward)) { return 0; } if (op2_info & MAY_BE_DOUBLE) { @@ -1177,7 +1218,7 @@ static int zend_jit_math(dasm_State **Dst, zend_op *opline, zend_op_array *op_ar | JNE_SLOW } } - if (!zend_jit_math_double_double(Dst, opline)) { + if (!zend_jit_math_double_double(Dst, opline, res_forward)) { return 0; } } @@ -1190,7 +1231,7 @@ static int zend_jit_math(dasm_State **Dst, zend_op *opline, zend_op_array *op_ar | cmp dword [FP + opline->op1.var + 8], IS_LONG | JNE_SLOW } - if (!zend_jit_math_long_double(Dst, opline)) { + if (!zend_jit_math_long_double(Dst, opline, res_forward)) { return 0; } if (op1_info & MAY_BE_DOUBLE) { @@ -1201,13 +1242,21 @@ static int zend_jit_math(dasm_State **Dst, zend_op *opline, zend_op_array *op_ar } |6: - valid_opline_offset++; + if (res_forward) { + valid_opline_offset += 2; + } else { + valid_opline_offset++; + } if (has_slow) { |.cold_code |9: | mov EX->opline, IP - | lea FCARG1a, [FP + opline->result.var] + if (res_forward) { + | lea FCARG1a, [r1 + res_forward] + } else { + | lea FCARG1a, [FP + opline->result.var] + } | LOAD_ZVAL_ADDR FCARG2a, opline->op1_type, opline->op1 |.if X64 | LOAD_ZVAL_ADDR CARG3, opline->op2_type, opline->op2 From fe1d7f0265a996d58b8cdef0136a9298db9d92e1 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 13 Sep 2016 17:43:32 +0300 Subject: [PATCH 182/569] Fixed conditions --- ext/opcache/jit/zend_jit_x86.dasc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 30b69e8343117..f4acb46e54dcd 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -1434,12 +1434,12 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, zend_op *opline, int b, |2: break; case ZEND_IS_SMALLER: - | seta al + | setnae al | movzx eax, al | add eax, 2 break; case ZEND_IS_SMALLER_OR_EQUAL: - | setae al + | setna al | movzx eax, al | add eax, 2 break; From b25f0e93aa1b4234f37ee533335793ce8dfacdfb Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 14 Sep 2016 09:55:27 +0300 Subject: [PATCH 183/569] Replace ZVAL_COPY_VALUE_REG with ZVAL_COPY_VALUE --- ext/opcache/jit/zend_jit_x86.dasc | 41 +++---------------------------- 1 file changed, 3 insertions(+), 38 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index f4acb46e54dcd..38dfdd353382f 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -442,41 +442,6 @@ static void* dasm_labels[zend_lb_MAX]; ||} |.endmacro -|.macro ZVAL_COPY_VALUE_REG, dst_base, dst_offset, src_reg, src_info, tmp_reg1, tmp_reg1d, tmp_reg2 -||if (src_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) { -|| if (!(src_info & MAY_BE_DOUBLE)) { -| mov tmp_reg2, aword [src_reg] -| mov aword [dst_base + dst_offset], tmp_reg2 -|| } else if ((src_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { -|.if X64 or SSE -| movsd xmm0, qword [src_reg] -| movsd qword [dst_base + dst_offset], xmm0 -|.else -| fld qword [src_reg] -| fstp qword [dst_base + dst_offset] -|.endif -|| } else { -|.if X64 -| mov tmp_reg2, aword [src_reg] -| mov aword [dst_base + dst_offset], tmp_reg2 -|.else -| mov tmp_reg2, dword [src_reg] -| mov tmp_reg1, dword [src_reg + 4] -| mov dword [dst_base + dst_offset], tmp_reg2 -| mov dword [dst_base + dst_offset + 4], tmp_reg1 -|.endif -|| } -||} -||if ((src_info & (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE)) && -|| has_concrete_type(src_info & MAY_BE_ANY)) { -|| zend_uchar type = concrete_type(src_info); -| SET_TYPE_INFO dst_base, dst_offset, type -||} else { -| mov tmp_reg1d, dword [src_reg + 8] -| SET_TYPE_INFO dst_base, dst_offset, tmp_reg1d -||} -|.endmacro - |.macro ADDREF_CONST, zv, tmp_reg |.if X64 || if (!IS_32BIT(Z_LVAL_P(zv))) { @@ -906,7 +871,7 @@ static int zend_jit_inc_dec(dasm_State **Dst, zend_op *opline, zend_op_array *op | ZVAL_DEREF FCARG1a, op1_info || if ((opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC)) { || if (opline->result_type != IS_UNUSED) { - | ZVAL_COPY_VALUE_REG FP, opline->result.var, FCARG1a, op1_info, r0, eax, r2 + | ZVAL_COPY_VALUE FP, opline->result.var, FCARG1a, 0, op1_info, r0, eax, r2 | // zval_opt_copy_ctor(var_ptr); | // ??? | TRY_ADDREF op1_info, ah, r2 @@ -2226,7 +2191,7 @@ static int zend_jit_return(dasm_State **Dst, zend_op *opline, zend_op_array *op_ if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { | lea r0, [FP + opline->op1.var] | ZVAL_DEREF r0, op1_info - | ZVAL_COPY_VALUE_REG r1, 0, r0, op1_info, r0, eax, r2 + | ZVAL_COPY_VALUE r1, 0, r0, 0, op1_info, r0, eax, r2 } else { | ZVAL_COPY_VALUE r1, 0, FP, opline->op1.var, op1_info, r0, eax, r2 } @@ -2239,7 +2204,7 @@ static int zend_jit_return(dasm_State **Dst, zend_op *opline, zend_op_array *op_ | jne >4 | mov FCARG1a, [FCARG1a] | add FCARG1a, 8 - | ZVAL_COPY_VALUE_REG r1, 0, FCARG1a, op1_info, r1, ebx, r2 + | ZVAL_COPY_VALUE r1, 0, FCARG1a, 0, op1_info, r1, ebx, r2 | sub FCARG1a, 8 | dec dword [r0] | jne >3 From a07b7140479a742316e7b081506f0e7eda901263 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 14 Sep 2016 10:05:26 +0300 Subject: [PATCH 184/569] Fixed macro --- ext/opcache/jit/zend_jit_x86.dasc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 38dfdd353382f..fa8ef7e48f97b 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -67,6 +67,8 @@ static void* dasm_labels[zend_lb_MAX]; |.section code, cold_code +#define IS_32BIT(addr) (((uintptr_t)(addr)) <= 0x7fffffff) + |.macro LOAD_ADDR, reg, addr |.if X64 ||if (((ptrdiff_t)addr) <= 0x7fffffff) { @@ -105,8 +107,6 @@ static void* dasm_labels[zend_lb_MAX]; |.endif |.endmacro -#define IS_32BIT(addr) (((uintptr_t)(addr)) <= 0x7fffffff) - #if ZEND_JIT_LEVEL >= ZEND_JIT_LEVEL_OT_FUNC |.macro LOAD_ZVAL_ADDR, reg, op_type, op @@ -125,9 +125,9 @@ static void* dasm_labels[zend_lb_MAX]; |.macro PUSH_ZVAL_ADDR, op_type, op, tmp_reg ||if (op_type == IS_CONST) { | .if X64 -| mov reg, aword EX->literals -| add reg, op.constant -| push reg +| mov tmp_reg, aword EX->literals +| add tmp_reg, op.constant +| push tmp_reg | .else | push op.zv | .endif From 0eaaef639e7a160916fd1f9aefedaf8199f96e43 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 14 Sep 2016 10:13:24 +0300 Subject: [PATCH 185/569] Split ZVAL_PTR_DTOR from FREE_OP macro --- ext/opcache/jit/zend_jit_x86.dasc | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index fa8ef7e48f97b..1282d4d0405fa 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -481,12 +481,10 @@ static void* dasm_labels[zend_lb_MAX]; || has_slow = 1; |.endmacro -|.macro FREE_OP, op_type, op, op_info, op_array, lineno -||if ((op_type & (IS_VAR|IS_TMP_VAR)) && -|| (op_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { -| and byte [FP + op.var + 9], IS_TYPE_REFCOUNTED +|.macro ZVAL_PTR_DTOR, base, offset, op_array, lineno +| and byte [base + offset + 9], IS_TYPE_REFCOUNTED | je >1 -| mov FCARG1a, aword [FP + op.var] +| mov FCARG1a, aword [base + offset] | dec dword [FCARG1a] | jnz >1 || if (ZEND_DEBUG) { @@ -498,8 +496,14 @@ static void* dasm_labels[zend_lb_MAX]; | push lineno |.endif || } -| EXT_CALL _zval_dtor_func, r0 -| 1: +| EXT_CALL _zval_dtor_func, r0 +|1: +|.endmacro + +|.macro FREE_OP, op_type, op, op_info, op_array, lineno +||if ((op_type & (IS_VAR|IS_TMP_VAR)) && +|| (op_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { +| ZVAL_PTR_DTOR FP, op.var, op_array, lineno ||} |.endmacro From c848c467d2bb922db6c78c7b81cea51a4e528093 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 14 Sep 2016 20:07:50 +0300 Subject: [PATCH 186/569] Implemented JIT for RETRUN opcode (works but needs more optimization and cleanup) --- ext/opcache/jit/zend_jit.c | 7 + ext/opcache/jit/zend_jit_disasm_x86.c | 3 +- ext/opcache/jit/zend_jit_helpers.c | 135 +++------ ext/opcache/jit/zend_jit_x86.dasc | 411 +++++++++++++++++++++----- 4 files changed, 386 insertions(+), 170 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 6a504c9185b99..1d10767d1121e 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -838,6 +838,11 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) goto jit_failure; } break; + case ZEND_RETURN: + if (!zend_jit_return(&dasm_state, opline, op_array, ssa)) { + goto jit_failure; + } + break; #endif case ZEND_RECV_INIT: if (ssa->cfg.split_at_recv) { @@ -873,7 +878,9 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) case ZEND_GENERATOR_CREATE: case ZEND_GENERATOR_RETURN: case ZEND_RETURN_BY_REF: +#if ZEND_JIT_LEVEL < ZEND_JIT_LEVEL_OPT_FUNC case ZEND_RETURN: +#endif case ZEND_EXIT: /* switch through trampoline */ case ZEND_YIELD: diff --git a/ext/opcache/jit/zend_jit_disasm_x86.c b/ext/opcache/jit/zend_jit_disasm_x86.c index b1628eccaa6e1..3755b16359705 100644 --- a/ext/opcache/jit/zend_jit_disasm_x86.c +++ b/ext/opcache/jit/zend_jit_disasm_x86.c @@ -384,7 +384,8 @@ static int zend_jit_disasm_init(void) (uint64_t)(uintptr_t)n, sizeof(void*)); REGISTER_HELPER(zend_jit_find_func_helper); REGISTER_HELPER(zend_jit_extend_stack_helper); - REGISTER_HELPER(zend_jit_leave_helper); + REGISTER_HELPER(zend_jit_leave_nested_func_helper); + REGISTER_HELPER(zend_jit_leave_top_func_helper); #undef REGISTER_HELPER zend_elf_load_symbols(); diff --git a/ext/opcache/jit/zend_jit_helpers.c b/ext/opcache/jit/zend_jit_helpers.c index c573a33b4662d..945c769585538 100644 --- a/ext/opcache/jit/zend_jit_helpers.c +++ b/ext/opcache/jit/zend_jit_helpers.c @@ -61,117 +61,52 @@ static zend_always_inline void i_free_compiled_variables(zend_execute_data *exec } } -static int ZEND_FASTCALL zend_jit_leave_helper(zend_execute_data *execute_data) { +static void ZEND_FASTCALL zend_jit_leave_nested_func_helper(uint32_t call_info, zend_execute_data *execute_data) +{ zend_execute_data *old_execute_data; - uint32_t call_info = EX_CALL_INFO(); - - if (EXPECTED((call_info & (ZEND_CALL_CODE|ZEND_CALL_TOP|ZEND_CALL_HAS_SYMBOL_TABLE|ZEND_CALL_FREE_EXTRA_ARGS|ZEND_CALL_ALLOCATED)) == 0)) { - i_free_compiled_variables(execute_data); - EG(current_execute_data) = EX(prev_execute_data); - if (UNEXPECTED(call_info & ZEND_CALL_RELEASE_THIS)) { - zend_object *object = Z_OBJ(execute_data->This); - if (UNEXPECTED(EG(exception) != NULL) && (call_info & ZEND_CALL_CTOR)) { - GC_REFCOUNT(object)--; - zend_object_store_ctor_failed(object); - } - OBJ_RELEASE(object); - } else if (UNEXPECTED(call_info & ZEND_CALL_CLOSURE)) { - OBJ_RELEASE((zend_object*)execute_data->func->op_array.prototype); + if (UNEXPECTED(call_info & ZEND_CALL_HAS_SYMBOL_TABLE)) { + zend_clean_and_cache_symbol_table(EX(symbol_table)); + } + EG(current_execute_data) = EX(prev_execute_data); + if (UNEXPECTED(call_info & ZEND_CALL_RELEASE_THIS)) { + zend_object *object = Z_OBJ(execute_data->This); + if (UNEXPECTED(EG(exception) != NULL) && (call_info & ZEND_CALL_CTOR)) { + GC_REFCOUNT(object)--; + zend_object_store_ctor_failed(object); } - EG(vm_stack_top) = (zval*)execute_data; - execute_data = EX(prev_execute_data); + OBJ_RELEASE(object); + } else if (UNEXPECTED(call_info & ZEND_CALL_CLOSURE)) { + OBJ_RELEASE((zend_object*)execute_data->func->op_array.prototype); + } - if (UNEXPECTED(EG(exception) != NULL)) { - const zend_op *old_opline = EX(opline); - zend_throw_exception_internal(NULL); - if (RETURN_VALUE_USED(old_opline)) { - zval_ptr_dtor(EX_VAR(old_opline->result.var)); - } - return 2; + zend_vm_stack_free_extra_args_ex(call_info, execute_data); + old_execute_data = execute_data; + execute_data = EX(prev_execute_data); + zend_vm_stack_free_call_frame_ex(call_info, old_execute_data); + + if (UNEXPECTED(EG(exception) != NULL)) { + const zend_op *old_opline = EX(opline); + zend_throw_exception_internal(NULL); + if (RETURN_VALUE_USED(old_opline)) { + zval_ptr_dtor(EX_VAR(old_opline->result.var)); } + } else { EX(opline)++; - return 0; - } else if (EXPECTED((call_info & (ZEND_CALL_CODE|ZEND_CALL_TOP)) == 0)) { - i_free_compiled_variables(execute_data); + } +} +static void ZEND_FASTCALL zend_jit_leave_top_func_helper(uint32_t call_info, zend_execute_data *execute_data) +{ + if (UNEXPECTED(call_info & (ZEND_CALL_HAS_SYMBOL_TABLE|ZEND_CALL_FREE_EXTRA_ARGS))) { if (UNEXPECTED(call_info & ZEND_CALL_HAS_SYMBOL_TABLE)) { zend_clean_and_cache_symbol_table(EX(symbol_table)); } - EG(current_execute_data) = EX(prev_execute_data); - if (UNEXPECTED(call_info & ZEND_CALL_RELEASE_THIS)) { - zend_object *object = Z_OBJ(execute_data->This); - if (UNEXPECTED(EG(exception) != NULL) && (call_info & ZEND_CALL_CTOR)) { - GC_REFCOUNT(object)--; - zend_object_store_ctor_failed(object); - } - OBJ_RELEASE(object); - } else if (UNEXPECTED(call_info & ZEND_CALL_CLOSURE)) { - OBJ_RELEASE((zend_object*)execute_data->func->op_array.prototype); - } - zend_vm_stack_free_extra_args_ex(call_info, execute_data); - old_execute_data = execute_data; - execute_data = EX(prev_execute_data); - zend_vm_stack_free_call_frame_ex(call_info, old_execute_data); - - if (UNEXPECTED(EG(exception) != NULL)) { - const zend_op *old_opline = EX(opline); - zend_throw_exception_internal(NULL); - if (RETURN_VALUE_USED(old_opline)) { - zval_ptr_dtor(EX_VAR(old_opline->result.var)); - } - return 2; - } - EX(opline)++; - return 0; - } else if (EXPECTED((call_info & ZEND_CALL_TOP) == 0)) { - zend_detach_symbol_table(execute_data); - destroy_op_array(&EX(func)->op_array); - efree_size(EX(func), sizeof(zend_op_array)); - old_execute_data = execute_data; - execute_data = EG(current_execute_data) = EX(prev_execute_data); - zend_vm_stack_free_call_frame_ex(call_info, old_execute_data); - - zend_attach_symbol_table(execute_data); - if (UNEXPECTED(EG(exception) != NULL)) { - zend_throw_exception_internal(NULL); - return 2; - } - EX(opline)++; - return 0; - } else { - if (EXPECTED((call_info & ZEND_CALL_CODE) == 0)) { - i_free_compiled_variables(execute_data); - i_free_compiled_variables(execute_data); - if (UNEXPECTED(call_info & (ZEND_CALL_HAS_SYMBOL_TABLE|ZEND_CALL_FREE_EXTRA_ARGS))) { - if (UNEXPECTED(call_info & ZEND_CALL_HAS_SYMBOL_TABLE)) { - zend_clean_and_cache_symbol_table(EX(symbol_table)); - } - zend_vm_stack_free_extra_args_ex(call_info, execute_data); - } - EG(current_execute_data) = EX(prev_execute_data); - if (UNEXPECTED(call_info & ZEND_CALL_CLOSURE)) { - OBJ_RELEASE((zend_object*)EX(func)->op_array.prototype); - } - return -1; - } else /* if (call_kind == ZEND_CALL_TOP_CODE) */ { - zend_array *symbol_table = EX(symbol_table); - - zend_detach_symbol_table(execute_data); - old_execute_data = EX(prev_execute_data); - while (old_execute_data) { - if (old_execute_data->func && (ZEND_CALL_INFO(old_execute_data) & ZEND_CALL_HAS_SYMBOL_TABLE)) { - if (old_execute_data->symbol_table == symbol_table) { - zend_attach_symbol_table(old_execute_data); - } - break; - } - old_execute_data = old_execute_data->prev_execute_data; - } - EG(current_execute_data) = EX(prev_execute_data); - return -1; - } + } + EG(current_execute_data) = EX(prev_execute_data); + if (UNEXPECTED(call_info & ZEND_CALL_CLOSURE)) { + OBJ_RELEASE((zend_object*)EX(func)->op_array.prototype); } } diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 1282d4d0405fa..08b726c525ace 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -58,7 +58,7 @@ |.endif |.type EX, zend_execute_data, FP -|.type OP, zend_opline, IP +|.type OP, zend_op, IP |.actionlist dasm_actions @@ -458,7 +458,7 @@ static void* dasm_labels[zend_lb_MAX]; |.macro TRY_ADDREF, val_info, type_flags_reg, value_ptr_reg ||if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { || if (val_info & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { -| and type_flags_reg, IS_TYPE_REFCOUNTED +| test type_flags_reg, IS_TYPE_REFCOUNTED | je >1 || } | inc dword [value_ptr_reg] @@ -481,14 +481,8 @@ static void* dasm_labels[zend_lb_MAX]; || has_slow = 1; |.endmacro -|.macro ZVAL_PTR_DTOR, base, offset, op_array, lineno -| and byte [base + offset + 9], IS_TYPE_REFCOUNTED -| je >1 -| mov FCARG1a, aword [base + offset] -| dec dword [FCARG1a] -| jnz >1 +|.macro ZVAL_DTOR_FUNC, filename, lineno // arg1 must be in FCARG1a || if (ZEND_DEBUG) { -|| const char *filename = op_array->filename ? op_array->filename->val : NULL; | LOAD_ADDR FCARG2a, filename |.if X64 | mov CARG3d, lineno @@ -497,16 +491,110 @@ static void* dasm_labels[zend_lb_MAX]; |.endif || } | EXT_CALL _zval_dtor_func, r0 +|.endmacro + +|.macro ZVAL_PTR_DTOR_NOGC, base, offset, filename, lineno +| test byte [base + offset + 9], IS_TYPE_REFCOUNTED +| je >1 +| mov FCARG1a, aword [base + offset] +| dec dword [FCARG1a] +| jnz >1 +| ZVAL_DTOR_FUNC filename, lineno +|1: +|.endmacro + +|.macro ZVAL_PTR_DTOR, base, offset, info, filename, lineno +|| if (info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { +| // if (Z_REFCOUNTED_P(cv)) { +| test byte [base + offset + 9], IS_TYPE_REFCOUNTED +| je >3 +|| } +| // if (!Z_DELREF_P(cv)) { +| mov FCARG1a, aword [base + offset] +| dec dword [FCARG1a] +| jnz >1 +| // ZVAL_NULL(cv); +| mov dword [base + offset + 8], IS_NULL +| // zval_dtor_func(r); +| ZVAL_DTOR_FUNC filename, lineno +| jmp >3 |1: +|| if (info & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) { +|| if (info & MAY_BE_REF) { +| lea r1, [base + offset] +| // ZVAL_DEREF(z); +| ZVAL_DEREF r1, info +| // if (Z_COLLECTABLE_P(z)) +| test byte [r1 + 9], IS_TYPE_COLLECTABLE +| je >3 +| // if (UNEXPECTED(!Z_GC_INFO_P(z))) +| mov FCARG1a, aword [r1] +| cmp word [FCARG1a + 6], 0 +| jne >3 +|| } else { +| // if (Z_COLLECTABLE_P(z)) +| test byte [base + offset + 9], IS_TYPE_COLLECTABLE +| je >3 +| // if (UNEXPECTED(!Z_GC_INFO_P(z))) +| mov FCARG1a, aword [base + offset] +| cmp word [FCARG1a + 6], 0 +| jne >3 +|| } +| // gc_possible_root(Z_COUNTED_P(z)) +| EXT_CALL gc_possible_root, r0 +|| } +|3: |.endmacro |.macro FREE_OP, op_type, op, op_info, op_array, lineno ||if ((op_type & (IS_VAR|IS_TMP_VAR)) && || (op_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { -| ZVAL_PTR_DTOR FP, op.var, op_array, lineno +| ZVAL_PTR_DTOR_NOGC FP, op.var, (op_array->filename ? ZSTR_VAL(op_array->filename) : NULL), lineno ||} |.endmacro +|.macro EFREE_SIZE, ptr, size, op_array, opline +| mov FCARG1a, ptr +|| if (ZEND_DEBUG) { +|| const char *filename = op_array->filename ? op_array->filename->val : NULL; +| LOAD_ADDR FCARG2a, filename +|.if X64 +| mov CARG3d, opline->lineno +| xor CARG4, CARG4 +| xor CARG5, CARG5 +|.else +| push opline->lineno +| push 0 +| push 0 +|.endif +|| } +| EXT_CALL _efree, r0 +|.endmacro + +|.macro OBJ_RELEASE, reg, exit_label +| // if (--GC_REFCOUNT(obj) == 0) { +| dec dword [reg] +| jne >1 +| // zend_objects_store_del(obj); +|.if X64 +| mov CARG1, reg +| EXT_CALL zend_objects_store_del, r0 +|.else +| sub r4, 12 +| push reg +| EXT_CALL zend_objects_store_del, r0 +| add r4, 16 +|.endif +| jmp >exit_label +|1: +| // if (UNEXPECTED(!GC_INFO(obj))) { +| cmp word [reg + 6], 0 +| jne >exit_label +| // gc_possible_root(obj) +| mov FCARG1a, reg +| EXT_CALL gc_possible_root, r0 +|.endmacro + const zend_op *valid_opline; static uint32_t valid_opline_offset; #endif @@ -608,9 +696,67 @@ static int zend_jit_exception_handler_stub(dasm_State **Dst) return 1; } +static int zend_jit_leave_function_stub(dasm_State **Dst) +{ + |->leave_function_handler: + | test FCARG1d, ZEND_CALL_TOP + | mov FCARG2a, FP + | jne >1 + | // TODO: Use tail call ??? + | EXT_CALL zend_jit_leave_nested_func_helper, r0 + | //execute_data = EG(current_execute_data); + | mov FP, aword [&EG(current_execute_data)] + | // LOAD_OPLINE(); + | mov IP, EX->opline + | //ZEND_VM_CONTINUE() + | add r4, SPAD // stack alignment + | jmp aword [IP] + |1: + | // TODO: Use tail call ??? + | EXT_CALL zend_jit_leave_top_func_helper, r0 + | //execute_data = EG(current_execute_data); + | mov FP, aword [&EG(current_execute_data)] + | xor IP, IP + | add r4, SPAD // stack alignment + | ret + + return 1; +} + +static int zend_jit_leave_throw_stub(dasm_State **Dst) +{ + |->leave_throw_handler: + | // if (opline->opcode != ZEND_HANDLE_EXCEPTION) { + | cmp byte OP:IP->opcode, ZEND_HANDLE_EXCEPTION + | je >5 + | // EG(opline_before_exception) = opline; + | mov aword [&EG(opline_before_exception)], IP + | // if (RETURN_VALUE_USED(old_opline)) { + | cmp byte OP:IP->result_type, IS_UNUSED + | je >1 + | // zval_ptr_dtor(EX_VAR(old_opline->result.var)); + |.if X64 + | movsxd r0, dword OP:IP->result.var + |.else + | mov r0, OP:IP->result.var + |.endif + | add r0, FP + | ZVAL_PTR_DTOR r0, 0, MAY_BE_ANY, NULL, 0 + |5: + | // EX(opline) = EG(exception_op); + | LOAD_ADDR IP, &EG(exception_op) + | mov EX->opline, IP + | add r4, SPAD // stack alignment + | EXT_JMP EG(exception_op)->handler, r0 + + return 1; +} + static const zend_jit_stub zend_jit_stubs[] = { JIT_STUB(interrupt_handler), JIT_STUB(exception_handler), + JIT_STUB(leave_function), + JIT_STUB(leave_throw), }; static int zend_jit_align_func(dasm_State **Dst) @@ -2035,7 +2181,7 @@ static int zend_jit_send_val(dasm_State **Dst, zend_op *opline, zend_op_array *o | mov r0, EX:r1->func if (arg_num <= MAX_ARG_FLAG_NUM) { | mov eax, dword [r0 + offsetof(zend_function, quick_arg_flags)] - | and eax, mask + | test eax, mask | jnz >1 } else { ZEND_ASSERT(0); @@ -2163,26 +2309,140 @@ fallback: return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); } +static int zend_jit_free_compiled_variables(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) +{ + uint32_t i, j, info; + + // Use type inference to avoid useless zval_ptr_dtor() + for (i = 0 ; i < op_array->last_var; i++) { + if (ssa->vars && ssa->var_info) { + info = ssa->var_info[i].type; + for (j = op_array->last_var; j < ssa->vars_count; j++) { + if (ssa->vars[j].var == i) { + info |= ssa->var_info[j].type; + } + } + } else { + info = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_UNDEF; + } + + if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { + uint32_t offset = (uint32_t)(uintptr_t)ZEND_CALL_VAR_NUM(NULL, i); + | ZVAL_PTR_DTOR FP, offset, info, (op_array->filename ? ZSTR_VAL(op_array->filename) : NULL), opline->lineno + } + } + return 1; +} + +static int zend_jit_leave_func(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) +{ + // TODO: avoid multiple leave sequnces ??? + + // i_free_compiled_variables(execute_data); + if (!zend_jit_free_compiled_variables(Dst, opline, op_array, ssa)) { + return 0; + } + + | movzx FCARG1d, byte [FP + offsetof(zend_execute_data, This.u1.type_info) + 2] + | test FCARG1d, (ZEND_CALL_TOP|ZEND_CALL_HAS_SYMBOL_TABLE|ZEND_CALL_FREE_EXTRA_ARGS|ZEND_CALL_ALLOCATED) + | jne ->leave_function_handler + | // EG(current_execute_data) = EX(prev_execute_data); + | mov r0, EX->prev_execute_data + | mov aword [&EG(current_execute_data)], r0 + | // if (call_info & ZEND_CALL_RELEASE_THIS) + if (op_array->scope) { + | test FCARG1d, ZEND_CALL_RELEASE_THIS + if (op_array->fn_flags & ZEND_ACC_STATIC) { + | jne >1 + |.cold_code + |1: + } else { + | je >2 + } + | // zend_object *object = Z_OBJ(execute_data->This); + | mov r0, EX->This.value.obj + | // TODO: JIT ???? + | // if (UNEXPECTED(EG(exception) != NULL) && (call_info & ZEND_CALL_CTOR)) { + | // GC_REFCOUNT(object)--; + | // zend_object_store_ctor_failed(object); + | // ???? + | // OBJ_RELEASE(object); + | OBJ_RELEASE r0, 4 + | jmp >4 + if (op_array->fn_flags & ZEND_ACC_STATIC) { + |.code + } else { + |2: + } + } + | test FCARG1d, ZEND_CALL_CLOSURE + | jne >3 + |.cold_code + |3: + | // OBJ_RELEASE((zend_object*)execute_data->func->op_array.prototype); + | mov r0, EX->func + | mov r0, aword [r0 + offsetof(zend_op_array, prototype)] + | OBJ_RELEASE r0, 4 + | jmp >4 + |.code + |4: + | // EG(vm_stack_top) = (zval*)execute_data; + | mov aword [&EG(vm_stack_top)], FP + | // execute_data = EX(prev_execute_data); + | mov FP, EX->prev_execute_data + | // if (EG(exception)) + | cmp aword [&EG(exception)], 0 + | mov IP, EX->opline + | jne ->leave_throw_handler + | // EX(opline)++ + | add IP, sizeof(zend_op) + | mov EX->opline, IP + | add r4, SPAD // stack alignment + | jmp aword [IP] + + return 1; +} + static int zend_jit_return(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) { uint32_t op1_info; if (op_array->type == ZEND_EVAL_CODE || !op_array->function_name || !ssa->ops || !ssa->var_info) { + // TODO: support for top-level code return zend_jit_tail_handler(Dst, opline); } op1_info = OP1_INFO(); if (opline->op1_type == IS_CV && (op1_info & MAY_BE_UNDEF)) { + // TODO: support for IS_UNDEF ??? return zend_jit_tail_handler(Dst, opline); } // if (!EX(return_value)) | mov r1, EX->return_value | test r1, r1 - | jnz >2 - | FREE_OP opline->op1_type, opline->op1, op1_info, op_array, opline->lineno - | jmp >9 - |2: + if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && + (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { + | jnz >2 + | test byte [FP + opline->op1.var + 9], IS_TYPE_REFCOUNTED + | jnz >1 + | jmp >9 + |.cold_code + |1: + | mov FCARG1a, aword [FP + opline->op1.var] + | dec dword [FCARG1a] + | jnz >9 + | //SAVE_OPLINE() + | mov EX->opline, IP + | ZVAL_DTOR_FUNC (op_array->filename ? ZSTR_VAL(op_array->filename) : NULL), opline->lineno + | mov r1, EX->return_value // reload ??? + | jmp >9 + |.code + |2: + } else { + | jz >9 + } + if (opline->op1_type == IS_CONST) { zval *zv = RT_CONSTANT(op_array, opline->op1); | ZVAL_COPY_CONST r1, 0, -1, zv, r0 @@ -2192,71 +2452,84 @@ static int zend_jit_return(dasm_State **Dst, zend_op *opline, zend_op_array *op_ } else if (opline->op1_type == IS_TMP_VAR) { | ZVAL_COPY_VALUE r1, 0, FP, opline->op1.var, op1_info, r0, eax, r2 } else if (opline->op1_type == IS_CV) { - if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { - | lea r0, [FP + opline->op1.var] - | ZVAL_DEREF r0, op1_info - | ZVAL_COPY_VALUE r1, 0, r0, 0, op1_info, r0, eax, r2 - } else { - | ZVAL_COPY_VALUE r1, 0, FP, opline->op1.var, op1_info, r0, eax, r2 + if (op1_info & MAY_BE_REF) { + | cmp byte [FP + opline->op1.var + 8], IS_REFERENCE + | jz >1 + |.cold_code + |1: + | // retval_ptr = Z_REFVAL_P(retval_ptr); + | mov r0, [FP + opline->op1.var] + | // ZVAL_COPY(return_value, retval_ptr); + | mov edx, dword [r0 + 16] + | mov dword [r1 + 8], edx + | test dh, IS_TYPE_REFCOUNTED + | jnz >2 + |.if X64 + | mov r2, aword [r0 + 8] + | mov aword [r1], r2 + |.else + | mov r2, aword [r0 + 8] + | mov aword [r1], r2 + | mov r2, aword [r0 + 12] + | mov aword [r1 + 4], r2 + |.endif + | jmp >3 + |2: + |.if X64 + | mov r2, aword [r0 + 8] + | mov aword [r1], r2 + |.else + | mov r2, aword [r0 + 8] + | mov aword [r1], r2 + |.endif + | inc dword [r2] + | jmp >3 + |.code } + | ZVAL_COPY_VALUE r1, 0, FP, opline->op1.var, op1_info, r0, eax, r2 + | // TODO: JIT: if (EXPECTED(!(EX_CALL_INFO() & ZEND_CALL_CODE))) ZVAL_NULL(retval_ptr); ??? | TRY_ADDREF op1_info, ah, r2 + |3: } else { - if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { - if (op1_info & MAY_BE_REF) { - | lea FCARG1a, [FP + opline->op1.var] - | cmp byte [FCARG1a + 8], IS_REFERENCE - | jne >4 - | mov FCARG1a, [FCARG1a] - | add FCARG1a, 8 - | ZVAL_COPY_VALUE r1, 0, FCARG1a, 0, op1_info, r1, ebx, r2 - | sub FCARG1a, 8 - | dec dword [r0] - | jne >3 - if (ZEND_DEBUG) { - const char *filename = op_array->filename ? op_array->filename->val : NULL; - | LOAD_ADDR FCARG1a, filename + if (op1_info & MAY_BE_REF) { + | cmp byte [FP + opline->op1.var + 8], IS_REFERENCE + | je >1 + |.cold_code + |1: + | // zend_refcounted *ref = Z_COUNTED_P(retval_ptr); + | mov r0, aword [FP + opline->op1.var] + | // ZVAL_COPY_VALUE(return_value, &ref->value); |.if X64 - | mov CARG3d, opline->lineno + | mov r2, aword [r0 + 8] + | mov aword [r1], r2 |.else - | push opline->lineno - |.endif - } - | EXT_CALL _efree, r0 - if (ZEND_DEBUG) { - |.if not X64 - | add r4, sizeof(ptrdiff_t) + | mov r2, aword [r0 + 8] + | mov aword [r1], r2 + | mov r2, aword [r0 + 12] + | mov aword [r1 + 4], r2 |.endif - } + | mov edx, dword [r0 + 16] + | mov dword [r1 + 8], edx + | dec dword [r0] + | je >2 + | // if (IS_REFCOUNTED()) + | test dh, IS_TYPE_REFCOUNTED + | je >9 + | // ADDREF + | mov r2, aword [r1] + | inc dword [r2] | jmp >9 - |3: - | TRY_ADDREF op1_info, bh, r2 + |2: + | EFREE_SIZE r0, sizeof(zend_reference), op_array, opline | jmp >9 - |4: - | ZVAL_COPY_VALUE r1, 0, FP, opline->op1.var, op1_info, r0, eax, r2 - } else { - | ZVAL_COPY_VALUE r1, 0, FP, opline->op1.var, op1_info, r0, eax, r2 - } - } else { - | ZVAL_COPY_VALUE r1, 0, FP, opline->op1.var, op1_info, r0, eax, r2 + |.code } + | ZVAL_COPY_VALUE r1, 0, FP, opline->op1.var, op1_info, r0, eax, r2 } + |9: - | mov FCARG1a, FP - if (!zend_jit_set_valid_ip(Dst)) { - return 0; - } - | mov [FP], IP - | EXT_CALL zend_jit_leave_helper, r0 - zend_jit_check_exception(Dst); - | cmp al, 0 - | jg >1 - | LOAD_ADDR FP, &EG(current_execute_data) - | mov FP, [FP] - | mov IP, EX->opline - |1: - | add r4, SPAD - | ret - return 1; + //JIT: ZEND_VM_DISPATCH_TO_HELPER(zend_leave_helper); + return zend_jit_leave_func(Dst, opline, op_array, ssa); } #endif From a5776065e0641806ce3b237cf23eadb5802ca591 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 15 Sep 2016 00:06:45 +0300 Subject: [PATCH 187/569] Fixed support for exceptions in constructors --- ext/opcache/jit/zend_jit_x86.dasc | 34 +++++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 08b726c525ace..0d901464565ea 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -733,7 +733,7 @@ static int zend_jit_leave_throw_stub(dasm_State **Dst) | mov aword [&EG(opline_before_exception)], IP | // if (RETURN_VALUE_USED(old_opline)) { | cmp byte OP:IP->result_type, IS_UNUSED - | je >1 + | je >5 | // zval_ptr_dtor(EX_VAR(old_opline->result.var)); |.if X64 | movsxd r0, dword OP:IP->result.var @@ -2361,11 +2361,33 @@ static int zend_jit_leave_func(dasm_State **Dst, zend_op *opline, zend_op_array } | // zend_object *object = Z_OBJ(execute_data->This); | mov r0, EX->This.value.obj - | // TODO: JIT ???? - | // if (UNEXPECTED(EG(exception) != NULL) && (call_info & ZEND_CALL_CTOR)) { - | // GC_REFCOUNT(object)--; - | // zend_object_store_ctor_failed(object); - | // ???? + if (op_array->scope && op_array->scope->constructor == (zend_function*)op_array) { + | // if (UNEXPECTED(EG(exception) != NULL) + | cmp aword [&EG(exception)], 0 + | jne >6 + |.cold_code + |6: + | // if (call_info & ZEND_CALL_CTOR) + | test FCARG1d, ZEND_CALL_CTOR + | je >5 + | // GC_REFCOUNT(object)--; + | dec dword [r0] + | // zend_object_store_ctor_failed(object); + |.if X64 + | mov CARG1, r0 + | EXT_CALL zend_object_store_ctor_failed, r0 + |.else + | sub r4, 12 + | push r0 + | EXT_CALL zend_object_store_ctor_failed, r0 + | add r4, 16 + |.endif + | // reload registers + | mov r0, EX->This.value.obj + | jmp >5 + |.code + |5: + } | // OBJ_RELEASE(object); | OBJ_RELEASE r0, 4 | jmp >4 From c02c319aac1be2cb90a474feb2964b2173612274 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 15 Sep 2016 00:40:20 +0300 Subject: [PATCH 188/569] Generate single "leave" sequence for each function. --- ext/opcache/jit/zend_jit.c | 2 +- ext/opcache/jit/zend_jit_x86.dasc | 56 ++++++++++++++++++++++++++----- 2 files changed, 48 insertions(+), 10 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 1d10767d1121e..d6dd279ccb24e 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -702,7 +702,7 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) dasm_setupglobal(&dasm_state, dasm_labels, zend_lb_MAX); dasm_setup(&dasm_state, dasm_actions); - dasm_growpc(&dasm_state, ssa->cfg.blocks_count * 2); + dasm_growpc(&dasm_state, ssa->cfg.blocks_count * 2 + 1); zend_jit_align_func(&dasm_state); for (b = 0; b < ssa->cfg.blocks_count; b++) { diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 0d901464565ea..ca06b8e544ebd 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -595,8 +595,9 @@ static void* dasm_labels[zend_lb_MAX]; | EXT_CALL gc_possible_root, r0 |.endmacro -const zend_op *valid_opline; +static const zend_op *valid_opline; static uint32_t valid_opline_offset; +static int jit_return_label; #endif /* bit helpers */ @@ -763,6 +764,7 @@ static int zend_jit_align_func(dasm_State **Dst) { valid_opline = NULL; valid_opline_offset = 0; + jit_return_label = -1; |.align 16 return 1; } @@ -2336,7 +2338,15 @@ static int zend_jit_free_compiled_variables(dasm_State **Dst, zend_op *opline, z static int zend_jit_leave_func(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) { - // TODO: avoid multiple leave sequnces ??? + // Avoid multiple leave sequnces + if (jit_return_label >= 0) { + | jmp =>jit_return_label + return 1; + } + + jit_return_label = ssa->cfg.blocks_count * 2; + + |=>jit_return_label: // i_free_compiled_variables(execute_data); if (!zend_jit_free_compiled_variables(Dst, opline, op_array, ssa)) { @@ -2448,21 +2458,37 @@ static int zend_jit_return(dasm_State **Dst, zend_op *opline, zend_op_array *op_ | jnz >2 | test byte [FP + opline->op1.var + 9], IS_TYPE_REFCOUNTED | jnz >1 - | jmp >9 + || if (jit_return_label >= 0) { + | jmp =>jit_return_label + || } else { + | jmp >9 + || } |.cold_code |1: | mov FCARG1a, aword [FP + opline->op1.var] | dec dword [FCARG1a] - | jnz >9 + || if (jit_return_label >= 0) { + | jnz =>jit_return_label + || } else { + | jnz >9 + || } | //SAVE_OPLINE() | mov EX->opline, IP | ZVAL_DTOR_FUNC (op_array->filename ? ZSTR_VAL(op_array->filename) : NULL), opline->lineno | mov r1, EX->return_value // reload ??? - | jmp >9 + || if (jit_return_label >= 0) { + | jmp =>jit_return_label + || } else { + | jmp >9 + || } |.code |2: } else { - | jz >9 + || if (jit_return_label >= 0) { + | jz =>jit_return_label + || } else { + | jz >9 + || } } if (opline->op1_type == IS_CONST) { @@ -2536,14 +2562,26 @@ static int zend_jit_return(dasm_State **Dst, zend_op *opline, zend_op_array *op_ | je >2 | // if (IS_REFCOUNTED()) | test dh, IS_TYPE_REFCOUNTED - | je >9 + || if (jit_return_label >= 0) { + | je =>jit_return_label + || } else { + | je >9 + || } | // ADDREF | mov r2, aword [r1] | inc dword [r2] - | jmp >9 + || if (jit_return_label >= 0) { + | jmp =>jit_return_label + || } else { + | jmp >9 + || } |2: | EFREE_SIZE r0, sizeof(zend_reference), op_array, opline - | jmp >9 + || if (jit_return_label >= 0) { + | jmp =>jit_return_label + || } else { + | jmp >9 + || } |.code } | ZVAL_COPY_VALUE r1, 0, FP, opline->op1.var, op1_info, r0, eax, r2 From e70125b4a6cda7cda7d9508a90054f00cad8cf38 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 15 Sep 2016 02:03:26 +0300 Subject: [PATCH 189/569] Fixed is_bool() and is_resource() --- ext/opcache/jit/zend_jit_x86.dasc | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index ca06b8e544ebd..50574385967e5 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -2258,7 +2258,7 @@ fallback: static int zend_jit_type_check(dasm_State **Dst, zend_op *opline, int b, int *opnum, zend_op_array *op_array, zend_ssa *ssa) { - uint32_t op1_info; + uint32_t op1_info, mask; unsigned int target_label; zend_uchar type; @@ -2268,7 +2268,14 @@ static int zend_jit_type_check(dasm_State **Dst, zend_op *opline, int b, int *op op1_info = OP1_INFO(); type = opline->extended_value; - if ((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) == (1<opcode == ZEND_JMPZ && (opline+1)->op1_type == IS_TMP_VAR && (opline+1)->op1.var == opline->result.var) { @@ -2284,7 +2291,7 @@ static int zend_jit_type_check(dasm_State **Dst, zend_op *opline, int b, int *op | SET_TYPE_INFO FP, opline->result.var, IS_TRUE valid_opline_offset++; } - } else if (!(op1_info & (1<opcode == ZEND_JMPZ && (opline+1)->op1_type == IS_TMP_VAR && (opline+1)->op1.var == opline->result.var) { From 5083fbd92adac22888b21634d3d3d5796e2a57e7 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Thu, 15 Sep 2016 14:58:50 +0800 Subject: [PATCH 190/569] Use tail call --- ext/opcache/jit/zend_jit_helpers.c | 33 +++++++++++++----------------- ext/opcache/jit/zend_jit_x86.dasc | 20 +++--------------- 2 files changed, 17 insertions(+), 36 deletions(-) diff --git a/ext/opcache/jit/zend_jit_helpers.c b/ext/opcache/jit/zend_jit_helpers.c index 945c769585538..a3dea7b1f0fb7 100644 --- a/ext/opcache/jit/zend_jit_helpers.c +++ b/ext/opcache/jit/zend_jit_helpers.c @@ -43,25 +43,17 @@ static zend_execute_data* ZEND_FASTCALL zend_jit_extend_stack_helper(uint32_t us return call; } -static zend_always_inline void i_free_compiled_variables(zend_execute_data *execute_data) -{ - zval *cv = EX_VAR_NUM(0); - zval *end = cv + EX(func)->op_array.last_var; - while (EXPECTED(cv != end)) { - if (Z_REFCOUNTED_P(cv)) { - if (!Z_DELREF_P(cv)) { - zend_refcounted *r = Z_COUNTED_P(cv); - ZVAL_NULL(cv); - zval_dtor_func(r); - } else { - GC_ZVAL_CHECK_POSSIBLE_ROOT(cv); - } - } - cv++; - } -} +#pragma GCC diagnostic ignored "-Wvolatile-register-var" +#if defined(__x86_64__) +register zend_execute_data* volatile execute_data __asm__("%r14"); +register const zend_op* volatile opline __asm__("%r15"); +#else +register zend_execute_data* volatile execute_data __asm__("%esi"); +register const zend_op* volatile opline __asm__("%edi"); +#endif +#pragma GCC diagnostic warning "-Wvolatile-register-var" -static void ZEND_FASTCALL zend_jit_leave_nested_func_helper(uint32_t call_info, zend_execute_data *execute_data) +static void ZEND_FASTCALL zend_jit_leave_nested_func_helper(uint32_t call_info) { zend_execute_data *old_execute_data; @@ -93,10 +85,11 @@ static void ZEND_FASTCALL zend_jit_leave_nested_func_helper(uint32_t call_info, } } else { EX(opline)++; + opline = EX(opline); } } -static void ZEND_FASTCALL zend_jit_leave_top_func_helper(uint32_t call_info, zend_execute_data *execute_data) +static void ZEND_FASTCALL zend_jit_leave_top_func_helper(uint32_t call_info) { if (UNEXPECTED(call_info & (ZEND_CALL_HAS_SYMBOL_TABLE|ZEND_CALL_FREE_EXTRA_ARGS))) { if (UNEXPECTED(call_info & ZEND_CALL_HAS_SYMBOL_TABLE)) { @@ -108,6 +101,8 @@ static void ZEND_FASTCALL zend_jit_leave_top_func_helper(uint32_t call_info, zen if (UNEXPECTED(call_info & ZEND_CALL_CLOSURE)) { OBJ_RELEASE((zend_object*)EX(func)->op_array.prototype); } + execute_data = EG(current_execute_data); + opline = NULL; } /* diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 50574385967e5..257ac08b33dd0 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -700,26 +700,12 @@ static int zend_jit_exception_handler_stub(dasm_State **Dst) static int zend_jit_leave_function_stub(dasm_State **Dst) { |->leave_function_handler: + | add r4, SPAD | test FCARG1d, ZEND_CALL_TOP - | mov FCARG2a, FP | jne >1 - | // TODO: Use tail call ??? - | EXT_CALL zend_jit_leave_nested_func_helper, r0 - | //execute_data = EG(current_execute_data); - | mov FP, aword [&EG(current_execute_data)] - | // LOAD_OPLINE(); - | mov IP, EX->opline - | //ZEND_VM_CONTINUE() - | add r4, SPAD // stack alignment - | jmp aword [IP] + | EXT_JMP zend_jit_leave_nested_func_helper, r0 |1: - | // TODO: Use tail call ??? - | EXT_CALL zend_jit_leave_top_func_helper, r0 - | //execute_data = EG(current_execute_data); - | mov FP, aword [&EG(current_execute_data)] - | xor IP, IP - | add r4, SPAD // stack alignment - | ret + | EXT_JMP zend_jit_leave_top_func_helper, r0 return 1; } From 0e0cad25b9dd83a25d8c068d91a10dc06469f57e Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 15 Sep 2016 10:16:35 +0300 Subject: [PATCH 191/569] Initialize EX(run_time_cache) only if necessary. --- ext/opcache/jit/zend_jit_x86.dasc | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 257ac08b33dd0..dd7274409305d 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -2110,14 +2110,19 @@ static int zend_jit_do_fcall(dasm_State **Dst, zend_op *opline, zend_op_array *o || } | | //EX_LOAD_RUN_TIME_CACHE(op_array); - || if (func && op_array == &func->op_array) { - || /* recursive call */ - | mov r0, EX->run_time_cache - || } else { - | mov r2, EX:r1->func - | mov r0, aword [r2 + offsetof(zend_op_array, run_time_cache)] + || if (!func || func->op_array.cache_size) { + || if (func && op_array == &func->op_array) { + || /* recursive call */ + || if (func->op_array.cache_size > sizeof(void*)) { + | mov r0, EX->run_time_cache + | mov EX:r1->run_time_cache, r0 + || } + || } else { + | mov r2, EX:r1->func + | mov r0, aword [r2 + offsetof(zend_op_array, run_time_cache)] + | mov EX:r1->run_time_cache, r0 + || } || } - | mov EX:r1->run_time_cache, r0 | //EX_LOAD_LITERALS(op_array); |.if X64 | LOAD_ADDR r0, func->op_array.literals From ccb25870ce7f0cd95b6acb6851c6e3096919564b Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Thu, 15 Sep 2016 15:25:40 +0800 Subject: [PATCH 192/569] Added zend_jit_vm_helper(minimize global registers side-affect) --- ext/opcache/config.m4 | 2 +- ext/opcache/jit/zend_jit.c | 1 + ext/opcache/jit/zend_jit_helpers.c | 62 ------------------- ext/opcache/jit/zend_jit_vm_helper.c | 91 ++++++++++++++++++++++++++++ ext/opcache/jit/zend_jit_vm_helper.h | 34 +++++++++++ 5 files changed, 127 insertions(+), 63 deletions(-) create mode 100644 ext/opcache/jit/zend_jit_vm_helper.c create mode 100644 ext/opcache/jit/zend_jit_vm_helper.h diff --git a/ext/opcache/config.m4 b/ext/opcache/config.m4 index 6dad85dc97afb..35f01ad99aa4f 100644 --- a/ext/opcache/config.m4 +++ b/ext/opcache/config.m4 @@ -30,7 +30,7 @@ if test "$PHP_OPCACHE" != "no"; then if test "$PHP_OPCACHE_JIT" = "yes"; then AC_DEFINE(HAVE_JIT, 1, [Define to enable JIT]) - ZEND_JIT_SRC=jit/zend_jit.c + ZEND_JIT_SRC="jit/zend_jit.c jit/zend_jit_vm_helper.c" dnl Find out which ABI we are using. echo 'int i;' > conftest.$ac_ext diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index d6dd279ccb24e..9299dbc1bc0b5 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -22,6 +22,7 @@ #include "Zend/zend_exceptions.h" #include "zend_smart_str.h" #include "jit/zend_jit.h" +#include "jit/zend_jit_vm_helper.h" #ifdef HAVE_JIT diff --git a/ext/opcache/jit/zend_jit_helpers.c b/ext/opcache/jit/zend_jit_helpers.c index a3dea7b1f0fb7..9445385c99d87 100644 --- a/ext/opcache/jit/zend_jit_helpers.c +++ b/ext/opcache/jit/zend_jit_helpers.c @@ -43,68 +43,6 @@ static zend_execute_data* ZEND_FASTCALL zend_jit_extend_stack_helper(uint32_t us return call; } -#pragma GCC diagnostic ignored "-Wvolatile-register-var" -#if defined(__x86_64__) -register zend_execute_data* volatile execute_data __asm__("%r14"); -register const zend_op* volatile opline __asm__("%r15"); -#else -register zend_execute_data* volatile execute_data __asm__("%esi"); -register const zend_op* volatile opline __asm__("%edi"); -#endif -#pragma GCC diagnostic warning "-Wvolatile-register-var" - -static void ZEND_FASTCALL zend_jit_leave_nested_func_helper(uint32_t call_info) -{ - zend_execute_data *old_execute_data; - - if (UNEXPECTED(call_info & ZEND_CALL_HAS_SYMBOL_TABLE)) { - zend_clean_and_cache_symbol_table(EX(symbol_table)); - } - EG(current_execute_data) = EX(prev_execute_data); - if (UNEXPECTED(call_info & ZEND_CALL_RELEASE_THIS)) { - zend_object *object = Z_OBJ(execute_data->This); - if (UNEXPECTED(EG(exception) != NULL) && (call_info & ZEND_CALL_CTOR)) { - GC_REFCOUNT(object)--; - zend_object_store_ctor_failed(object); - } - OBJ_RELEASE(object); - } else if (UNEXPECTED(call_info & ZEND_CALL_CLOSURE)) { - OBJ_RELEASE((zend_object*)execute_data->func->op_array.prototype); - } - - zend_vm_stack_free_extra_args_ex(call_info, execute_data); - old_execute_data = execute_data; - execute_data = EX(prev_execute_data); - zend_vm_stack_free_call_frame_ex(call_info, old_execute_data); - - if (UNEXPECTED(EG(exception) != NULL)) { - const zend_op *old_opline = EX(opline); - zend_throw_exception_internal(NULL); - if (RETURN_VALUE_USED(old_opline)) { - zval_ptr_dtor(EX_VAR(old_opline->result.var)); - } - } else { - EX(opline)++; - opline = EX(opline); - } -} - -static void ZEND_FASTCALL zend_jit_leave_top_func_helper(uint32_t call_info) -{ - if (UNEXPECTED(call_info & (ZEND_CALL_HAS_SYMBOL_TABLE|ZEND_CALL_FREE_EXTRA_ARGS))) { - if (UNEXPECTED(call_info & ZEND_CALL_HAS_SYMBOL_TABLE)) { - zend_clean_and_cache_symbol_table(EX(symbol_table)); - } - zend_vm_stack_free_extra_args_ex(call_info, execute_data); - } - EG(current_execute_data) = EX(prev_execute_data); - if (UNEXPECTED(call_info & ZEND_CALL_CLOSURE)) { - OBJ_RELEASE((zend_object*)EX(func)->op_array.prototype); - } - execute_data = EG(current_execute_data); - opline = NULL; -} - /* * Local variables: * tab-width: 4 diff --git a/ext/opcache/jit/zend_jit_vm_helper.c b/ext/opcache/jit/zend_jit_vm_helper.c new file mode 100644 index 0000000000000..c3c43025a0cb4 --- /dev/null +++ b/ext/opcache/jit/zend_jit_vm_helper.c @@ -0,0 +1,91 @@ +/* + +----------------------------------------------------------------------+ + | Zend JIT | + +----------------------------------------------------------------------+ + | Copyright (c) 1998-2016 The PHP Group | + +----------------------------------------------------------------------+ + | This source file is subject to version 3.01 of the PHP license, | + | that is bundled with this package in the file LICENSE, and is | + | available through the world-wide-web at the following url: | + | http://www.php.net/license/3_01.txt | + | If you did not receive a copy of the PHP license and are unable to | + | obtain it through the world-wide-web, please send a note to | + | license@php.net so we can mail you a copy immediately. | + +----------------------------------------------------------------------+ + | Authors: Dmitry Stogov | + | Xinchen Hui | + +----------------------------------------------------------------------+ +*/ + +#include "Zend/zend_execute.h" +#include "zend_jit_vm_helper.h" + +#pragma GCC diagnostic ignored "-Wvolatile-register-var" +#if defined(__x86_64__) +register zend_execute_data* volatile execute_data __asm__("%r14"); +register const zend_op* volatile opline __asm__("%r15"); +#else +register zend_execute_data* volatile execute_data __asm__("%esi"); +register const zend_op* volatile opline __asm__("%edi"); +#endif +#pragma GCC diagnostic warning "-Wvolatile-register-var" + +void ZEND_FASTCALL zend_jit_leave_nested_func_helper(uint32_t call_info) +{ + zend_execute_data *old_execute_data; + + if (UNEXPECTED(call_info & ZEND_CALL_HAS_SYMBOL_TABLE)) { + zend_clean_and_cache_symbol_table(EX(symbol_table)); + } + EG(current_execute_data) = EX(prev_execute_data); + if (UNEXPECTED(call_info & ZEND_CALL_RELEASE_THIS)) { + zend_object *object = Z_OBJ(execute_data->This); + if (UNEXPECTED(EG(exception) != NULL) && (call_info & ZEND_CALL_CTOR)) { + GC_REFCOUNT(object)--; + zend_object_store_ctor_failed(object); + } + OBJ_RELEASE(object); + } else if (UNEXPECTED(call_info & ZEND_CALL_CLOSURE)) { + OBJ_RELEASE((zend_object*)execute_data->func->op_array.prototype); + } + + zend_vm_stack_free_extra_args_ex(call_info, execute_data); + old_execute_data = execute_data; + execute_data = EX(prev_execute_data); + zend_vm_stack_free_call_frame_ex(call_info, old_execute_data); + + if (UNEXPECTED(EG(exception) != NULL)) { + const zend_op *old_opline = EX(opline); + zend_throw_exception_internal(NULL); + if (RETURN_VALUE_USED(old_opline)) { + zval_ptr_dtor(EX_VAR(old_opline->result.var)); + } + } else { + EX(opline)++; + opline = EX(opline); + } +} + +void ZEND_FASTCALL zend_jit_leave_top_func_helper(uint32_t call_info) +{ + if (UNEXPECTED(call_info & (ZEND_CALL_HAS_SYMBOL_TABLE|ZEND_CALL_FREE_EXTRA_ARGS))) { + if (UNEXPECTED(call_info & ZEND_CALL_HAS_SYMBOL_TABLE)) { + zend_clean_and_cache_symbol_table(EX(symbol_table)); + } + zend_vm_stack_free_extra_args_ex(call_info, execute_data); + } + EG(current_execute_data) = EX(prev_execute_data); + if (UNEXPECTED(call_info & ZEND_CALL_CLOSURE)) { + OBJ_RELEASE((zend_object*)EX(func)->op_array.prototype); + } + execute_data = EG(current_execute_data); + opline = NULL; +} + +/* + * Local variables: + * tab-width: 4 + * c-basic-offset: 4 + * indent-tabs-mode: t + * End: + */ diff --git a/ext/opcache/jit/zend_jit_vm_helper.h b/ext/opcache/jit/zend_jit_vm_helper.h new file mode 100644 index 0000000000000..5eedf423b0650 --- /dev/null +++ b/ext/opcache/jit/zend_jit_vm_helper.h @@ -0,0 +1,34 @@ +/* + +----------------------------------------------------------------------+ + | Zend JIT | + +----------------------------------------------------------------------+ + | Copyright (c) 1998-2016 The PHP Group | + +----------------------------------------------------------------------+ + | This source file is subject to version 3.01 of the PHP license, | + | that is bundled with this package in the file LICENSE, and is | + | available through the world-wide-web at the following url: | + | http://www.php.net/license/3_01.txt | + | If you did not receive a copy of the PHP license and are unable to | + | obtain it through the world-wide-web, please send a note to | + | license@php.net so we can mail you a copy immediately. | + +----------------------------------------------------------------------+ + | Authors: Dmitry Stogov | + | Xinchen Hui | + +----------------------------------------------------------------------+ +*/ + +#ifndef ZEND_JIT_VM_HELPER +#define ZEND_JIT_VM_HELPER + +void ZEND_FASTCALL zend_jit_leave_nested_func_helper(uint32_t call_info); +void ZEND_FASTCALL zend_jit_leave_top_func_helper(uint32_t call_info); + +#endif + +/* + * Local variables: + * tab-width: 4 + * c-basic-offset: 4 + * indent-tabs-mode: t + * End: + */ From 4fe1e54b1f7d5c70f279477c2e7192eef61cc126 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 15 Sep 2016 10:35:44 +0300 Subject: [PATCH 193/569] Fixed compilation issues --- ext/opcache/jit/zend_jit_vm_helper.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_vm_helper.c b/ext/opcache/jit/zend_jit_vm_helper.c index c3c43025a0cb4..bd1411c8e2eb9 100644 --- a/ext/opcache/jit/zend_jit_vm_helper.c +++ b/ext/opcache/jit/zend_jit_vm_helper.c @@ -18,6 +18,7 @@ */ #include "Zend/zend_execute.h" +#include "Zend/zend_exceptions.h" #include "zend_jit_vm_helper.h" #pragma GCC diagnostic ignored "-Wvolatile-register-var" @@ -57,7 +58,7 @@ void ZEND_FASTCALL zend_jit_leave_nested_func_helper(uint32_t call_info) if (UNEXPECTED(EG(exception) != NULL)) { const zend_op *old_opline = EX(opline); zend_throw_exception_internal(NULL); - if (RETURN_VALUE_USED(old_opline)) { + if (old_opline->result_type != IS_UNDEF) { zval_ptr_dtor(EX_VAR(old_opline->result.var)); } } else { From 3c81dd470c3e28aab8f160ed74ca2caf7bd6136e Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 15 Sep 2016 11:43:17 +0300 Subject: [PATCH 194/569] Move "slow" code into .cold_code --- ext/opcache/jit/zend_jit_x86.dasc | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index dd7274409305d..0df19df94563d 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -2453,16 +2453,15 @@ static int zend_jit_return(dasm_State **Dst, zend_op *opline, zend_op_array *op_ | test r1, r1 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { - | jnz >2 + | jz >1 + |.cold_code + |1: | test byte [FP + opline->op1.var + 9], IS_TYPE_REFCOUNTED - | jnz >1 || if (jit_return_label >= 0) { - | jmp =>jit_return_label + | jz =>jit_return_label || } else { - | jmp >9 + | jz >9 || } - |.cold_code - |1: | mov FCARG1a, aword [FP + opline->op1.var] | dec dword [FCARG1a] || if (jit_return_label >= 0) { @@ -2480,7 +2479,6 @@ static int zend_jit_return(dasm_State **Dst, zend_op *opline, zend_op_array *op_ | jmp >9 || } |.code - |2: } else { || if (jit_return_label >= 0) { | jz =>jit_return_label From 7cbf1943a8bf73b933cb60fbcf6501a73fbe331f Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 15 Sep 2016 12:20:40 +0300 Subject: [PATCH 195/569] Move CV dtor(s) code into .cold_code --- ext/opcache/jit/zend_jit_x86.dasc | 97 ++++++++++++++++++------------- 1 file changed, 57 insertions(+), 40 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 0df19df94563d..6e95d502fe9b7 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -503,47 +503,64 @@ static void* dasm_labels[zend_lb_MAX]; |1: |.endmacro -|.macro ZVAL_PTR_DTOR, base, offset, info, filename, lineno -|| if (info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { -| // if (Z_REFCOUNTED_P(cv)) { -| test byte [base + offset + 9], IS_TYPE_REFCOUNTED -| je >3 -|| } -| // if (!Z_DELREF_P(cv)) { -| mov FCARG1a, aword [base + offset] -| dec dword [FCARG1a] -| jnz >1 -| // ZVAL_NULL(cv); -| mov dword [base + offset + 8], IS_NULL -| // zval_dtor_func(r); -| ZVAL_DTOR_FUNC filename, lineno -| jmp >3 +|.macro ZVAL_PTR_DTOR, base, offset, info, cold, filename, lineno +|| if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { +|| if (info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { +| // if (Z_REFCOUNTED_P(cv)) { +| test byte [base + offset + 9], IS_TYPE_REFCOUNTED +|| if (cold) { +| jne >1 +|.cold_code |1: -|| if (info & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) { -|| if (info & MAY_BE_REF) { -| lea r1, [base + offset] -| // ZVAL_DEREF(z); -| ZVAL_DEREF r1, info -| // if (Z_COLLECTABLE_P(z)) -| test byte [r1 + 9], IS_TYPE_COLLECTABLE -| je >3 -| // if (UNEXPECTED(!Z_GC_INFO_P(z))) -| mov FCARG1a, aword [r1] -| cmp word [FCARG1a + 6], 0 -| jne >3 -|| } else { -| // if (Z_COLLECTABLE_P(z)) -| test byte [base + offset + 9], IS_TYPE_COLLECTABLE -| je >3 -| // if (UNEXPECTED(!Z_GC_INFO_P(z))) -| mov FCARG1a, aword [base + offset] -| cmp word [FCARG1a + 6], 0 -| jne >3 +|| } else { +| je >3 +|| } +|| } +| // if (!Z_DELREF_P(cv)) { +| mov FCARG1a, aword [base + offset] +| dec dword [FCARG1a] +| jnz >1 +| // ZVAL_NULL(cv); +| mov dword [base + offset + 8], IS_NULL +| // zval_dtor_func(r); +| ZVAL_DTOR_FUNC filename, lineno +|| if (info & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) { +| jmp >3 +|| } +|1: +|| if (info & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) { +|| if (info & MAY_BE_REF) { +| lea r1, [base + offset] +| // ZVAL_DEREF(z); +| ZVAL_DEREF r1, info +| // if (Z_COLLECTABLE_P(z)) +| test byte [r1 + 9], IS_TYPE_COLLECTABLE +| je >3 +| // if (UNEXPECTED(!Z_GC_INFO_P(z))) +| mov FCARG1a, aword [r1] +| cmp word [FCARG1a + 6], 0 +| jne >3 +|| } else { +| // if (Z_COLLECTABLE_P(z)) +| test byte [base + offset + 9], IS_TYPE_COLLECTABLE +| je >3 +| // if (UNEXPECTED(!Z_GC_INFO_P(z))) +| mov FCARG1a, aword [base + offset] +| cmp word [FCARG1a + 6], 0 +| jne >3 +|| } +| // gc_possible_root(Z_COUNTED_P(z)) +| EXT_CALL gc_possible_root, r0 +|| if (cold && (info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE)))) { +| jmp >3 +|.code +|| } +|| } else if (cold && (info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE)))) { +| jmp >3 +|.code || } -| // gc_possible_root(Z_COUNTED_P(z)) -| EXT_CALL gc_possible_root, r0 -|| } |3: +|| } |.endmacro |.macro FREE_OP, op_type, op, op_info, op_array, lineno @@ -728,7 +745,7 @@ static int zend_jit_leave_throw_stub(dasm_State **Dst) | mov r0, OP:IP->result.var |.endif | add r0, FP - | ZVAL_PTR_DTOR r0, 0, MAY_BE_ANY, NULL, 0 + | ZVAL_PTR_DTOR r0, 0, MAY_BE_ANY, 0, NULL, 0 |5: | // EX(opline) = EG(exception_op); | LOAD_ADDR IP, &EG(exception_op) @@ -2328,7 +2345,7 @@ static int zend_jit_free_compiled_variables(dasm_State **Dst, zend_op *opline, z if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { uint32_t offset = (uint32_t)(uintptr_t)ZEND_CALL_VAR_NUM(NULL, i); - | ZVAL_PTR_DTOR FP, offset, info, (op_array->filename ? ZSTR_VAL(op_array->filename) : NULL), opline->lineno + | ZVAL_PTR_DTOR FP, offset, info, 1, (op_array->filename ? ZSTR_VAL(op_array->filename) : NULL), opline->lineno } } return 1; From a3eb65d97a95948ffd40ccedd47bb65aaa9abd64 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 15 Sep 2016 18:30:30 +0300 Subject: [PATCH 196/569] JIT for FETCH_DIM_R and FETCH_DIM_IS (incomplete) --- ext/opcache/jit/zend_jit_disasm_x86.c | 1 + ext/opcache/jit/zend_jit_helpers.c | 25 +++ ext/opcache/jit/zend_jit_x86.dasc | 211 ++++++++++++++++++++++++++ 3 files changed, 237 insertions(+) diff --git a/ext/opcache/jit/zend_jit_disasm_x86.c b/ext/opcache/jit/zend_jit_disasm_x86.c index 3755b16359705..4118a487a8c69 100644 --- a/ext/opcache/jit/zend_jit_disasm_x86.c +++ b/ext/opcache/jit/zend_jit_disasm_x86.c @@ -386,6 +386,7 @@ static int zend_jit_disasm_init(void) REGISTER_HELPER(zend_jit_extend_stack_helper); REGISTER_HELPER(zend_jit_leave_nested_func_helper); REGISTER_HELPER(zend_jit_leave_top_func_helper); + REGISTER_HELPER(zend_jit_symtable_find); #undef REGISTER_HELPER zend_elf_load_symbols(); diff --git a/ext/opcache/jit/zend_jit_helpers.c b/ext/opcache/jit/zend_jit_helpers.c index 9445385c99d87..e272f60efb335 100644 --- a/ext/opcache/jit/zend_jit_helpers.c +++ b/ext/opcache/jit/zend_jit_helpers.c @@ -43,6 +43,31 @@ static zend_execute_data* ZEND_FASTCALL zend_jit_extend_stack_helper(uint32_t us return call; } +static zval* ZEND_FASTCALL zend_jit_symtable_find(HashTable *ht, zend_string *str) +{ + zend_ulong idx; + register const char *tmp = str->val; + + do { + if (*tmp > '9') { + break; + } else if (*tmp < '0') { + if (*tmp != '-') { + break; + } + tmp++; + if (*tmp > '9' || *tmp < '0') { + break; + } + } + if (_zend_handle_numeric_str_ex(str->val, str->len, &idx)) { + return zend_hash_index_find(ht, idx); + } + } while (0); + + return zend_hash_find(ht, str); +} + /* * Local variables: * tab-width: 4 diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 6e95d502fe9b7..38484442a356c 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -2605,6 +2605,217 @@ static int zend_jit_return(dasm_State **Dst, zend_op *opline, zend_op_array *op_ return zend_jit_leave_func(Dst, opline, op_array, ssa); } +static int zend_jit_fetch_dim_read(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) +{ + uint32_t op1_info, op2_info, res_info; + + if (!ssa->ops || !ssa->var_info) { + goto fallback; + } + + op1_info = OP1_INFO(); + op2_info = OP2_INFO(); + res_info = RES_INFO(); + if ((op1_info & (/*MAY_BE_UNDEF|*/MAY_BE_REF)) || + (op2_info & (/*MAY_BE_UNDEF|*/MAY_BE_REF)) || + !(op1_info & MAY_BE_ARRAY) || + !(op2_info & (MAY_BE_LONG|MAY_BE_STRING))) { + // TODO: support for references ??? + goto fallback; + } + + if (op1_info & MAY_BE_ARRAY) { + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { + | // if (EXPECTED(Z_TYPE_P(container) == IS_ARRAY)) { + | cmp byte [FP + opline->op1.var + 8], IS_ARRAY + | jnz >7 // NOT_ARRAY + } + | LONG_LOAD opline->op1_type, opline->op1, FCARG1a + if (op2_info & MAY_BE_LONG) { + if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_LONG)) { + | // if (EXPECTED(Z_TYPE_P(dim) == IS_LONG)) { + | cmp byte [FP + opline->op2.var + 8], IS_LONG + | jnz >3 // NOT_LONG + } + | // hval = Z_LVAL_P(dim); + | LONG_LOAD opline->op2_type, opline->op2, FCARG2a + + if (opline->op2_type == IS_CONST) { + zend_long val = Z_LVAL_P(RT_CONSTANT(op_array, opline->op2)); + if (val >= 0 && val < HT_MAX_SIZE) { + | // ZEND_HASH_INDEX_FIND(ht, hval, retval, num_undef); + | test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED + | jz >1 // HASH_FIND + | // if (EXPECTED((zend_ulong)(_h) < (zend_ulong)(_ht)->nNumUsed)) { + |.if X64 + | movsxd r0, dword [FCARG1a + offsetof(zend_array, nNumUsed)] + | cmp r0, val + |.else + | cmp dword [FCARG1a + offsetof(zend_array, nNumUsed)], val + |.endif + | jle >2 // NOT_FOUND + | // _ret = &_ht->arData[_h].val; + | mov r0, aword [FCARG1a + offsetof(zend_array, arData)] + | add r0, val * sizeof(Bucket) + | cmp dword [r0 + 8], IS_UNDEF + | jne >8 // FOUND + | jmp >2 // NOT_FOUND + } + } else { + | // ZEND_HASH_INDEX_FIND(ht, hval, retval, num_undef); + | test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED + | jz >1 // HASH_FIND + | // if (EXPECTED((zend_ulong)(_h) < (zend_ulong)(_ht)->nNumUsed)) { + |.if X64 + | movsxd r0, dword [FCARG1a + offsetof(zend_array, nNumUsed)] + | cmp r0, FCARG2a + |.else + | cmp dword [FCARG1a + offsetof(zend_array, nNumUsed)], FCARG2a + |.endif + | jle >2 // NOT_FOUND + | // _ret = &_ht->arData[_h].val; + | imul FCARG2a, sizeof(Bucket) + | mov r0, aword [FCARG1a + offsetof(zend_array, arData)] + | add r0, FCARG2a + | cmp dword [r0 + 8], IS_UNDEF + | jne >8 // FOUND + | jmp >2 // NOT_FOUND + } + |1: + | EXT_CALL zend_hash_index_find, r0 + | test r0, r0 + | je >2 // NOT_FOUND + |.cold_code + |2: + if (opline->opcode == ZEND_FETCH_DIM_R) { + | // zend_error(E_NOTICE,"Undefined offset: " ZEND_LONG_FMT, hval); + |.if X64 + | mov CARG1, E_NOTICE + | LOAD_ADDR CARG2, "Undefined offset: " ZEND_LONG_FMT + | LONG_LOAD opline->op2_type, opline->op2, CARG3 + | EXT_CALL zend_error, r0 + |.else + | sub r4, 4 + | LONG_LOAD opline->op2_type, opline->op2, r0 + | push r0 + | push "Undefined offset: " ZEND_LONG_FMT + | push E_NOTICE + | EXT_CALL zend_error, r0 + | add r4, 16 + |.endif + } + | // retval = &EG(uninitialized_zval); + | mov dword [FP + opline->result.var + 8], IS_NULL + | jmp >9 // END + |.code + if (op2_info & (MAY_BE_ANY - MAY_BE_STRING)) { + | jmp >8 // FOUND + } + if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_LONG)) { + |3: + } + } + if (op2_info & MAY_BE_STRING) { + if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) { + | // if (EXPECTED(Z_TYPE_P(dim) == IS_STRING)) { + | cmp byte [FP + opline->op2.var + 8], IS_STRING + | jnz >3 // NOT_STRING + } + | // offset_key = Z_STR_P(dim); + | LONG_LOAD opline->op2_type, opline->op2, FCARG2a + | // retval = zend_hash_find(ht, offset_key); + if (opline->op2_type != IS_CONST) { + | EXT_CALL zend_jit_symtable_find, r0 + } else { + | EXT_CALL zend_hash_find, r0 + } + | test r0, r0 + | je >2 // NOT_FOUND + | // if (UNEXPECTED(Z_TYPE_P(retval) == IS_INDIRECT)) { + | cmp dword [r0 + 8], IS_INDIRECT + | jz >1 // SLOW + |.cold_code + |1: + | // retval = Z_INDIRECT_P(retval); + | mov r0, aword [r0] + | cmp dword [r0 + 8], IS_UNDEF + | jnz >8 // COPY + |2: + if (opline->opcode == ZEND_FETCH_DIM_R) { + // zend_error(E_NOTICE, "Undefined index: %s", ZSTR_VAL(offset_key)); + |.if X64 + | mov CARG1, E_NOTICE + | LOAD_ADDR CARG2, "Undefined index: %s" + | LONG_LOAD opline->op2_type, opline->op2, CARG3 + | add CARG3, offsetof(zend_string, val) + | EXT_CALL zend_error, r0 + |.else + | sub r4, 4 + | LONG_LOAD opline->op2_type, opline->op2, r0 + | add r0, offsetof(zend_string, val) + | push r0 + | push "Undefined index: %s" + | push E_NOTICE + | EXT_CALL zend_error, r0 + | add r4, 16 + |.endif + } + | mov dword [FP +opline->result.var + 8], IS_NULL + | jmp >9 // END + |.code + if (op2_info & (MAY_BE_ANY - (MAY_BE_LONG|MAY_BE_STRING))) { + | jmp >8 // FOUND + } + } + if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) { + if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) { + |.cold_code + |3: + } + | // ???? + | int3 + if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) { + |.code + } + } + } + + if (op1_info & (MAY_BE_ANY-MAY_BE_ARRAY)) { + if (op1_info & MAY_BE_ARRAY) { + |.cold_code + |7: + } + | // ???? + | int3 + if (op1_info & MAY_BE_ARRAY) { + |.code + } + } + + |8: // COPY + | // UNREF ???? + | ZVAL_COPY_VALUE FP, opline->result.var, r0, 0, MAY_BE_ANY, r1, ecx, r2 + | TRY_ADDREF res_info, ch, r2 + |9: // END + + | FREE_OP opline->op2_type, opline->op2, op2_info, op_array, opline->lineno + | FREE_OP opline->op1_type, opline->op1, op1_info, op_array, opline->lineno + + if (zend_may_throw(opline, op_array, ssa)) { + if (!zend_jit_check_exception(Dst)) { + return 0; + } + } + + valid_opline_offset++; + + return 1; + +fallback: + /* fallback to subroutine threading */ + return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); +} + #endif /* From 839e8eee43915b6ccf8e20da7058131dce39a683 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 16 Sep 2016 00:58:07 +0300 Subject: [PATCH 197/569] JIT for FETCH_DIM_R (incomplete) --- ext/opcache/jit/zend_jit_x86.dasc | 208 ++++++++++++++++++++++++++++-- 1 file changed, 197 insertions(+), 11 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 38484442a356c..925f9fdfec1ac 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -2616,21 +2616,35 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, zend_op *opline, zend_op_ar op1_info = OP1_INFO(); op2_info = OP2_INFO(); res_info = RES_INFO(); - if ((op1_info & (/*MAY_BE_UNDEF|*/MAY_BE_REF)) || - (op2_info & (/*MAY_BE_UNDEF|*/MAY_BE_REF)) || + if ((op2_info & (MAY_BE_UNDEF|MAY_BE_REF)) || !(op1_info & MAY_BE_ARRAY) || !(op2_info & (MAY_BE_LONG|MAY_BE_STRING))) { // TODO: support for references ??? goto fallback; } + if (op1_info & MAY_BE_REF) { + | lea FCARG1a, [FP + opline->op1.var] + | cmp byte [FCARG1a + 8], IS_REFERENCE + | jnz >1 + | mov FCARG1a, aword [FCARG1a] + | add FCARG1a, 8 + |1: + } + if (op1_info & MAY_BE_ARRAY) { - if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { - | // if (EXPECTED(Z_TYPE_P(container) == IS_ARRAY)) { - | cmp byte [FP + opline->op1.var + 8], IS_ARRAY + if (op1_info & MAY_BE_REF) { + | cmp byte [FCARG1a + 8], IS_ARRAY | jnz >7 // NOT_ARRAY + | mov FCARG1a, aword [FCARG1a] + } else { + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { + | // if (EXPECTED(Z_TYPE_P(container) == IS_ARRAY)) { + | cmp byte [FP + opline->op1.var + 8], IS_ARRAY + | jnz >7 // NOT_ARRAY + } + | LONG_LOAD opline->op1_type, opline->op1, FCARG1a } - | LONG_LOAD opline->op1_type, opline->op1, FCARG1a if (op2_info & MAY_BE_LONG) { if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_LONG)) { | // if (EXPECTED(Z_TYPE_P(dim) == IS_LONG)) { @@ -2662,6 +2676,7 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, zend_op *opline, zend_op_ar | jmp >2 // NOT_FOUND } } else { + |4: | // ZEND_HASH_INDEX_FIND(ht, hval, retval, num_undef); | test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED | jz >1 // HASH_FIND @@ -2725,6 +2740,7 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, zend_op *opline, zend_op_ar | LONG_LOAD opline->op2_type, opline->op2, FCARG2a | // retval = zend_hash_find(ht, offset_key); if (opline->op2_type != IS_CONST) { + |5: | EXT_CALL zend_jit_symtable_find, r0 } else { | EXT_CALL zend_hash_find, r0 @@ -2772,21 +2788,191 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, zend_op *opline, zend_op_ar |.cold_code |3: } - | // ???? - | int3 + if (op2_info & MAY_BE_REF) { + | cmp byte [FP + opline->op2.var + 8], IS_REFERENCE + | jnz >1 + | // ???? + | int3 + |1: + } + if (op2_info & MAY_BE_DOUBLE) { + | cmp byte [FP + opline->op2.var + 8], IS_DOUBLE + | jnz >1 + | // hval = zend_dval_to_lval(Z_DVAL_P(dim)); + | // ???? + | int3 + | // goto num_index; + | jmp <4 + |1: + } + if (op2_info & MAY_BE_NULL) { + | cmp byte [FP + opline->op2.var + 8], IS_NULL + | jnz >1 + | // offset_key = ZSTR_EMPTY_ALLOC(); + | LOAD_ADDR FCARG2a, CG(empty_string) + | // goto str_index; + | jmp <5 + |1: + } + if (op2_info & MAY_BE_FALSE) { + | cmp byte [FP + opline->op2.var + 8], IS_FALSE + | jnz >1 + | // hval = 0; + | xor FCARG2a, FCARG2a + | // goto num_index; + | jmp <4 + |1: + } + if (op2_info & MAY_BE_TRUE) { + | cmp byte [FP + opline->op2.var + 8], IS_TRUE + | jnz >1 + | // hval = 1; + | mov FCARG2a, 1 + | // goto num_index; + | jmp <4 + |1: + } + if (op2_info & MAY_BE_UNDEF) { + const char *name = ZSTR_VAL(op_array->vars[EX_VAR_TO_NUM(opline->op2.var)]); + + | cmp byte [FP + opline->op2.var + 8], IS_UNDEF + | jnz >1 + | // zend_error(E_NOTICE, "Undefined variable: %s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op2.var)))); + |.if X64 + | mov CARG1, E_NOTICE + | LOAD_ADDR CARG2, "Undefined variable: %s" + | LONG_ADDR CARG3, name + | EXT_CALL zend_error, r0 + |.else + | sub r4, 4 + | push name + | push "Undefined variable: %s" + | push E_NOTICE + | EXT_CALL zend_error, r0 + | add r4, 16 + |.endif + | // offset_key = ZSTR_EMPTY_ALLOC(); + | LOAD_ADDR FCARG2a, CG(empty_string) + | // goto str_index; + | jmp <5 + |1: + } + if (op2_info & MAY_BE_RESOURCE) { + | cmp byte [FP + opline->op2.var + 8], IS_RESOURCE + | jnz >1 + | // zend_error(E_NOTICE, "Resource ID#%d used as offset, casting to integer (%d)", Z_RES_HANDLE_P(dim), Z_RES_HANDLE_P(dim)); + | // hval = Z_RES_HANDLE_P(dim); + | // ???? + | int3 + | // goto num_index; + | jmp <4 + |1: + } + if (op2_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) { + | // zend_error(E_WARNING, "Illegal offset type"); + |.if X64 + | mov CARG1, E_WARNING + | LOAD_ADDR CARG2, "Illegal offset type" + | EXT_CALL zend_error, r0 + |.else + | sub r4, 8 + | push "Undefined variable: %s" + | push E_WARNING + | EXT_CALL zend_error, r0 + | add r4, 16 + |.endif + | mov dword [FP +opline->result.var + 8], IS_NULL + | jmp >9 // END + } + if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) { |.code } } } - if (op1_info & (MAY_BE_ANY-MAY_BE_ARRAY)) { + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY)) { if (op1_info & MAY_BE_ARRAY) { |.cold_code |7: } - | // ???? - | int3 + + if (op1_info & MAY_BE_STRING) { + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_STRING))) { + if (op1_info & MAY_BE_REF) { + | cmp byte [FCARG1a + 8], IS_STRING + } else { + | cmp byte [FP + opline->op1.var + 8], IS_STRING + } + | jnz >6 + } + | // ???? + | int3 + |6: + } + + if (op1_info & MAY_BE_OBJECT) { + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_STRING|MAY_BE_OBJECT))) { + if (op1_info & MAY_BE_REF) { + | cmp byte [FCARG1a + 8], IS_OBJECT + } else { + | cmp byte [FP + opline->op1.var + 8], IS_OBJECT + } + | jnz >6 + } + | // ???? + | int3 + |6: + } + + if (opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) { + const char *name = ZSTR_VAL(op_array->vars[EX_VAR_TO_NUM(opline->op1.var)]); + + if (op1_info & MAY_BE_REF) { + | cmp byte [FCARG1a + 8], IS_UNDEF + } else { + | cmp byte [FP + opline->op1.var + 8], IS_UNDEF + } + | jnz >1 + | // zend_error(E_NOTICE, "Undefined variable: %s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); + |.if X64 + | mov CARG1, E_NOTICE + | LOAD_ADDR CARG2, "Undefined variable: %s" + | LONG_ADDR CARG3, name + | EXT_CALL zend_error, r0 + |.else + | sub r4, 4 + | push name + | push "Undefined variable: %s" + | push E_NOTICE + | EXT_CALL zend_error, r0 + | add r4, 16 + |.endif + |1: + } + if (op2_info & MAY_BE_UNDEF) { + const char *name = ZSTR_VAL(op_array->vars[EX_VAR_TO_NUM(opline->op2.var)]); + + | cmp byte [FP + opline->op2.var + 8], IS_UNDEF + | jnz >1 + | // zend_error(E_NOTICE, "Undefined variable: %s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op2.var)))); + |.if X64 + | mov CARG1, E_NOTICE + | LOAD_ADDR CARG2, "Undefined variable: %s" + | LONG_ADDR CARG3, name + | EXT_CALL zend_error, r0 + |.else + | sub r4, 4 + | push name + | push "Undefined variable: %s" + | push E_NOTICE + | EXT_CALL zend_error, r0 + | add r4, 16 + |.endif + |1: + } + | mov dword [FP +opline->result.var + 8], IS_NULL + | jmp >9 // END if (op1_info & MAY_BE_ARRAY) { |.code } From cb0698930669bd100bdecc357e7bf01e753610bf Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 16 Sep 2016 01:04:31 +0300 Subject: [PATCH 198/569] type --- ext/opcache/jit/zend_jit_x86.dasc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 925f9fdfec1ac..6f32751d8a0e7 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -2841,7 +2841,7 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, zend_op *opline, zend_op_ar |.if X64 | mov CARG1, E_NOTICE | LOAD_ADDR CARG2, "Undefined variable: %s" - | LONG_ADDR CARG3, name + | LOAD_ADDR CARG3, name | EXT_CALL zend_error, r0 |.else | sub r4, 4 @@ -2938,7 +2938,7 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, zend_op *opline, zend_op_ar |.if X64 | mov CARG1, E_NOTICE | LOAD_ADDR CARG2, "Undefined variable: %s" - | LONG_ADDR CARG3, name + | LOAD_ADDR CARG3, name | EXT_CALL zend_error, r0 |.else | sub r4, 4 @@ -2959,7 +2959,7 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, zend_op *opline, zend_op_ar |.if X64 | mov CARG1, E_NOTICE | LOAD_ADDR CARG2, "Undefined variable: %s" - | LONG_ADDR CARG3, name + | LOAD_ADDR CARG3, name | EXT_CALL zend_error, r0 |.else | sub r4, 4 From 7c42291a9478b642e7664126948af9df6687c8b7 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 16 Sep 2016 01:04:47 +0300 Subject: [PATCH 199/569] Allow enabling incomplete JIT --- ext/opcache/jit/zend_jit.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 9299dbc1bc0b5..cba118e1cbdcc 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -844,6 +844,14 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) goto jit_failure; } break; +#ifdef INCOMPLETE_JIT + case ZEND_FETCH_DIM_R: + case ZEND_FETCH_DIM_IS: + if (!zend_jit_fetch_dim_read(&dasm_state, opline, op_array, ssa)) { + goto jit_failure; + } + break; +#endif #endif case ZEND_RECV_INIT: if (ssa->cfg.split_at_recv) { From 17d07c9ac0e138cc9127f9c58e32624d94683d6f Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 16 Sep 2016 14:02:30 +0300 Subject: [PATCH 200/569] JIT for FETCH_DIM_R (incomplete) --- ext/opcache/jit/zend_jit_disasm_x86.c | 8 + ext/opcache/jit/zend_jit_helpers.c | 291 ++++++++++++++++++++++++++ ext/opcache/jit/zend_jit_x86.dasc | 235 ++++++++------------- 3 files changed, 392 insertions(+), 142 deletions(-) diff --git a/ext/opcache/jit/zend_jit_disasm_x86.c b/ext/opcache/jit/zend_jit_disasm_x86.c index 4118a487a8c69..36fad428d4366 100644 --- a/ext/opcache/jit/zend_jit_disasm_x86.c +++ b/ext/opcache/jit/zend_jit_disasm_x86.c @@ -387,6 +387,14 @@ static int zend_jit_disasm_init(void) REGISTER_HELPER(zend_jit_leave_nested_func_helper); REGISTER_HELPER(zend_jit_leave_top_func_helper); REGISTER_HELPER(zend_jit_symtable_find); + REGISTER_HELPER(zend_jit_undefined_op_helper); + REGISTER_HELPER(zend_jit_fetch_dim_r_helper); + REGISTER_HELPER(zend_jit_fetch_dim_is_helper); + REGISTER_HELPER(zend_jit_fetch_dim_str_r_helper); + REGISTER_HELPER(zend_jit_fetch_dim_str_is_helper); + REGISTER_HELPER(zend_jit_fetch_dim_obj_r_helper); + REGISTER_HELPER(zend_jit_fetch_dim_obj_is_helper); + REGISTER_HELPER(zend_jit_zval_copy_unref_helper); #undef REGISTER_HELPER zend_elf_load_symbols(); diff --git a/ext/opcache/jit/zend_jit_helpers.c b/ext/opcache/jit/zend_jit_helpers.c index e272f60efb335..78b70dae70508 100644 --- a/ext/opcache/jit/zend_jit_helpers.c +++ b/ext/opcache/jit/zend_jit_helpers.c @@ -68,6 +68,297 @@ static zval* ZEND_FASTCALL zend_jit_symtable_find(HashTable *ht, zend_string *st return zend_hash_find(ht, str); } +static void ZEND_FASTCALL zend_jit_undefined_op_helper(uint32_t var) +{ + const zend_execute_data *execute_data = EG(current_execute_data); + zend_string *cv = EX(func)->op_array.vars[EX_VAR_TO_NUM(var)]; + + zend_error(E_NOTICE, "Undefined variable: %s", ZSTR_VAL(cv)); +} + +static void ZEND_FASTCALL zend_jit_fetch_dim_r_helper(zend_array *ht, zval *dim, zval *result) +{ + zend_long hval; + zend_string *offset_key; + zval *retval; + + if (Z_TYPE_P(dim) == IS_REFERENCE) { + dim = Z_REFVAL_P(dim); + } + + switch (Z_TYPE_P(dim)) { + case IS_UNDEF: + zend_jit_undefined_op_helper(EG(current_execute_data)->opline->op2.var); + /* break missing intentionally */ + case IS_NULL: + offset_key = ZSTR_EMPTY_ALLOC(); + goto str_index; + case IS_DOUBLE: + hval = zend_dval_to_lval(Z_DVAL_P(dim)); + goto num_index; + case IS_RESOURCE: + zend_error(E_NOTICE, "Resource ID#%d used as offset, casting to integer (%d)", Z_RES_HANDLE_P(dim), Z_RES_HANDLE_P(dim)); + hval = Z_RES_HANDLE_P(dim); + goto num_index; + case IS_FALSE: + hval = 0; + goto num_index; + case IS_TRUE: + hval = 1; + goto num_index; + default: + zend_error(E_WARNING, "Illegal offset type"); + ZVAL_NULL(result); + return; + } + +str_index: + retval = zend_hash_find(ht, offset_key); + if (retval) { + /* support for $GLOBALS[...] */ + if (UNEXPECTED(Z_TYPE_P(retval) == IS_INDIRECT)) { + retval = Z_INDIRECT_P(retval); + if (UNEXPECTED(Z_TYPE_P(retval) == IS_UNDEF)) { + zend_error(E_NOTICE, "Undefined index: %s", ZSTR_VAL(offset_key)); + ZVAL_NULL(result); + return; + } + } + } else { + zend_error(E_NOTICE, "Undefined index: %s", ZSTR_VAL(offset_key)); + ZVAL_NULL(result); + return; + } + ZVAL_COPY_UNREF(result, retval); + return; + +num_index: + ZEND_HASH_INDEX_FIND(ht, hval, retval, num_undef); + ZVAL_COPY_UNREF(result, retval); + return; + +num_undef: + zend_error(E_NOTICE,"Undefined offset: " ZEND_LONG_FMT, hval); + ZVAL_NULL(result); +} + +static void ZEND_FASTCALL zend_jit_fetch_dim_is_helper(zend_array *ht, zval *dim, zval *result) +{ + zend_long hval; + zend_string *offset_key; + zval *retval; + + if (Z_TYPE_P(dim) == IS_REFERENCE) { + dim = Z_REFVAL_P(dim); + } + + switch (Z_TYPE_P(dim)) { + case IS_UNDEF: + zend_jit_undefined_op_helper(EG(current_execute_data)->opline->op2.var); + /* break missing intentionally */ + case IS_NULL: + offset_key = ZSTR_EMPTY_ALLOC(); + goto str_index; + case IS_DOUBLE: + hval = zend_dval_to_lval(Z_DVAL_P(dim)); + goto num_index; + case IS_RESOURCE: + zend_error(E_NOTICE, "Resource ID#%d used as offset, casting to integer (%d)", Z_RES_HANDLE_P(dim), Z_RES_HANDLE_P(dim)); + hval = Z_RES_HANDLE_P(dim); + goto num_index; + case IS_FALSE: + hval = 0; + goto num_index; + case IS_TRUE: + hval = 1; + goto num_index; + default: + zend_error(E_WARNING, "Illegal offset type"); + ZVAL_NULL(result); + return; + } + +str_index: + retval = zend_hash_find(ht, offset_key); + if (retval) { + /* support for $GLOBALS[...] */ + if (UNEXPECTED(Z_TYPE_P(retval) == IS_INDIRECT)) { + retval = Z_INDIRECT_P(retval); + if (UNEXPECTED(Z_TYPE_P(retval) == IS_UNDEF)) { + ZVAL_NULL(result); + return; + } + } + } else { + ZVAL_NULL(result); + return; + } + ZVAL_COPY_UNREF(result, retval); + return; + +num_index: + ZEND_HASH_INDEX_FIND(ht, hval, retval, num_undef); + ZVAL_COPY_UNREF(result, retval); + return; + +num_undef: + ZVAL_NULL(result); +} + +static void ZEND_FASTCALL zend_jit_fetch_dim_str_r_helper(zval *container, zval *dim, zval *result) +{ + zend_long offset; + +try_string_offset: + if (UNEXPECTED(Z_TYPE_P(dim) != IS_LONG)) { + switch (Z_TYPE_P(dim)) { + /* case IS_LONG: */ + case IS_STRING: + if (IS_LONG == is_numeric_string(Z_STRVAL_P(dim), Z_STRLEN_P(dim), NULL, NULL, -1)) { + break; + } + zend_error(E_WARNING, "Illegal string offset '%s'", Z_STRVAL_P(dim)); + break; + case IS_UNDEF: + zend_jit_undefined_op_helper(EG(current_execute_data)->opline->op2.var); + case IS_DOUBLE: + case IS_NULL: + case IS_FALSE: + case IS_TRUE: + zend_error(E_NOTICE, "String offset cast occurred"); + break; + case IS_REFERENCE: + dim = Z_REFVAL_P(dim); + goto try_string_offset; + default: + zend_error(E_WARNING, "Illegal offset type"); + break; + } + + offset = _zval_get_long_func(dim); + } else { + offset = Z_LVAL_P(dim); + } + + if (UNEXPECTED(Z_STRLEN_P(container) < (size_t)((offset < 0) ? -offset : (offset + 1)))) { + zend_error(E_NOTICE, "Uninitialized string offset: " ZEND_LONG_FMT, offset); + ZVAL_EMPTY_STRING(result); + } else { + zend_uchar c; + zend_long real_offset; + + real_offset = (UNEXPECTED(offset < 0)) /* Handle negative offset */ + ? (zend_long)Z_STRLEN_P(container) + offset : offset; + c = (zend_uchar)Z_STRVAL_P(container)[real_offset]; + if (CG(one_char_string)[c]) { + ZVAL_INTERNED_STR(result, CG(one_char_string)[c]); + } else { + ZVAL_NEW_STR(result, zend_string_init(Z_STRVAL_P(container) + real_offset, 1, 0)); + } + } +} + +static void ZEND_FASTCALL zend_jit_fetch_dim_str_is_helper(zval *container, zval *dim, zval *result) +{ + zend_long offset; + +try_string_offset: + if (UNEXPECTED(Z_TYPE_P(dim) != IS_LONG)) { + switch (Z_TYPE_P(dim)) { + /* case IS_LONG: */ + case IS_STRING: + if (IS_LONG == is_numeric_string(Z_STRVAL_P(dim), Z_STRLEN_P(dim), NULL, NULL, -1)) { + break; + } + ZVAL_NULL(result); + return; + case IS_UNDEF: + zend_jit_undefined_op_helper(EG(current_execute_data)->opline->op2.var); + case IS_DOUBLE: + case IS_NULL: + case IS_FALSE: + case IS_TRUE: + break; + case IS_REFERENCE: + dim = Z_REFVAL_P(dim); + goto try_string_offset; + default: + zend_error(E_WARNING, "Illegal offset type"); + break; + } + + offset = _zval_get_long_func(dim); + } else { + offset = Z_LVAL_P(dim); + } + + if (UNEXPECTED(Z_STRLEN_P(container) < (size_t)((offset < 0) ? -offset : (offset + 1)))) { + ZVAL_NULL(result); + } else { + zend_uchar c; + zend_long real_offset; + + real_offset = (UNEXPECTED(offset < 0)) /* Handle negative offset */ + ? (zend_long)Z_STRLEN_P(container) + offset : offset; + c = (zend_uchar)Z_STRVAL_P(container)[real_offset]; + if (CG(one_char_string)[c]) { + ZVAL_INTERNED_STR(result, CG(one_char_string)[c]); + } else { + ZVAL_NEW_STR(result, zend_string_init(Z_STRVAL_P(container) + real_offset, 1, 0)); + } + } +} + +static void ZEND_FASTCALL zend_jit_fetch_dim_obj_r_helper(zval *container, zval *dim, zval *result) +{ + if (UNEXPECTED(Z_TYPE_P(dim) == IS_UNDEF)) { + zend_jit_undefined_op_helper(EG(current_execute_data)->opline->op2.var); + dim = &EG(uninitialized_zval); + } + if (!Z_OBJ_HT_P(container)->read_dimension) { + zend_throw_error(NULL, "Cannot use object as array"); + ZVAL_NULL(result); + } else { + zval *retval = Z_OBJ_HT_P(container)->read_dimension(container, dim, BP_VAR_R, result); + + if (retval) { + if (result != retval) { + ZVAL_COPY(result, retval); + } + } else { + ZVAL_NULL(result); + } + } +} + +static void ZEND_FASTCALL zend_jit_fetch_dim_obj_is_helper(zval *container, zval *dim, zval *result) +{ + if (UNEXPECTED(Z_TYPE_P(dim) == IS_UNDEF)) { + zend_jit_undefined_op_helper(EG(current_execute_data)->opline->op2.var); + dim = &EG(uninitialized_zval); + } + if (!Z_OBJ_HT_P(container)->read_dimension) { + zend_throw_error(NULL, "Cannot use object as array"); + ZVAL_NULL(result); + } else { + zval *retval = Z_OBJ_HT_P(container)->read_dimension(container, dim, BP_VAR_IS, result); + + if (retval) { + if (result != retval) { + ZVAL_COPY(result, retval); + } + } else { + ZVAL_NULL(result); + } + } +} + +static void ZEND_FASTCALL zend_jit_zval_copy_unref_helper(zval *dst, zval *src) +{ + ZVAL_UNREF(src); + ZVAL_COPY(dst, src); +} + /* * Local variables: * tab-width: 4 diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 6f32751d8a0e7..12b6787d09cc7 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -612,6 +612,20 @@ static void* dasm_labels[zend_lb_MAX]; | EXT_CALL gc_possible_root, r0 |.endmacro +|.macro SAVE_VALID_OPLINE +|| if (valid_opline || valid_opline_offset) { +|| if (!valid_opline) { +| lea r0, [IP + sizeof(zend_op) * valid_opline_offset] +|| } else { +|| const zend_op *target_opline = valid_opline + valid_opline_offset; +| LOAD_ADDR r0, target_opline +|| } +| mov EX->opline, r0 +|| } else { +| mov EX->opline, IP +|| } +|.endmacro + static const zend_op *valid_opline; static uint32_t valid_opline_offset; static int jit_return_label; @@ -2616,12 +2630,6 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, zend_op *opline, zend_op_ar op1_info = OP1_INFO(); op2_info = OP2_INFO(); res_info = RES_INFO(); - if ((op2_info & (MAY_BE_UNDEF|MAY_BE_REF)) || - !(op1_info & MAY_BE_ARRAY) || - !(op2_info & (MAY_BE_LONG|MAY_BE_STRING))) { - // TODO: support for references ??? - goto fallback; - } if (op1_info & MAY_BE_REF) { | lea FCARG1a, [FP + opline->op1.var] @@ -2676,7 +2684,6 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, zend_op *opline, zend_op_ar | jmp >2 // NOT_FOUND } } else { - |4: | // ZEND_HASH_INDEX_FIND(ht, hval, retval, num_undef); | test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED | jz >1 // HASH_FIND @@ -2740,7 +2747,6 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, zend_op *opline, zend_op_ar | LONG_LOAD opline->op2_type, opline->op2, FCARG2a | // retval = zend_hash_find(ht, offset_key); if (opline->op2_type != IS_CONST) { - |5: | EXT_CALL zend_jit_symtable_find, r0 } else { | EXT_CALL zend_hash_find, r0 @@ -2788,103 +2794,22 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, zend_op *opline, zend_op_ar |.cold_code |3: } - if (op2_info & MAY_BE_REF) { - | cmp byte [FP + opline->op2.var + 8], IS_REFERENCE - | jnz >1 - | // ???? - | int3 - |1: - } - if (op2_info & MAY_BE_DOUBLE) { - | cmp byte [FP + opline->op2.var + 8], IS_DOUBLE - | jnz >1 - | // hval = zend_dval_to_lval(Z_DVAL_P(dim)); - | // ???? - | int3 - | // goto num_index; - | jmp <4 - |1: - } - if (op2_info & MAY_BE_NULL) { - | cmp byte [FP + opline->op2.var + 8], IS_NULL - | jnz >1 - | // offset_key = ZSTR_EMPTY_ALLOC(); - | LOAD_ADDR FCARG2a, CG(empty_string) - | // goto str_index; - | jmp <5 - |1: - } - if (op2_info & MAY_BE_FALSE) { - | cmp byte [FP + opline->op2.var + 8], IS_FALSE - | jnz >1 - | // hval = 0; - | xor FCARG2a, FCARG2a - | // goto num_index; - | jmp <4 - |1: - } - if (op2_info & MAY_BE_TRUE) { - | cmp byte [FP + opline->op2.var + 8], IS_TRUE - | jnz >1 - | // hval = 1; - | mov FCARG2a, 1 - | // goto num_index; - | jmp <4 - |1: - } - if (op2_info & MAY_BE_UNDEF) { - const char *name = ZSTR_VAL(op_array->vars[EX_VAR_TO_NUM(opline->op2.var)]); - - | cmp byte [FP + opline->op2.var + 8], IS_UNDEF - | jnz >1 - | // zend_error(E_NOTICE, "Undefined variable: %s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op2.var)))); - |.if X64 - | mov CARG1, E_NOTICE - | LOAD_ADDR CARG2, "Undefined variable: %s" - | LOAD_ADDR CARG3, name - | EXT_CALL zend_error, r0 - |.else - | sub r4, 4 - | push name - | push "Undefined variable: %s" - | push E_NOTICE - | EXT_CALL zend_error, r0 - | add r4, 16 - |.endif - | // offset_key = ZSTR_EMPTY_ALLOC(); - | LOAD_ADDR FCARG2a, CG(empty_string) - | // goto str_index; - | jmp <5 - |1: - } - if (op2_info & MAY_BE_RESOURCE) { - | cmp byte [FP + opline->op2.var + 8], IS_RESOURCE - | jnz >1 - | // zend_error(E_NOTICE, "Resource ID#%d used as offset, casting to integer (%d)", Z_RES_HANDLE_P(dim), Z_RES_HANDLE_P(dim)); - | // hval = Z_RES_HANDLE_P(dim); - | // ???? - | int3 - | // goto num_index; - | jmp <4 - |1: - } - if (op2_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) { - | // zend_error(E_WARNING, "Illegal offset type"); - |.if X64 - | mov CARG1, E_WARNING - | LOAD_ADDR CARG2, "Illegal offset type" - | EXT_CALL zend_error, r0 - |.else - | sub r4, 8 - | push "Undefined variable: %s" - | push E_WARNING - | EXT_CALL zend_error, r0 - | add r4, 16 - |.endif - | mov dword [FP +opline->result.var + 8], IS_NULL - | jmp >9 // END + | SAVE_VALID_OPLINE + | LOAD_ZVAL_ADDR FCARG2a, opline->op2_type, opline->op2 + |.if X64 + | lea CARG3, [FP + opline->result.var] + |.else + | lea r0, [FP + opline->result.var] + | push r0 + |.endif + if (opline->opcode == ZEND_FETCH_DIM_R) { + | EXT_CALL zend_jit_fetch_dim_r_helper, r0 + } else if (opline->opcode == ZEND_FETCH_DIM_IS) { + | EXT_CALL zend_jit_fetch_dim_is_helper, r0 + } else { + ZEND_ASSERT(0); } - + | jmp >9 // END if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) { |.code } @@ -2906,8 +2831,25 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, zend_op *opline, zend_op_ar } | jnz >6 } - | // ???? - | int3 + | SAVE_VALID_OPLINE + if (!(op1_info & MAY_BE_REF)) { + | LOAD_ZVAL_ADDR FCARG1a, opline->op1_type, opline->op1 + } + | LOAD_ZVAL_ADDR FCARG2a, opline->op2_type, opline->op2 + |.if X64 + | lea CARG3, [FP + opline->result.var] + |.else + | lea r0, [FP + opline->result.var] + | push r0 + |.endif + if (opline->opcode == ZEND_FETCH_DIM_R) { + | EXT_CALL zend_jit_fetch_dim_str_r_helper, r0 + } else if (opline->opcode == ZEND_FETCH_DIM_IS) { + | EXT_CALL zend_jit_fetch_dim_str_is_helper, r0 + } else { + ZEND_ASSERT(0); + } + | jmp >9 // END |6: } @@ -2920,14 +2862,29 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, zend_op *opline, zend_op_ar } | jnz >6 } - | // ???? - | int3 + | SAVE_VALID_OPLINE + if (!(op1_info & MAY_BE_REF)) { + | LOAD_ZVAL_ADDR FCARG1a, opline->op1_type, opline->op1 + } + | LOAD_ZVAL_ADDR FCARG2a, opline->op2_type, opline->op2 + |.if X64 + | lea CARG3, [FP + opline->result.var] + |.else + | lea r0, [FP + opline->result.var] + | push r0 + |.endif + if (opline->opcode == ZEND_FETCH_DIM_R) { + | EXT_CALL zend_jit_fetch_dim_obj_r_helper, r0 + } else if (opline->opcode == ZEND_FETCH_DIM_IS) { + | EXT_CALL zend_jit_fetch_dim_obj_is_helper, r0 + } else { + ZEND_ASSERT(0); + } + | jmp >9 // END |6: } if (opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) { - const char *name = ZSTR_VAL(op_array->vars[EX_VAR_TO_NUM(opline->op1.var)]); - if (op1_info & MAY_BE_REF) { | cmp byte [FCARG1a + 8], IS_UNDEF } else { @@ -2935,53 +2892,47 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, zend_op *opline, zend_op_ar } | jnz >1 | // zend_error(E_NOTICE, "Undefined variable: %s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); - |.if X64 - | mov CARG1, E_NOTICE - | LOAD_ADDR CARG2, "Undefined variable: %s" - | LOAD_ADDR CARG3, name - | EXT_CALL zend_error, r0 - |.else - | sub r4, 4 - | push name - | push "Undefined variable: %s" - | push E_NOTICE - | EXT_CALL zend_error, r0 - | add r4, 16 - |.endif + | mov FCARG1d, opline->op1.var + | EXT_CALL zend_jit_undefined_op_helper, r0 |1: } if (op2_info & MAY_BE_UNDEF) { - const char *name = ZSTR_VAL(op_array->vars[EX_VAR_TO_NUM(opline->op2.var)]); - | cmp byte [FP + opline->op2.var + 8], IS_UNDEF | jnz >1 - | // zend_error(E_NOTICE, "Undefined variable: %s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op2.var)))); - |.if X64 - | mov CARG1, E_NOTICE - | LOAD_ADDR CARG2, "Undefined variable: %s" - | LOAD_ADDR CARG3, name - | EXT_CALL zend_error, r0 - |.else - | sub r4, 4 - | push name - | push "Undefined variable: %s" - | push E_NOTICE - | EXT_CALL zend_error, r0 - | add r4, 16 - |.endif + | mov FCARG1d, opline->op2.var + | EXT_CALL zend_jit_undefined_op_helper, r0 |1: } - | mov dword [FP +opline->result.var + 8], IS_NULL + | mov dword [FP + opline->result.var + 8], IS_NULL | jmp >9 // END if (op1_info & MAY_BE_ARRAY) { |.code } } - |8: // COPY - | // UNREF ???? + |8: + if (res_info & MAY_BE_REF) { + | // ZVAL_COPY_UNREF + | test byte [r0 + 9], IS_TYPE_REFCOUNTED + | jz >2 + | mov r1, aword [r0] + | cmp byte [r0 + 8], IS_REFERENCE + | jnz >1 + | cmp dword [r1], 1 + | jnz >1 + | lea FCARG1a, [FP + opline->result.var] + | mov FCARG2a, r0 + | EXT_CALL zend_jit_zval_copy_unref_helper, r0 + | jmp >9 + |1: + | inc dword [r1] + |2: + | ZVAL_COPY_VALUE FP, opline->result.var, r0, 0, MAY_BE_ANY, r1, ecx, r2 + } else { + | // ZVAL_COPY | ZVAL_COPY_VALUE FP, opline->result.var, r0, 0, MAY_BE_ANY, r1, ecx, r2 | TRY_ADDREF res_info, ch, r2 + } |9: // END | FREE_OP opline->op2_type, opline->op2, op2_info, op_array, opline->lineno From 359a7901251898740f7cb1c9d36a341d1f490d9b Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 16 Sep 2016 14:33:21 +0300 Subject: [PATCH 201/569] JIT for FETCH_DIM_R and FETCH_DIM_IS (done) --- ext/opcache/jit/zend_jit.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index cba118e1cbdcc..a963c8985f95b 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -844,14 +844,12 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) goto jit_failure; } break; -#ifdef INCOMPLETE_JIT case ZEND_FETCH_DIM_R: case ZEND_FETCH_DIM_IS: if (!zend_jit_fetch_dim_read(&dasm_state, opline, op_array, ssa)) { goto jit_failure; } break; -#endif #endif case ZEND_RECV_INIT: if (ssa->cfg.split_at_recv) { From df8946753998ce1fa7cbed9411a0914f5c406da8 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Sun, 18 Sep 2016 11:33:42 +0800 Subject: [PATCH 202/569] Implemented ZEND_BIND_GLOBALS --- ext/opcache/jit/zend_jit.c | 9 +- ext/opcache/jit/zend_jit_disasm_x86.c | 2 + ext/opcache/jit/zend_jit_helpers.c | 42 +++++++ ext/opcache/jit/zend_jit_x86.dasc | 162 +++++++++++++++++++++++++- 4 files changed, 206 insertions(+), 9 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index a963c8985f95b..dc12fc173848c 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -850,6 +850,11 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) goto jit_failure; } break; + case ZEND_BIND_GLOBAL: + if (!zend_jit_bind_global(&dasm_state, opline, op_array, ssa)) { + goto jit_failure; + } + break; #endif case ZEND_RECV_INIT: if (ssa->cfg.split_at_recv) { @@ -858,9 +863,7 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) } break; } - /* break missing intentionally */ - case ZEND_BIND_GLOBAL: - if (opline->opcode != op_array->opcodes[i+1].opcode) { + if (op_array->opcodes[i+1].opcode != ZEND_RECV_INIT) { /* repeatable opcodes */ if (!zend_jit_handler(&dasm_state, opline, zend_may_throw(opline, op_array, ssa))) { goto jit_failure; diff --git a/ext/opcache/jit/zend_jit_disasm_x86.c b/ext/opcache/jit/zend_jit_disasm_x86.c index 36fad428d4366..39f28373a73d0 100644 --- a/ext/opcache/jit/zend_jit_disasm_x86.c +++ b/ext/opcache/jit/zend_jit_disasm_x86.c @@ -395,6 +395,8 @@ static int zend_jit_disasm_init(void) REGISTER_HELPER(zend_jit_fetch_dim_obj_r_helper); REGISTER_HELPER(zend_jit_fetch_dim_obj_is_helper); REGISTER_HELPER(zend_jit_zval_copy_unref_helper); + REGISTER_HELPER(zend_jit_new_ref_helper); + REGISTER_HELPER(zend_jit_fetch_global_helper); #undef REGISTER_HELPER zend_elf_load_symbols(); diff --git a/ext/opcache/jit/zend_jit_helpers.c b/ext/opcache/jit/zend_jit_helpers.c index 78b70dae70508..54d06c4b6d721 100644 --- a/ext/opcache/jit/zend_jit_helpers.c +++ b/ext/opcache/jit/zend_jit_helpers.c @@ -359,6 +359,48 @@ static void ZEND_FASTCALL zend_jit_zval_copy_unref_helper(zval *dst, zval *src) ZVAL_COPY(dst, src); } +static zval* ZEND_FASTCALL zend_jit_new_ref_helper(zval *value) +{ + zend_reference *ref = (zend_reference*)emalloc(sizeof(zend_reference)); + GC_REFCOUNT(ref) = 1; + GC_TYPE_INFO(ref) = IS_REFERENCE; + ZVAL_COPY_VALUE(&ref->val, value); + Z_REF_P(value) = ref; + Z_TYPE_INFO_P(value) = IS_REFERENCE_EX; + + return value; +} + +static zval* ZEND_FASTCALL zend_jit_fetch_global_helper(zend_execute_data *execute_data, zval *varname) +{ + uint32_t idx; + zval *value = zend_hash_find(&EG(symbol_table), Z_STR_P(varname)); + + if (UNEXPECTED(value == NULL)) { + value = zend_hash_add_new(&EG(symbol_table), Z_STR_P(varname), &EG(uninitialized_zval)); + idx = ((char*)value - (char*)EG(symbol_table).arData) / sizeof(Bucket); + /* Store "hash slot index" + 1 (NULL is a mark of uninitialized cache slot) */ + CACHE_PTR(Z_CACHE_SLOT_P(varname), (void*)(uintptr_t)(idx + 1)); + } else { + idx = ((char*)value - (char*)EG(symbol_table).arData) / sizeof(Bucket); + /* Store "hash slot index" + 1 (NULL is a mark of uninitialized cache slot) */ + CACHE_PTR(Z_CACHE_SLOT_P(varname), (void*)(uintptr_t)(idx + 1)); + /* GLOBAL variable may be an INDIRECT pointer to CV */ + if (UNEXPECTED(Z_TYPE_P(value) == IS_INDIRECT)) { + value = Z_INDIRECT_P(value); + if (UNEXPECTED(Z_TYPE_P(value) == IS_UNDEF)) { + ZVAL_NULL(value); + } + } + } + + if (UNEXPECTED(!Z_ISREF_P(value))) { + value = zend_jit_new_ref_helper(value); + } + + return value; +} + /* * Local variables: * tab-width: 4 diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 12b6787d09cc7..14e39b1c8f2df 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -2647,7 +2647,7 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, zend_op *opline, zend_op_ar | mov FCARG1a, aword [FCARG1a] } else { if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { - | // if (EXPECTED(Z_TYPE_P(container) == IS_ARRAY)) { + | // if (EXPECTED(Z_TYPE_P(container) == IS_ARRAY)) | cmp byte [FP + opline->op1.var + 8], IS_ARRAY | jnz >7 // NOT_ARRAY } @@ -2655,7 +2655,7 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, zend_op *opline, zend_op_ar } if (op2_info & MAY_BE_LONG) { if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_LONG)) { - | // if (EXPECTED(Z_TYPE_P(dim) == IS_LONG)) { + | // if (EXPECTED(Z_TYPE_P(dim) == IS_LONG)) | cmp byte [FP + opline->op2.var + 8], IS_LONG | jnz >3 // NOT_LONG } @@ -2668,7 +2668,7 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, zend_op *opline, zend_op_ar | // ZEND_HASH_INDEX_FIND(ht, hval, retval, num_undef); | test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED | jz >1 // HASH_FIND - | // if (EXPECTED((zend_ulong)(_h) < (zend_ulong)(_ht)->nNumUsed)) { + | // if (EXPECTED((zend_ulong)(_h) < (zend_ulong)(_ht)->nNumUsed)) |.if X64 | movsxd r0, dword [FCARG1a + offsetof(zend_array, nNumUsed)] | cmp r0, val @@ -2687,7 +2687,7 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, zend_op *opline, zend_op_ar | // ZEND_HASH_INDEX_FIND(ht, hval, retval, num_undef); | test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED | jz >1 // HASH_FIND - | // if (EXPECTED((zend_ulong)(_h) < (zend_ulong)(_ht)->nNumUsed)) { + | // if (EXPECTED((zend_ulong)(_h) < (zend_ulong)(_ht)->nNumUsed)) |.if X64 | movsxd r0, dword [FCARG1a + offsetof(zend_array, nNumUsed)] | cmp r0, FCARG2a @@ -2696,7 +2696,11 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, zend_op *opline, zend_op_ar |.endif | jle >2 // NOT_FOUND | // _ret = &_ht->arData[_h].val; + |.if X64 + | shl FCARG2a, 5 + |.else | imul FCARG2a, sizeof(Bucket) + |.endif | mov r0, aword [FCARG1a + offsetof(zend_array, arData)] | add r0, FCARG2a | cmp dword [r0 + 8], IS_UNDEF @@ -2739,7 +2743,7 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, zend_op *opline, zend_op_ar } if (op2_info & MAY_BE_STRING) { if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) { - | // if (EXPECTED(Z_TYPE_P(dim) == IS_STRING)) { + | // if (EXPECTED(Z_TYPE_P(dim) == IS_STRING)) | cmp byte [FP + opline->op2.var + 8], IS_STRING | jnz >3 // NOT_STRING } @@ -2753,7 +2757,7 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, zend_op *opline, zend_op_ar } | test r0, r0 | je >2 // NOT_FOUND - | // if (UNEXPECTED(Z_TYPE_P(retval) == IS_INDIRECT)) { + | // if (UNEXPECTED(Z_TYPE_P(retval) == IS_INDIRECT)) | cmp dword [r0 + 8], IS_INDIRECT | jz >1 // SLOW |.cold_code @@ -2953,6 +2957,152 @@ fallback: return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); } +static int zend_jit_bind_global(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) +{ + uint32_t op1_info; + zval *varname = RT_CONSTANT(op_array, opline->op2); + + if (!ssa->ops || !ssa->var_info) { + op1_info = MAY_BE_ANY|MAY_BE_REF; + } else { + op1_info = OP1_INFO(); + } + + //idx = (uint32_t)(uintptr_t)CACHED_PTR(Z_CACHE_SLOT_P(varname)) - 1; + | mov r0, EX->run_time_cache + | mov r0, aword [r0 + Z_CACHE_SLOT_P(varname)] + | dec r0 + //if (EXPECTED(idx < EG(symbol_table).nNumUsed)) + | cmp r0, [&EG(symbol_table).nNumUsed] + | jbe >9 + //Bucket *p = EG(symbol_table).arData + idx; + |.if X64 + | shl r0, 5 + |.else + | imul r0, sizeof(Bucket) + |.endif + |.if X64 + | LOAD_ADDR r1, &EG(symbol_table.arData) + | add r0, [r1] + |.else + | add r0, [&EG(symbol_table).arData] + |.endif + //if (EXPECTED(Z_TYPE(p->val) != IS_UNDEF) + | cmp byte [r0 + 8], IS_UNDEF + | je >9 + // (EXPECTED(p->key == Z_STR_P(varname)) + | LOAD_ADDR r1, Z_PTR_P(varname) + | cmp [r0 + offsetof(Bucket, key)], r1 + | jne >2 + |1: + |.cold_code + |2: + |.if X64 + | mov64 r2, ZSTR_H(Z_STR_P(varname)) + | cmp qword [r0 + offsetof(Bucket, h)], r2 + |.else + | cmp dword [r0 + offsetof(Bucket, h)], ZSTR_H(Z_STR_P(varname)) + |.endif + | jne >9 + | test r1, r1 + | jz >9 + | cmp dword [r0 + offsetof(zend_string, len)], Z_STRLEN_P(varname) + | jne >9 + | push r0 + |.if X64 + | mov CARG1, r1 + | LOAD_ADDR CARG2, Z_STRVAL_P(varname) + | mov CARG3, Z_STRLEN_P(varname) + | call &memcmp + |.else + | add r4, 12 + | push r1 + | push Z_STRVAL_P(varname) + | push Z_STRLEN_P(varname) + | call &memcmp + | sub r4, 24 + |.endif + | test al, al + | pop r0 + | jnz >9 + | jmp <1 + |.code + // if (UNEXPECTED(Z_TYPE_P(value) == IS_INDIRECT)) + | cmp byte [r0 + 8], IS_INDIRECT + | je >2 + |1: + |.cold_code + |2: + //value = Z_INDIRECT_P(value) + | mov r0, [r0] + | cmp byte [r0 + 8], IS_UNDEF + | jne <1 + | SET_TYPE_INFO r0, 0, IS_NULL + | jmp >8 + |.code + | cmp byte [r0 + 8], IS_REFERENCE + | jne >8 + |4: + if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { + | mov r2, r0 + } + | mov r0, [r0] + | inc dword [r0] + // End of handler + if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { + if (op1_info & (MAY_BE_ANY - (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { + | test byte [FP + opline->op1.var + 9], IS_TYPE_REFCOUNTED + | jnz >1 + } + |.cold_code + |1: + | lea FCARG1a, aword [FP + opline->op1.var] + | cmp FCARG1a, r2 + | je >3 + | mov FCARG1a, [FCARG1a] + | dec dword [FCARG1a] + | jnz >2 + | push r0 + | ZVAL_DTOR_FUNC (op_array->filename ? ZSTR_VAL(op_array->filename) : NULL), opline->lineno + | pop r0 + | jmp >5 + |2: + | test byte [FP + opline->op1.var + 9], IS_TYPE_COLLECTABLE + | jz >5 + | cmp word [FCARG1a + 6], 0 + | jne >5 + | push r0 + | EXT_CALL gc_possible_root, r1 + | pop r0 + | jmp >5 + |3: + | mov FCARG1a, [FCARG1a] + | dec dword [FCARG1a] + | jmp >5 + |.code + } + |5: + | mov aword [FP + opline->op1.var], r0 + | mov dword [FP + opline->op1.var + 8], IS_REFERENCE_EX + //END of handler + + |.cold_code + |8: + | mov FCARG1a, r0 + | EXT_CALL zend_jit_new_ref_helper, r1 + | jmp <4 + |9: + | mov FCARG1a, FP + | LOAD_ADDR FCARG2a, (ptrdiff_t)varname + | EXT_CALL zend_jit_fetch_global_helper, r1 + | jmp <4 + |.code + + valid_opline_offset++; + + return 1; +} + #endif /* From f6d98a4a0a75a5257c78867f25b561c61b049183 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Sun, 18 Sep 2016 12:00:11 +0800 Subject: [PATCH 203/569] Micro opt for ZEND_MUL --- ext/opcache/jit/zend_jit_x86.dasc | 33 +++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 14e39b1c8f2df..24d406a6a1449 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -1075,12 +1075,33 @@ static int zend_jit_math_long_long(dasm_State **Dst, zend_op *opline, zend_op_ar uint32_t res_info = RES_INFO(); zend_bool same_ops = (opline->op1_type == opline->op2_type) && (opline->op1.var == opline->op2.var); - | LONG_LOAD opline->op1_type, opline->op1, r0 - || if (same_ops && opline->opcode != ZEND_DIV) { - | LONG_MATH2 opline->opcode, r0, r0 - || } else { - | LONG_MATH opline->opcode, opline->op2_type, opline->op2, r0 - || } + if (opline->opcode == ZEND_MUL && + ((opline->op2_type == IS_CONST && + IS_32BIT(Z_LVAL_P(RT_CONSTANT(op_array, opline->op2))) && + is_power_of_two(Z_LVAL_P(RT_CONSTANT(op_array, opline->op2)))) || + (opline->op1_type == IS_CONST && + IS_32BIT(Z_LVAL_P(RT_CONSTANT(op_array, opline->op1))) && + is_power_of_two(Z_LVAL_P(RT_CONSTANT(op_array, opline->op1)))))) { + if (opline->op2_type == IS_CONST) { + | LONG_LOAD opline->op1_type, opline->op1, r0 + | shl r0, floor_log2(Z_LVAL_P(RT_CONSTANT(op_array, opline->op2))) + } else { + | LONG_LOAD opline->op2_type, opline->op2, r0 + | shl r0, floor_log2(Z_LVAL_P(RT_CONSTANT(op_array, opline->op1))) + } + } else if (opline->opcode == ZEND_DIV && + (opline->op2_type == IS_CONST && + is_power_of_two(Z_LVAL_P(RT_CONSTANT(op_array, opline->op2))))) { + | LONG_LOAD opline->op1_type, opline->op1, r0 + | shr r0, floor_log2(Z_LVAL_P(RT_CONSTANT(op_array, opline->op2))) + } else { + | LONG_LOAD opline->op1_type, opline->op1, r0 + if (same_ops && opline->opcode != ZEND_DIV) { + | LONG_MATH2 opline->opcode, r0, r0 + } else { + | LONG_MATH opline->opcode, opline->op2_type, opline->op2, r0 + } + } if ((res_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) { if (res_forward) { | LONG_STORE r1, res_forward, r0 From d84e9814a2d24acdc2365ec3d16355584a582d97 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Sun, 18 Sep 2016 16:11:20 +0800 Subject: [PATCH 204/569] Implemeneted jit for ZEND_SL/SR --- ext/opcache/jit/zend_jit.c | 6 +++ ext/opcache/jit/zend_jit_x86.dasc | 69 +++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index dc12fc173848c..845391bba9b71 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -786,6 +786,12 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) goto jit_failure; } break; + case ZEND_SR: + case ZEND_SL: + if (!zend_jit_shift(&dasm_state, opline, op_array, ssa)) { + goto jit_failure; + } + break; case ZEND_ADD: case ZEND_SUB: case ZEND_MUL: diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 24d406a6a1449..4740e27133043 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -1222,6 +1222,75 @@ static int zend_jit_math_double_double(dasm_State **Dst, zend_op *opline, uint32 return 1; } + +static int zend_jit_shift(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) +{ + zval *op2; + uint32_t op1_info; + zend_bool has_slow = 0; + + if (!ssa->ops || !ssa->var_info) { + goto fallback; + } + + op1_info = OP1_INFO(); + + if ((op1_info & MAY_BE_UNDEF) || + opline->op2_type != IS_CONST || + Z_TYPE_P(RT_CONSTANT(op_array, opline->op2)) != IS_LONG || + Z_LVAL_P(RT_CONSTANT(op_array, opline->op2)) >= SIZEOF_ZEND_LONG * 8) { + goto fallback; + } + + op2 = RT_CONSTANT(op_array, opline->op2); + + if (op1_info & (MAY_BE_ANY-MAY_BE_LONG)) { + | cmp dword [FP + opline->op1.var + 8], IS_LONG + | JNE_SLOW + } + | LONG_LOAD opline->op1_type, opline->op1, r0 + if (opline->opcode == ZEND_SR) { + | shr, r0, Z_LVAL_P(op2) + } else { + | shl, r0, Z_LVAL_P(op2) + } + | LONG_STORE FP, opline->result.var, r0 + | SET_TYPE_INFO FP, opline->result.var, IS_LONG + if (has_slow) { + |2: + } + | FREE_OP opline->op1_type, opline->op1, op1_info, op_array, opline->lineno + //END + + if (has_slow) { + |.cold_code + |9: + | lea FCARG1a, aword [FP + opline->result.var] + |.if X64 + | mov FCARG2a, r0 + | mov CARG3, Z_LVAL_P(op2) + |.else + | push r0 + | push dword Z_LVAL_P(op2) + |.endif + if (opline->opcode == ZEND_SR) { + | EXT_CALL shift_right_function, r0 + } else { + | EXT_CALL shift_left_function, r0 + } + | jmp <2 + |.code + } + + valid_opline_offset++; + + return 1; + +fallback: + /* fallback to subroutine threading */ + return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); +} + static int zend_jit_math(dasm_State **Dst, zend_op *opline, int *opnum, zend_op_array *op_array, zend_ssa *ssa) { uint32_t op1_info, op2_info; From 392388f9494065136c74bf496732df63cfade219 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Sun, 18 Sep 2016 19:59:09 +0800 Subject: [PATCH 205/569] Cleanup --- ext/opcache/jit/zend_jit_helpers.c | 6 +- ext/opcache/jit/zend_jit_x86.dasc | 121 +++++++++++++++-------------- 2 files changed, 65 insertions(+), 62 deletions(-) diff --git a/ext/opcache/jit/zend_jit_helpers.c b/ext/opcache/jit/zend_jit_helpers.c index 54d06c4b6d721..dfa9ea69a822b 100644 --- a/ext/opcache/jit/zend_jit_helpers.c +++ b/ext/opcache/jit/zend_jit_helpers.c @@ -359,7 +359,7 @@ static void ZEND_FASTCALL zend_jit_zval_copy_unref_helper(zval *dst, zval *src) ZVAL_COPY(dst, src); } -static zval* ZEND_FASTCALL zend_jit_new_ref_helper(zval *value) +static ZEND_FASTCALL zend_jit_new_ref_helper(zval *value) { zend_reference *ref = (zend_reference*)emalloc(sizeof(zend_reference)); GC_REFCOUNT(ref) = 1; @@ -367,8 +367,6 @@ static zval* ZEND_FASTCALL zend_jit_new_ref_helper(zval *value) ZVAL_COPY_VALUE(&ref->val, value); Z_REF_P(value) = ref; Z_TYPE_INFO_P(value) = IS_REFERENCE_EX; - - return value; } static zval* ZEND_FASTCALL zend_jit_fetch_global_helper(zend_execute_data *execute_data, zval *varname) @@ -395,7 +393,7 @@ static zval* ZEND_FASTCALL zend_jit_fetch_global_helper(zend_execute_data *execu } if (UNEXPECTED(!Z_ISREF_P(value))) { - value = zend_jit_new_ref_helper(value); + zend_jit_new_ref_helper(value); } return value; diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 4740e27133043..ea03e1be7108c 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -3059,133 +3059,138 @@ static int zend_jit_bind_global(dasm_State **Dst, zend_op *opline, zend_op_array } //idx = (uint32_t)(uintptr_t)CACHED_PTR(Z_CACHE_SLOT_P(varname)) - 1; - | mov r0, EX->run_time_cache - | mov r0, aword [r0 + Z_CACHE_SLOT_P(varname)] - | dec r0 + | mov r3, EX->run_time_cache + | mov r3, aword [r3 + Z_CACHE_SLOT_P(varname)] + | dec r3 //if (EXPECTED(idx < EG(symbol_table).nNumUsed)) - | cmp r0, [&EG(symbol_table).nNumUsed] + | cmp r3, [&EG(symbol_table).nNumUsed] | jbe >9 //Bucket *p = EG(symbol_table).arData + idx; |.if X64 - | shl r0, 5 + | shl r3, 5 |.else - | imul r0, sizeof(Bucket) + | imul r3, sizeof(Bucket) |.endif |.if X64 - | LOAD_ADDR r1, &EG(symbol_table.arData) - | add r0, [r1] + | LOAD_ADDR r0, &EG(symbol_table.arData) + | add r3, [r0] |.else - | add r0, [&EG(symbol_table).arData] + | add r3, [&EG(symbol_table).arData] |.endif //if (EXPECTED(Z_TYPE(p->val) != IS_UNDEF) - | cmp byte [r0 + 8], IS_UNDEF + | cmp byte [r3 + 8], IS_UNDEF | je >9 // (EXPECTED(p->key == Z_STR_P(varname)) - | LOAD_ADDR r1, Z_PTR_P(varname) - | cmp [r0 + offsetof(Bucket, key)], r1 - | jne >2 - |1: + | LOAD_ADDR r0, Z_PTR_P(varname) + | cmp [r3 + offsetof(Bucket, key)], r0 + | jne >1 |.cold_code - |2: + |1: + //(EXPECTED(p->h == ZSTR_H(Z_STR_P(varname))) |.if X64 - | mov64 r2, ZSTR_H(Z_STR_P(varname)) - | cmp qword [r0 + offsetof(Bucket, h)], r2 + | mov64 r1, ZSTR_H(Z_STR_P(varname)) + | cmp qword [r3 + offsetof(Bucket, h)], r1 |.else - | cmp dword [r0 + offsetof(Bucket, h)], ZSTR_H(Z_STR_P(varname)) + | cmp dword [r3 + offsetof(Bucket, h)], ZSTR_H(Z_STR_P(varname)) |.endif | jne >9 - | test r1, r1 + //EXPECTED(p->key != NULL) + | mov r2, [r3 + offsetof(Bucket, key)] + | test r2, r2 | jz >9 - | cmp dword [r0 + offsetof(zend_string, len)], Z_STRLEN_P(varname) + //EXPECTED(ZSTR_LEN(p->key) == Z_STRLEN_P(varname)) + | cmp dword [r2 + offsetof(zend_string, len)], Z_STRLEN_P(varname) | jne >9 - | push r0 + //EXPECTED(memcmp(ZSTR_VAL(p->key), Z_STRVAL_P(varname), Z_STRLEN_P(varname)) == 0) + | add r2, offsetof(zend_string, val) |.if X64 - | mov CARG1, r1 + | mov CARG1, r2 | LOAD_ADDR CARG2, Z_STRVAL_P(varname) | mov CARG3, Z_STRLEN_P(varname) | call &memcmp |.else - | add r4, 12 - | push r1 + | push r2 | push Z_STRVAL_P(varname) | push Z_STRLEN_P(varname) | call &memcmp - | sub r4, 24 + | sub r4, 12 |.endif | test al, al - | pop r0 | jnz >9 - | jmp <1 + | jmp >2 |.code + |2: // if (UNEXPECTED(Z_TYPE_P(value) == IS_INDIRECT)) - | cmp byte [r0 + 8], IS_INDIRECT - | je >2 - |1: + | mov al, byte [r3 + 8] + | cmp al, IS_INDIRECT + | je >1 |.cold_code - |2: + |1: //value = Z_INDIRECT_P(value) - | mov r0, [r0] - | cmp byte [r0 + 8], IS_UNDEF - | jne <1 - | SET_TYPE_INFO r0, 0, IS_NULL + | mov r3, [r3] + | cmp byte [r3 + 8], IS_UNDEF + | jne >2 + | SET_TYPE_INFO r3, 0, IS_NULL | jmp >8 |.code - | cmp byte [r0 + 8], IS_REFERENCE + |2: + | cmp al, IS_REFERENCE | jne >8 - |4: + |1: if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { - | mov r2, r0 + | mov r0, r3 } - | mov r0, [r0] - | inc dword [r0] - // End of handler + | mov r3, [r3] + | inc dword [r3] + //if (UNEXPECTED(Z_REFCOUNTED_P(variable_ptr))) if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { if (op1_info & (MAY_BE_ANY - (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { | test byte [FP + opline->op1.var + 9], IS_TYPE_REFCOUNTED - | jnz >1 + | jnz >2 } |.cold_code - |1: + //zval_dtor_func(Z_COUNTED_P(variable_ptr)) + |2: + //if (EXPECTED(variable_ptr != value)) | lea FCARG1a, aword [FP + opline->op1.var] - | cmp FCARG1a, r2 - | je >3 + | cmp FCARG1a, r0 + | je >4 | mov FCARG1a, [FCARG1a] | dec dword [FCARG1a] - | jnz >2 - | push r0 + | jnz >3 | ZVAL_DTOR_FUNC (op_array->filename ? ZSTR_VAL(op_array->filename) : NULL), opline->lineno - | pop r0 | jmp >5 - |2: + |3: + // GC_ZVAL_CHECK_POSSIBLE_ROOT(variable_ptr) | test byte [FP + opline->op1.var + 9], IS_TYPE_COLLECTABLE | jz >5 | cmp word [FCARG1a + 6], 0 | jne >5 - | push r0 | EXT_CALL gc_possible_root, r1 - | pop r0 | jmp >5 - |3: + |4: | mov FCARG1a, [FCARG1a] | dec dword [FCARG1a] | jmp >5 |.code } |5: - | mov aword [FP + opline->op1.var], r0 + //ZVAL_REF(variable_ptr, ref) + | mov aword [FP + opline->op1.var], r3 | mov dword [FP + opline->op1.var + 8], IS_REFERENCE_EX //END of handler |.cold_code |8: - | mov FCARG1a, r0 - | EXT_CALL zend_jit_new_ref_helper, r1 - | jmp <4 + | mov FCARG1a, r3 + | EXT_CALL zend_jit_new_ref_helper, r0 + | jmp <1 |9: | mov FCARG1a, FP | LOAD_ADDR FCARG2a, (ptrdiff_t)varname - | EXT_CALL zend_jit_fetch_global_helper, r1 - | jmp <4 + | EXT_CALL zend_jit_fetch_global_helper, r0 + | mov r3, r0 + | jmp <1 |.code valid_opline_offset++; From bcfd54f5bb9e705a3b165b5959043c5653037950 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Sun, 18 Sep 2016 20:06:12 +0800 Subject: [PATCH 206/569] Fixed compiler warning --- ext/opcache/jit/zend_jit_helpers.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_helpers.c b/ext/opcache/jit/zend_jit_helpers.c index dfa9ea69a822b..f71b9ee6069b8 100644 --- a/ext/opcache/jit/zend_jit_helpers.c +++ b/ext/opcache/jit/zend_jit_helpers.c @@ -359,7 +359,7 @@ static void ZEND_FASTCALL zend_jit_zval_copy_unref_helper(zval *dst, zval *src) ZVAL_COPY(dst, src); } -static ZEND_FASTCALL zend_jit_new_ref_helper(zval *value) +static void ZEND_FASTCALL zend_jit_new_ref_helper(zval *value) { zend_reference *ref = (zend_reference*)emalloc(sizeof(zend_reference)); GC_REFCOUNT(ref) = 1; From 5746c7fd3e6d2e22c32abf64e861774d684c645f Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Sun, 18 Sep 2016 23:27:08 +0800 Subject: [PATCH 207/569] test with jz/jnz, cmp with je/jne --- ext/opcache/jit/zend_jit_x86.dasc | 60 +++++++++++++++---------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index ea03e1be7108c..3fb894a0a6b4b 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -459,7 +459,7 @@ static void* dasm_labels[zend_lb_MAX]; ||if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { || if (val_info & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { | test type_flags_reg, IS_TYPE_REFCOUNTED -| je >1 +| jz >1 || } | inc dword [value_ptr_reg] |1: @@ -495,7 +495,7 @@ static void* dasm_labels[zend_lb_MAX]; |.macro ZVAL_PTR_DTOR_NOGC, base, offset, filename, lineno | test byte [base + offset + 9], IS_TYPE_REFCOUNTED -| je >1 +| jz >1 | mov FCARG1a, aword [base + offset] | dec dword [FCARG1a] | jnz >1 @@ -509,11 +509,11 @@ static void* dasm_labels[zend_lb_MAX]; | // if (Z_REFCOUNTED_P(cv)) { | test byte [base + offset + 9], IS_TYPE_REFCOUNTED || if (cold) { -| jne >1 +| jnz >1 |.cold_code |1: || } else { -| je >3 +| jz >3 || } || } | // if (!Z_DELREF_P(cv)) { @@ -733,7 +733,7 @@ static int zend_jit_leave_function_stub(dasm_State **Dst) |->leave_function_handler: | add r4, SPAD | test FCARG1d, ZEND_CALL_TOP - | jne >1 + | jnz >1 | EXT_JMP zend_jit_leave_nested_func_helper, r0 |1: | EXT_JMP zend_jit_leave_top_func_helper, r0 @@ -877,7 +877,7 @@ static int zend_jit_jmp(dasm_State **Dst, unsigned int target_label) static int zend_jit_cond_jmp(dasm_State **Dst, zend_op *next_opline, unsigned int target_label) { | cmp IPl, next_opline - | jnz =>target_label + | jne =>target_label return 1; } @@ -888,9 +888,9 @@ static int zend_jit_smart_branch(dasm_State **Dst, zend_op *opline, unsigned int zend_op *target_opline = OP_JMP_ADDR(opline, opline->op2); | cmp IPl, next_opline - | jz =>next_label + | je =>next_label | cmp IPl, target_opline - | jz =>target_label + | je =>target_label return 1; } @@ -906,7 +906,7 @@ static int zend_jit_context_threaded_call(dasm_State **Dst, zend_op *opline) zend_op *next_opline = opline + 1; | cmp IPl, next_opline - | jz >1 + | je >1 | call aword [IP] zend_jit_check_exception(Dst); |1: @@ -951,7 +951,7 @@ static int zend_jit_new(dasm_State **Dst, zend_op *opline, int *opnum, zend_op_a if (!ce || ce->constructor) { zend_op *next_opline = opline + 1; | cmp IPl, next_opline - | jnz >1 + | jne >1 zend_jit_call(Dst, next_opline); |1: } else { @@ -2474,7 +2474,7 @@ static int zend_jit_leave_func(dasm_State **Dst, zend_op *opline, zend_op_array | movzx FCARG1d, byte [FP + offsetof(zend_execute_data, This.u1.type_info) + 2] | test FCARG1d, (ZEND_CALL_TOP|ZEND_CALL_HAS_SYMBOL_TABLE|ZEND_CALL_FREE_EXTRA_ARGS|ZEND_CALL_ALLOCATED) - | jne ->leave_function_handler + | jnz ->leave_function_handler | // EG(current_execute_data) = EX(prev_execute_data); | mov r0, EX->prev_execute_data | mov aword [&EG(current_execute_data)], r0 @@ -2482,7 +2482,7 @@ static int zend_jit_leave_func(dasm_State **Dst, zend_op *opline, zend_op_array if (op_array->scope) { | test FCARG1d, ZEND_CALL_RELEASE_THIS if (op_array->fn_flags & ZEND_ACC_STATIC) { - | jne >1 + | jnz >1 |.cold_code |1: } else { @@ -2498,7 +2498,7 @@ static int zend_jit_leave_func(dasm_State **Dst, zend_op *opline, zend_op_array |6: | // if (call_info & ZEND_CALL_CTOR) | test FCARG1d, ZEND_CALL_CTOR - | je >5 + | jz >5 | // GC_REFCOUNT(object)--; | dec dword [r0] | // zend_object_store_ctor_failed(object); @@ -2527,7 +2527,7 @@ static int zend_jit_leave_func(dasm_State **Dst, zend_op *opline, zend_op_array } } | test FCARG1d, ZEND_CALL_CLOSURE - | jne >3 + | jnz >3 |.cold_code |3: | // OBJ_RELEASE((zend_object*)execute_data->func->op_array.prototype); @@ -2619,7 +2619,7 @@ static int zend_jit_return(dasm_State **Dst, zend_op *opline, zend_op_array *op_ } else if (opline->op1_type == IS_CV) { if (op1_info & MAY_BE_REF) { | cmp byte [FP + opline->op1.var + 8], IS_REFERENCE - | jz >1 + | je >1 |.cold_code |1: | // retval_ptr = Z_REFVAL_P(retval_ptr); @@ -2724,7 +2724,7 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, zend_op *opline, zend_op_ar if (op1_info & MAY_BE_REF) { | lea FCARG1a, [FP + opline->op1.var] | cmp byte [FCARG1a + 8], IS_REFERENCE - | jnz >1 + | jne >1 | mov FCARG1a, aword [FCARG1a] | add FCARG1a, 8 |1: @@ -2733,13 +2733,13 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, zend_op *opline, zend_op_ar if (op1_info & MAY_BE_ARRAY) { if (op1_info & MAY_BE_REF) { | cmp byte [FCARG1a + 8], IS_ARRAY - | jnz >7 // NOT_ARRAY + | jne >7 // NOT_ARRAY | mov FCARG1a, aword [FCARG1a] } else { if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { | // if (EXPECTED(Z_TYPE_P(container) == IS_ARRAY)) | cmp byte [FP + opline->op1.var + 8], IS_ARRAY - | jnz >7 // NOT_ARRAY + | jne >7 // NOT_ARRAY } | LONG_LOAD opline->op1_type, opline->op1, FCARG1a } @@ -2747,7 +2747,7 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, zend_op *opline, zend_op_ar if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_LONG)) { | // if (EXPECTED(Z_TYPE_P(dim) == IS_LONG)) | cmp byte [FP + opline->op2.var + 8], IS_LONG - | jnz >3 // NOT_LONG + | jne >3 // NOT_LONG } | // hval = Z_LVAL_P(dim); | LONG_LOAD opline->op2_type, opline->op2, FCARG2a @@ -2800,7 +2800,7 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, zend_op *opline, zend_op_ar |1: | EXT_CALL zend_hash_index_find, r0 | test r0, r0 - | je >2 // NOT_FOUND + | jz >2 // NOT_FOUND |.cold_code |2: if (opline->opcode == ZEND_FETCH_DIM_R) { @@ -2835,7 +2835,7 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, zend_op *opline, zend_op_ar if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) { | // if (EXPECTED(Z_TYPE_P(dim) == IS_STRING)) | cmp byte [FP + opline->op2.var + 8], IS_STRING - | jnz >3 // NOT_STRING + | jne >3 // NOT_STRING } | // offset_key = Z_STR_P(dim); | LONG_LOAD opline->op2_type, opline->op2, FCARG2a @@ -2846,16 +2846,16 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, zend_op *opline, zend_op_ar | EXT_CALL zend_hash_find, r0 } | test r0, r0 - | je >2 // NOT_FOUND + | jz >2 // NOT_FOUND | // if (UNEXPECTED(Z_TYPE_P(retval) == IS_INDIRECT)) | cmp dword [r0 + 8], IS_INDIRECT - | jz >1 // SLOW + | je >1 // SLOW |.cold_code |1: | // retval = Z_INDIRECT_P(retval); | mov r0, aword [r0] | cmp dword [r0 + 8], IS_UNDEF - | jnz >8 // COPY + | jne >8 // COPY |2: if (opline->opcode == ZEND_FETCH_DIM_R) { // zend_error(E_NOTICE, "Undefined index: %s", ZSTR_VAL(offset_key)); @@ -2923,7 +2923,7 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, zend_op *opline, zend_op_ar } else { | cmp byte [FP + opline->op1.var + 8], IS_STRING } - | jnz >6 + | jne >6 } | SAVE_VALID_OPLINE if (!(op1_info & MAY_BE_REF)) { @@ -2954,7 +2954,7 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, zend_op *opline, zend_op_ar } else { | cmp byte [FP + opline->op1.var + 8], IS_OBJECT } - | jnz >6 + | jne >6 } | SAVE_VALID_OPLINE if (!(op1_info & MAY_BE_REF)) { @@ -2984,7 +2984,7 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, zend_op *opline, zend_op_ar } else { | cmp byte [FP + opline->op1.var + 8], IS_UNDEF } - | jnz >1 + | jne >1 | // zend_error(E_NOTICE, "Undefined variable: %s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); | mov FCARG1d, opline->op1.var | EXT_CALL zend_jit_undefined_op_helper, r0 @@ -2992,7 +2992,7 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, zend_op *opline, zend_op_ar } if (op2_info & MAY_BE_UNDEF) { | cmp byte [FP + opline->op2.var + 8], IS_UNDEF - | jnz >1 + | jne >1 | mov FCARG1d, opline->op2.var | EXT_CALL zend_jit_undefined_op_helper, r0 |1: @@ -3011,9 +3011,9 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, zend_op *opline, zend_op_ar | jz >2 | mov r1, aword [r0] | cmp byte [r0 + 8], IS_REFERENCE - | jnz >1 + | jne >1 | cmp dword [r1], 1 - | jnz >1 + | jne >1 | lea FCARG1a, [FP + opline->result.var] | mov FCARG2a, r0 | EXT_CALL zend_jit_zval_copy_unref_helper, r0 From b1576e0140d15eb0447c00f8667c7e36d64b29c2 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 19 Sep 2016 10:00:55 +0300 Subject: [PATCH 208/569] BIND_GLOBAL related fixes (it doesn't work anyway, at least on 32-bit) --- ext/opcache/jit/zend_jit.c | 6 +++++- ext/opcache/jit/zend_jit_x86.dasc | 5 +++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 845391bba9b71..760bcb3968cc1 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -869,7 +869,11 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) } break; } - if (op_array->opcodes[i+1].opcode != ZEND_RECV_INIT) { + /* break missing intentionally */ +#if ZEND_JIT_LEVEL < ZEND_JIT_LEVEL_OPT_FUNC + case ZEND_BIND_GLOBAL: +#endif + if (opline->opcode != op_array->opcodes[i+1].opcode) { /* repeatable opcodes */ if (!zend_jit_handler(&dasm_state, opline, zend_may_throw(opline, op_array, ssa))) { goto jit_failure; diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 3fb894a0a6b4b..3110dfe4851b0 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -3064,7 +3064,7 @@ static int zend_jit_bind_global(dasm_State **Dst, zend_op *opline, zend_op_array | dec r3 //if (EXPECTED(idx < EG(symbol_table).nNumUsed)) | cmp r3, [&EG(symbol_table).nNumUsed] - | jbe >9 + | jae >9 //Bucket *p = EG(symbol_table).arData + idx; |.if X64 | shl r3, 5 @@ -3109,11 +3109,12 @@ static int zend_jit_bind_global(dasm_State **Dst, zend_op *opline, zend_op_array | mov CARG3, Z_STRLEN_P(varname) | call &memcmp |.else + | sub r4, 4 | push r2 | push Z_STRVAL_P(varname) | push Z_STRLEN_P(varname) | call &memcmp - | sub r4, 12 + | add r4, 16 |.endif | test al, al | jnz >9 From 3f438237fd686596291d0eeefa71a64a6bd9b05b Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 19 Sep 2016 10:03:17 +0300 Subject: [PATCH 209/569] Disable incomplete JIT for BIND_GLOBAL --- ext/opcache/jit/zend_jit.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 760bcb3968cc1..01c1f4686e391 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -856,11 +856,12 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) goto jit_failure; } break; - case ZEND_BIND_GLOBAL: - if (!zend_jit_bind_global(&dasm_state, opline, op_array, ssa)) { - goto jit_failure; - } - break; +// TODO: Doesn't work on 32-bit beacause of %ebx clobbering +// case ZEND_BIND_GLOBAL: +// if (!zend_jit_bind_global(&dasm_state, opline, op_array, ssa)) { +// goto jit_failure; +// } +// break; #endif case ZEND_RECV_INIT: if (ssa->cfg.split_at_recv) { @@ -870,7 +871,7 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) break; } /* break missing intentionally */ -#if ZEND_JIT_LEVEL < ZEND_JIT_LEVEL_OPT_FUNC +#if 1 || ZEND_JIT_LEVEL < ZEND_JIT_LEVEL_OPT_FUNC case ZEND_BIND_GLOBAL: #endif if (opline->opcode != op_array->opcodes[i+1].opcode) { From 357574f527be229fc94c62c051cb15cff47dde78 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 19 Sep 2016 12:49:30 +0300 Subject: [PATCH 210/569] Improved JIT for PRE/POST_INC/DEC --- ext/opcache/jit/zend_jit_x86.dasc | 94 ++++++++++++++++++++++++++----- 1 file changed, 79 insertions(+), 15 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 3110dfe4851b0..541cd052a06c8 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -466,6 +466,38 @@ static void* dasm_labels[zend_lb_MAX]; ||} |.endmacro +// zval should be in FCARG1a +|.macro ZVAL_COPY_CTOR_FUNC, filename, lineno // arg1 must be in FCARG1a +|| if (ZEND_DEBUG) { +| LOAD_ADDR FCARG2a, filename +|.if X64 +| mov CARG3d, lineno +|.else +| push lineno +|.endif +|| } +| EXT_CALL _zval_copy_ctor_func, r0 +|.endmacro + +// zval should be in FCARG1a +|.macro ZVAL_COPY_CTOR, val_info, type_flags_reg, value_ptr_reg, filename, lineno +||if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { +|| if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY)) { +| test type_flags_reg, IS_TYPE_REFCOUNTED + IS_TYPE_IMMUTABLE +| jz >2 +| test type_flags_reg, IS_TYPE_COPYABLE + IS_TYPE_IMMUTABLE +| jz >1 +| inc dword [value_ptr_reg] +| jmp >2 +|1: +| ZVAL_COPY_CTOR_FUNC filename, lineno +|2: +|| } else { +| TRY_ADDREF val_info, type_flags_reg, value_ptr_reg +|| } +||} +|.endmacro + |.macro ZVAL_DEREF, reg, info || if (info & MAY_BE_REF) { | cmp byte [reg + 8], IS_REFERENCE @@ -481,6 +513,7 @@ static void* dasm_labels[zend_lb_MAX]; || has_slow = 1; |.endmacro +// zval should be in FCARG1a |.macro ZVAL_DTOR_FUNC, filename, lineno // arg1 must be in FCARG1a || if (ZEND_DEBUG) { | LOAD_ADDR FCARG2a, filename @@ -967,16 +1000,8 @@ static int zend_jit_inc_dec(dasm_State **Dst, zend_op *opline, zend_op_array *op { uint32_t op1_info, op1_def_info; - if (!ssa->ops || !ssa->var_info || opline->op1_type != IS_CV) { - goto fallback; - } op1_info = OP1_INFO(); - - if (op1_info & (MAY_BE_UNDEF|MAY_BE_STRING|MAY_BE_ARRAY)) { - // TODO: support for IS_STRING and IS_ARRAY ??? - goto fallback; - } - if (!(op1_info & MAY_BE_LONG)) { + if (opline->op1_type != IS_CV || !(op1_info & MAY_BE_LONG)) { goto fallback; } @@ -1032,22 +1057,60 @@ static int zend_jit_inc_dec(dasm_State **Dst, zend_op *opline, zend_op_array *op | ZVAL_COPY_VALUE FP, opline->result.var, FP, opline->op1.var, MAY_BE_LONG, r0, eax, r1 || } } - if (op1_info & (MAY_BE_ANY-MAY_BE_LONG)) { + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { |.cold_code |2: + || if (op1_info & MAY_BE_UNDEF) { + | cmp byte [FP + opline->op1.var + 8], IS_UNDEF + | jne >2 + | // zend_error(E_NOTICE, "Undefined variable: %s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); + | mov FCARG1d, opline->op1.var + | EXT_CALL zend_jit_undefined_op_helper, r0 + | mov dword [FP + opline->op1.var + 8], IS_NULL + || } + |2: | lea FCARG1a, [FP + opline->op1.var] | // ZVAL_DEREF(var_ptr); | ZVAL_DEREF FCARG1a, op1_info || if ((opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC)) { || if (opline->result_type != IS_UNUSED) { | ZVAL_COPY_VALUE FP, opline->result.var, FCARG1a, 0, op1_info, r0, eax, r2 - | // zval_opt_copy_ctor(var_ptr); - | // ??? - | TRY_ADDREF op1_info, ah, r2 + | //ZVAL_COPY_CTOR op1_info, ah, r2, (op_array->filename ? ZSTR_VAL(op_array->filename) : NULL), opline->lineno + || if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { + || if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY)) { + | test ah, IS_TYPE_REFCOUNTED + IS_TYPE_IMMUTABLE + | jz >2 + | test ah, IS_TYPE_COPYABLE + IS_TYPE_IMMUTABLE + | jnz >1 + | inc dword [r2] + | jmp >2 + |1: + | mov aword [r4], FCARG1a // save + | ZVAL_COPY_CTOR_FUNC (op_array->filename ? ZSTR_VAL(op_array->filename) : NULL), opline->lineno + | mov FCARG1a, aword [r4] // restore + |2: + || } else { + | TRY_ADDREF op1_info, ah, r2 + || } + || } || } || } else { - | // SEPARATE_ZVAL_NOREF(var_ptr); - | // ??? + || if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY)) { + | // SEPARATE_ZVAL_NOREF(var_ptr); + | test byte [FCARG1a + 9], IS_TYPE_COPYABLE + IS_TYPE_IMMUTABLE + | jz >2 + | mov r0, [FCARG1a] + | cmp dword [r0], 1 + | jbe >2 + | test byte [FCARG1a + 9], IS_TYPE_IMMUTABLE + | jne >1 + | dec dword [r0] + |1: + | mov aword [r4], FCARG1a // save + | ZVAL_COPY_CTOR_FUNC (op_array->filename ? ZSTR_VAL(op_array->filename) : NULL), opline->lineno + | mov FCARG1a, aword [r4] // restore + |2: + || } || } || if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { | EXT_CALL increment_function, r0 @@ -1057,6 +1120,7 @@ static int zend_jit_inc_dec(dasm_State **Dst, zend_op *opline, zend_op_array *op || if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && || opline->result_type != IS_UNUSED) { | ZVAL_COPY_VALUE FP, opline->result.var, FP, opline->op1.var, op1_def_info, r0, eax, r1 + | TRY_ADDREF op1_def_info, ah, r1 || } | jmp >3 |.code From 4615a50e2f1b0544d9b06db495925f052a596a34 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Mon, 19 Sep 2016 17:52:27 +0800 Subject: [PATCH 211/569] Fixed BIND_GLOBAL --- ext/opcache/jit/zend_jit_helpers.c | 6 ++- ext/opcache/jit/zend_jit_x86.dasc | 77 +++++++++++++++++------------- 2 files changed, 48 insertions(+), 35 deletions(-) diff --git a/ext/opcache/jit/zend_jit_helpers.c b/ext/opcache/jit/zend_jit_helpers.c index f71b9ee6069b8..88c50a7da34f4 100644 --- a/ext/opcache/jit/zend_jit_helpers.c +++ b/ext/opcache/jit/zend_jit_helpers.c @@ -359,7 +359,7 @@ static void ZEND_FASTCALL zend_jit_zval_copy_unref_helper(zval *dst, zval *src) ZVAL_COPY(dst, src); } -static void ZEND_FASTCALL zend_jit_new_ref_helper(zval *value) +static zval* ZEND_FASTCALL zend_jit_new_ref_helper(zval *value) { zend_reference *ref = (zend_reference*)emalloc(sizeof(zend_reference)); GC_REFCOUNT(ref) = 1; @@ -367,6 +367,8 @@ static void ZEND_FASTCALL zend_jit_new_ref_helper(zval *value) ZVAL_COPY_VALUE(&ref->val, value); Z_REF_P(value) = ref; Z_TYPE_INFO_P(value) = IS_REFERENCE_EX; + + return value; } static zval* ZEND_FASTCALL zend_jit_fetch_global_helper(zend_execute_data *execute_data, zval *varname) @@ -393,7 +395,7 @@ static zval* ZEND_FASTCALL zend_jit_fetch_global_helper(zend_execute_data *execu } if (UNEXPECTED(!Z_ISREF_P(value))) { - zend_jit_new_ref_helper(value); + return zend_jit_new_ref_helper(value); } return value; diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 3110dfe4851b0..842001ad6281f 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -3059,90 +3059,94 @@ static int zend_jit_bind_global(dasm_State **Dst, zend_op *opline, zend_op_array } //idx = (uint32_t)(uintptr_t)CACHED_PTR(Z_CACHE_SLOT_P(varname)) - 1; - | mov r3, EX->run_time_cache - | mov r3, aword [r3 + Z_CACHE_SLOT_P(varname)] - | dec r3 + | mov r0, EX->run_time_cache + | mov r0, aword [r0 + Z_CACHE_SLOT_P(varname)] + | dec r0 //if (EXPECTED(idx < EG(symbol_table).nNumUsed)) - | cmp r3, [&EG(symbol_table).nNumUsed] + | cmp r0, [&EG(symbol_table).nNumUsed] | jae >9 //Bucket *p = EG(symbol_table).arData + idx; |.if X64 - | shl r3, 5 + | shl r0, 5 |.else - | imul r3, sizeof(Bucket) + | imul r0, sizeof(Bucket) |.endif |.if X64 - | LOAD_ADDR r0, &EG(symbol_table.arData) - | add r3, [r0] + | LOAD_ADDR r1, &EG(symbol_table.arData) + | add r0, [r1] |.else - | add r3, [&EG(symbol_table).arData] + | add r0, [&EG(symbol_table).arData] |.endif //if (EXPECTED(Z_TYPE(p->val) != IS_UNDEF) - | cmp byte [r3 + 8], IS_UNDEF + | cmp byte [r0 + 8], IS_UNDEF | je >9 // (EXPECTED(p->key == Z_STR_P(varname)) - | LOAD_ADDR r0, Z_PTR_P(varname) - | cmp [r3 + offsetof(Bucket, key)], r0 + | LOAD_ADDR r1, Z_PTR_P(varname) + | cmp [r0 + offsetof(Bucket, key)], r1 | jne >1 |.cold_code |1: //(EXPECTED(p->h == ZSTR_H(Z_STR_P(varname))) |.if X64 | mov64 r1, ZSTR_H(Z_STR_P(varname)) - | cmp qword [r3 + offsetof(Bucket, h)], r1 + | cmp qword [r0 + offsetof(Bucket, h)], r1 |.else - | cmp dword [r3 + offsetof(Bucket, h)], ZSTR_H(Z_STR_P(varname)) + | cmp dword [r0 + offsetof(Bucket, h)], ZSTR_H(Z_STR_P(varname)) |.endif | jne >9 //EXPECTED(p->key != NULL) - | mov r2, [r3 + offsetof(Bucket, key)] - | test r2, r2 + | mov r1, [r0 + offsetof(Bucket, key)] + | test r1, r1 | jz >9 //EXPECTED(ZSTR_LEN(p->key) == Z_STRLEN_P(varname)) - | cmp dword [r2 + offsetof(zend_string, len)], Z_STRLEN_P(varname) + | cmp dword [r1 + offsetof(zend_string, len)], Z_STRLEN_P(varname) | jne >9 //EXPECTED(memcmp(ZSTR_VAL(p->key), Z_STRVAL_P(varname), Z_STRLEN_P(varname)) == 0) - | add r2, offsetof(zend_string, val) + | add r1, offsetof(zend_string, val) + | mov [r4], r0 |.if X64 - | mov CARG1, r2 + | mov CARG1, r1 | LOAD_ADDR CARG2, Z_STRVAL_P(varname) | mov CARG3, Z_STRLEN_P(varname) | call &memcmp |.else | sub r4, 4 - | push r2 + | push r1 | push Z_STRVAL_P(varname) | push Z_STRLEN_P(varname) | call &memcmp | add r4, 16 |.endif | test al, al + | mov r0, aword [r4] | jnz >9 | jmp >2 |.code |2: // if (UNEXPECTED(Z_TYPE_P(value) == IS_INDIRECT)) - | mov al, byte [r3 + 8] - | cmp al, IS_INDIRECT + | mov cl, byte [r0 + 8] + | cmp cl, IS_INDIRECT | je >1 |.cold_code |1: //value = Z_INDIRECT_P(value) - | mov r3, [r3] - | cmp byte [r3 + 8], IS_UNDEF + | mov r0, [r0] + | mov cl, byte [r0 + 8] + | cmp cl, IS_UNDEF | jne >2 - | SET_TYPE_INFO r3, 0, IS_NULL + | SET_TYPE_INFO r0, 0, IS_NULL | jmp >8 |.code |2: - | cmp al, IS_REFERENCE + | cmp cl, IS_REFERENCE | jne >8 |1: if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { - | mov r0, r3 + //stash this for later use + | mov r2, r0 } - | mov r3, [r3] - | inc dword [r3] + | mov r0, [r0] + | inc dword [r0] //if (UNEXPECTED(Z_REFCOUNTED_P(variable_ptr))) if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { if (op1_info & (MAY_BE_ANY - (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { @@ -3154,21 +3158,29 @@ static int zend_jit_bind_global(dasm_State **Dst, zend_op *opline, zend_op_array |2: //if (EXPECTED(variable_ptr != value)) | lea FCARG1a, aword [FP + opline->op1.var] - | cmp FCARG1a, r0 + | cmp FCARG1a, r2 | je >4 | mov FCARG1a, [FCARG1a] | dec dword [FCARG1a] + if (op1_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) { | jnz >3 + } + | mov [r4], r0 | ZVAL_DTOR_FUNC (op_array->filename ? ZSTR_VAL(op_array->filename) : NULL), opline->lineno + | mov r0, aword [r4] | jmp >5 + if (op1_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) { |3: // GC_ZVAL_CHECK_POSSIBLE_ROOT(variable_ptr) | test byte [FP + opline->op1.var + 9], IS_TYPE_COLLECTABLE | jz >5 | cmp word [FCARG1a + 6], 0 | jne >5 + | mov [r4], r0 | EXT_CALL gc_possible_root, r1 + | mov r0, aword [r4] | jmp >5 + } |4: | mov FCARG1a, [FCARG1a] | dec dword [FCARG1a] @@ -3177,20 +3189,19 @@ static int zend_jit_bind_global(dasm_State **Dst, zend_op *opline, zend_op_array } |5: //ZVAL_REF(variable_ptr, ref) - | mov aword [FP + opline->op1.var], r3 + | mov aword [FP + opline->op1.var], r0 | mov dword [FP + opline->op1.var + 8], IS_REFERENCE_EX //END of handler |.cold_code |8: - | mov FCARG1a, r3 + | mov FCARG1a, r0 | EXT_CALL zend_jit_new_ref_helper, r0 | jmp <1 |9: | mov FCARG1a, FP | LOAD_ADDR FCARG2a, (ptrdiff_t)varname | EXT_CALL zend_jit_fetch_global_helper, r0 - | mov r3, r0 | jmp <1 |.code From 99ff3cef1bcc7edac4ba95a365858b64514e7e75 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 19 Sep 2016 13:04:07 +0300 Subject: [PATCH 212/569] Reenabled JIT for BIND_GLOBAL --- ext/opcache/jit/zend_jit.c | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 01c1f4686e391..760bcb3968cc1 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -856,12 +856,11 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) goto jit_failure; } break; -// TODO: Doesn't work on 32-bit beacause of %ebx clobbering -// case ZEND_BIND_GLOBAL: -// if (!zend_jit_bind_global(&dasm_state, opline, op_array, ssa)) { -// goto jit_failure; -// } -// break; + case ZEND_BIND_GLOBAL: + if (!zend_jit_bind_global(&dasm_state, opline, op_array, ssa)) { + goto jit_failure; + } + break; #endif case ZEND_RECV_INIT: if (ssa->cfg.split_at_recv) { @@ -871,7 +870,7 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) break; } /* break missing intentionally */ -#if 1 || ZEND_JIT_LEVEL < ZEND_JIT_LEVEL_OPT_FUNC +#if ZEND_JIT_LEVEL < ZEND_JIT_LEVEL_OPT_FUNC case ZEND_BIND_GLOBAL: #endif if (opline->opcode != op_array->opcodes[i+1].opcode) { From 8dd97dfc54bf2ce0b4144a1de2cd843e57fbd4fc Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 19 Sep 2016 16:17:34 +0300 Subject: [PATCH 213/569] JIT support for assignment of IS_UNDEF and IS_REFERENCE. --- ext/opcache/jit/zend_jit_x86.dasc | 80 ++++++++++++++++++++----------- 1 file changed, 52 insertions(+), 28 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 046130187fbde..c8faee521334e 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -2071,47 +2071,70 @@ fallback: static int zend_jit_simple_assign(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, uint32_t var, uint32_t var_info, zend_uchar val_type, znode_op val, uint32_t val_info) { - if (val_info & (MAY_BE_UNDEF|MAY_BE_REF)) { - /* TODO: Support for references ??? */ - goto fallback; - } else if (val_type == IS_CONST) { + if (val_type == IS_CONST) { zval *zv = RT_CONSTANT(op_array, val); | ZVAL_COPY_CONST FP, var, var_info, zv, r0 || if (Z_REFCOUNTED_P(zv)) { | ADDREF_CONST zv, r0 || } - valid_opline_offset++; - return 1; + } else { + if (val_info & MAY_BE_UNDEF) { + | cmp byte [FP + val.var + 8], IS_UNDEF + | jz >1 + |.cold_code + |1: + | // zend_error(E_NOTICE, "Undefined variable: %s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); + | mov FCARG1d, opline->op1.var + | EXT_CALL zend_jit_undefined_op_helper, r0 + | mov dword[FP + var + 8], IS_NULL + | jmp >3 + |.code + } + if (val_info & MAY_BE_REF) { + | lea r2, [FP + val.var] + if (val_type == IS_CV) { + | ZVAL_DEREF r2, val_info + | ZVAL_COPY_VALUE FP, var, r2, 0, val_info, r0, eax, r1 + } else { + | cmp byte [FP + val.var + 8], IS_REFERENCE + | je >1 + |.cold_code + |1: + | // zend_refcounted *ref = Z_COUNTED_P(retval_ptr); + | mov r0, aword [FP + opline->op1.var] + | // ZVAL_COPY_VALUE(return_value, &ref->value); + | ZVAL_COPY_VALUE FP, var, r0, 8, val_info, r2, edx, r1 + | dec dword [r0] + | je >2 + | // if (IS_REFCOUNTED()) + | test dh, IS_TYPE_REFCOUNTED + | jz >3 + | // ADDREF + | inc dword [r1] + | jmp >3 + |2: + | EFREE_SIZE r0, sizeof(zend_reference), op_array, opline + | jmp >3 + |.code + | ZVAL_COPY_VALUE FP, var, FP, val.var, val_info, r0, eax, r1 + } + } else { + | ZVAL_COPY_VALUE FP, var, FP, val.var, val_info, r0, eax, r1 + } + || if (val_type == IS_CV) { + | TRY_ADDREF val_info, ah, r1 + || } + |3: } - - | ZVAL_COPY_VALUE FP, var, FP, val.var, val_info, r0, eax, r1 - || if (val_type == IS_CV) { - | TRY_ADDREF val_info, ah, r1 - || } - valid_opline_offset++; - return 1; - -fallback: - /* fallback to subroutine threading */ - return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); } static int zend_jit_qm_assign(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) { - uint32_t op1_info; - - if (!ssa->ops || !ssa->var_info) { - goto fallback; - } - - op1_info = OP1_INFO(); + uint32_t op1_info = OP1_INFO(); + valid_opline_offset++; return zend_jit_simple_assign(Dst, opline, op_array, ssa, opline->result.var, -1, opline->op1_type, opline->op1, op1_info); - -fallback: - /* fallback to subroutine threading */ - return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); } static int zend_jit_assign(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) @@ -2126,6 +2149,7 @@ static int zend_jit_assign(dasm_State **Dst, zend_op *opline, zend_op_array *op_ op2_info = OP2_INFO(); if (!(op1_info & (MAY_BE_STRING|MAY_BE_RESOURCE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_REF))) { + valid_opline_offset++; return zend_jit_simple_assign(Dst, opline, op_array, ssa, opline->op1.var, op1_info, opline->op2_type, opline->op2, op2_info); } From ec734e43abd9db6050983da9ec6d293311536c00 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 19 Sep 2016 22:32:18 +0300 Subject: [PATCH 214/569] Refactoring using more DynAsm macros --- ext/opcache/jit/zend_jit_x86.dasc | 847 +++++++++++++++--------------- 1 file changed, 434 insertions(+), 413 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index c8faee521334e..18b49a5281cf7 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -59,6 +59,7 @@ |.type EX, zend_execute_data, FP |.type OP, zend_op, IP +|.type ZVAL, zval |.actionlist dasm_actions @@ -137,8 +138,36 @@ static void* dasm_labels[zend_lb_MAX]; ||} |.endmacro -|.macro SET_TYPE_INFO, dst_base, dst_offset, type -| mov dword [dst_base + dst_offset + 8], type +|.macro GET_Z_TYPE_INFO, reg, zv +| mov reg, dword [zv + 8] +|.endmacro + +|.macro SET_Z_TYPE_INFO, zv, type +| mov dword [zv + 8], type +|.endmacro + +|.macro GET_Z_LVAL, reg, zv +| mov reg, aword [zv] +|.endmacro + +|.macro SET_Z_LVAL, zv, val +| mov aword [zv], val +|.endmacro + +|.macro GET_Z_PTR, reg, zv +| mov reg, aword [zv] +|.endmacro + +|.macro SET_Z_PTR, zv, val +| mov aword [zv], val +|.endmacro + +|.macro GET_Z_W2, reg, zv +| mov reg, dword [zv + 4] +|.endmacro + +|.macro SET_Z_W2, zv, val +| mov dword [zv + 4], val |.endmacro |.macro FPU_OP, fp_ins, op_type, op @@ -175,11 +204,11 @@ static void* dasm_labels[zend_lb_MAX]; ||} |.endmacro -|.macro FPU_STORE, dst_base, dst_offset -| fstp qword [dst_base + dst_offset] +|.macro FPU_STORE, zv +| fstp qword [zv] |.endmacro -|.macro SSE_OP, sse_ins, op_type, op, reg +|.macro SSE_OP, sse_ins, reg, op_type, op ||if (op_type == IS_CONST) { | .if X64 | mov r0, aword EX->literals @@ -192,7 +221,7 @@ static void* dasm_labels[zend_lb_MAX]; ||} |.endmacro -|.macro SSE_LOAD_LONG, op_type, op, reg +|.macro SSE_LOAD_LONG, reg, op_type, op ||if (op_type == IS_CONST) { | .if X64 | mov r0, aword EX->literals @@ -209,57 +238,57 @@ static void* dasm_labels[zend_lb_MAX]; ||} |.endmacro -|.macro SSE_LOAD, op_type, op, reg -| SSE_OP movsd, op_type, op, reg +|.macro SSE_LOAD, reg, op_type, op +| SSE_OP movsd, reg, op_type, op |.endmacro -|.macro SSE_MATH, opcode, op_type, op, reg +|.macro SSE_MATH, opcode, reg, op_type, op ||switch (opcode) { || case ZEND_ADD: -| SSE_OP addsd, op_type, op, reg +| SSE_OP addsd, reg, op_type, op || break; || case ZEND_SUB: -| SSE_OP subsd, op_type, op, reg +| SSE_OP subsd, reg, op_type, op || break; || case ZEND_MUL: -| SSE_OP mulsd, op_type, op, reg +| SSE_OP mulsd, reg, op_type, op || break; || case ZEND_DIV: -| SSE_OP divsd, op_type, op, reg +| SSE_OP divsd, reg, op_type, op || break; ||} |.endmacro -|.macro SSE_MATH2, opcode, reg1, reg2 +|.macro SSE_MATH2, opcode, dst_reg, src_reg ||switch (opcode) { || case ZEND_ADD: -| addsd reg2, reg1 +| addsd dst_reg, src_reg || break; || case ZEND_SUB: -| subsd reg2, reg1 +| subsd dst_reg, src_reg || break; || case ZEND_MUL: -| mulsd reg2, reg1 +| mulsd dst_reg, src_reg || break; || case ZEND_DIV: -| divsd reg2, reg1 +| divsd dst_reg, src_reg || break; ||} |.endmacro -|.macro SSE_STORE, dst_base, dst_offset, reg -| movsd qword [dst_base + dst_offset], reg +|.macro SSE_STORE, zv, reg +| movsd qword [zv], reg |.endmacro -|.macro DOUBLE_STORE, dst_base, dst_offset, reg +|.macro DOUBLE_STORE, zv, reg |.if X64 or SSE -| SSE_STORE dst_base, dst_offset, reg +| SSE_STORE zv, reg |.else -| FPU_STORE dst_base, dst_offset +| FPU_STORE zv |.endif |.endmacro -|.macro LONG_OP, long_ins, op_type, op, reg +|.macro LONG_OP, long_ins, reg, op_type, op ||if (op_type == IS_CONST) { | .if X64 || if (!IS_32BIT(Z_LVAL_P(RT_CONSTANT(op_array, op)))) { @@ -276,7 +305,7 @@ static void* dasm_labels[zend_lb_MAX]; ||} |.endmacro -|.macro LONG_LOAD, op_type, op, reg +|.macro LONG_LOAD, reg, op_type, op ||if (op_type == IS_CONST) { | .if X64 || if (!IS_32BIT(Z_LVAL_P(RT_CONSTANT(op_array, op)))) { @@ -288,20 +317,20 @@ static void* dasm_labels[zend_lb_MAX]; | mov reg, Z_LVAL_P(RT_CONSTANT(op_array, op)) | .endif ||} else { -| mov reg, aword [FP + op.var] +| GET_Z_LVAL reg, FP + op.var ||} |.endmacro -|.macro LONG_MATH, opcode, op_type, op, reg +|.macro LONG_MATH, opcode, reg, op_type, op ||switch (opcode) { || case ZEND_ADD: -| LONG_OP add, op_type, op, reg +| LONG_OP add, reg, op_type, op || break; || case ZEND_SUB: -| LONG_OP sub, op_type, op, reg +| LONG_OP sub, reg, op_type, op || break; || case ZEND_MUL: -| LONG_OP imul, op_type, op, reg +| LONG_OP imul, reg, op_type, op || break; || case ZEND_DIV: | idiv aword [FP + op.var] // (reg == r0) @@ -309,19 +338,19 @@ static void* dasm_labels[zend_lb_MAX]; ||} |.endmacro -|.macro LONG_MATH2, opcode, reg1, reg2 +|.macro LONG_MATH2, opcode, dst_reg, src_reg ||switch (opcode) { || case ZEND_ADD: -| add reg2, reg1 +| add dst_reg, src_reg || break; || case ZEND_SUB: -| sub reg2, reg1 +| sub dst_reg, src_reg || break; || case ZEND_MUL: -| imul reg2, reg1 +| imul dst_reg, src_reg || break; || case ZEND_DIV: -| idiv reg2 // (reg1 == r0) +| idiv src_reg // (reg1 == r0) || break; ||} |.endmacro @@ -360,11 +389,7 @@ static void* dasm_labels[zend_lb_MAX]; ||} |.endmacro -|.macro LONG_STORE, dst_base, dst_offset, reg -| mov aword [dst_base + dst_offset], reg -|.endmacro - -|.macro ZVAL_COPY_CONST, dst_base, dst_offset, dst_info, zv, tmp_reg +|.macro ZVAL_COPY_CONST, dst, dst_info, zv, tmp_reg ||if (Z_TYPE_P(zv) > IS_TRUE) { || if (Z_TYPE_P(zv) == IS_DOUBLE) { |.if X64 or SSE @@ -378,7 +403,7 @@ static void* dasm_labels[zend_lb_MAX]; || } else { | movsd xmm0, qword [((uint32_t)(uintptr_t)zv)] || } -| movsd qword [dst_base + dst_offset], xmm0 +| movsd qword [dst], xmm0 |.else || if (Z_DVAL_P(zv) == 0.0 && !is_signed(Z_DVAL_P(zv))) { | fldz @@ -387,61 +412,131 @@ static void* dasm_labels[zend_lb_MAX]; || } else { | fld qword [zv] || } -| fstp qword [dst_base + dst_offset] +| fstp qword [dst] |.endif || } else { |.if X64 || if (!IS_32BIT(Z_LVAL_P(zv))) { | mov64 tmp_reg, ((uintptr_t)Z_LVAL_P(zv)) -| mov qword [dst_base + dst_offset], tmp_reg +| SET_Z_LVAL dst, tmp_reg || } else { -| mov qword [dst_base + dst_offset], Z_LVAL_P(zv) +| SET_Z_LVAL dst, Z_LVAL_P(zv) || } |.else -| mov dword [dst_base + dst_offset], Z_LVAL_P(zv) +| SET_Z_LVAL dst, Z_LVAL_P(zv) |.endif || } ||} ||if ((dst_info & MAY_BE_ANY) != (1<1 +| IF_NOT_REFCOUNTED type_flags_reg, >1 || } -| inc dword [value_ptr_reg] +| GC_ADDREF value_ptr_reg |1: ||} |.endmacro @@ -483,11 +577,9 @@ static void* dasm_labels[zend_lb_MAX]; |.macro ZVAL_COPY_CTOR, val_info, type_flags_reg, value_ptr_reg, filename, lineno ||if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { || if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY)) { -| test type_flags_reg, IS_TYPE_REFCOUNTED + IS_TYPE_IMMUTABLE -| jz >2 -| test type_flags_reg, IS_TYPE_COPYABLE + IS_TYPE_IMMUTABLE -| jz >1 -| inc dword [value_ptr_reg] +| IF_NOT_FLAGS type_flags_reg, IS_TYPE_REFCOUNTED + IS_TYPE_IMMUTABLE, >2 +| IF_NOT_FLAGS type_flags_reg, IS_TYPE_COPYABLE + IS_TYPE_IMMUTABLE, >1 +| GC_ADDREF value_ptr_reg | jmp >2 |1: | ZVAL_COPY_CTOR_FUNC filename, lineno @@ -500,19 +592,13 @@ static void* dasm_labels[zend_lb_MAX]; |.macro ZVAL_DEREF, reg, info || if (info & MAY_BE_REF) { -| cmp byte [reg + 8], IS_REFERENCE -| jne >1 -| mov reg, [reg] -| add reg, 8 +| IF_NOT_Z_TYPE, reg, IS_REFERENCE, >1 +| GET_Z_PTR reg, reg +| add reg, offsetof(zend_reference, val) |1: || } |.endmacro -|.macro JNE_SLOW -| jne >9 -|| has_slow = 1; -|.endmacro - // zval should be in FCARG1a |.macro ZVAL_DTOR_FUNC, filename, lineno // arg1 must be in FCARG1a || if (ZEND_DEBUG) { @@ -526,35 +612,33 @@ static void* dasm_labels[zend_lb_MAX]; | EXT_CALL _zval_dtor_func, r0 |.endmacro -|.macro ZVAL_PTR_DTOR_NOGC, base, offset, filename, lineno -| test byte [base + offset + 9], IS_TYPE_REFCOUNTED -| jz >1 -| mov FCARG1a, aword [base + offset] -| dec dword [FCARG1a] +|.macro ZVAL_PTR_DTOR_NOGC, zv, filename, lineno +| IF_NOT_Z_REFCOUNTED zv, >1 +| GET_Z_PTR FCARG1a, zv +| GC_DELREF FCARG1a | jnz >1 | ZVAL_DTOR_FUNC filename, lineno |1: |.endmacro -|.macro ZVAL_PTR_DTOR, base, offset, info, cold, filename, lineno +|.macro ZVAL_PTR_DTOR, zv, info, cold, filename, lineno || if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { || if (info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { | // if (Z_REFCOUNTED_P(cv)) { -| test byte [base + offset + 9], IS_TYPE_REFCOUNTED || if (cold) { -| jnz >1 +| IF_Z_REFCOUNTED zv, >1 |.cold_code |1: || } else { -| jz >3 +| IF_NOT_Z_REFCOUNTED zv, >3 || } || } | // if (!Z_DELREF_P(cv)) { -| mov FCARG1a, aword [base + offset] -| dec dword [FCARG1a] +| GET_Z_PTR FCARG1a, zv +| GC_DELREF FCARG1a | jnz >1 | // ZVAL_NULL(cv); -| mov dword [base + offset + 8], IS_NULL +| SET_Z_TYPE_INFO zv, IS_NULL | // zval_dtor_func(r); | ZVAL_DTOR_FUNC filename, lineno || if (info & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) { @@ -563,24 +647,20 @@ static void* dasm_labels[zend_lb_MAX]; |1: || if (info & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) { || if (info & MAY_BE_REF) { -| lea r1, [base + offset] +| lea r1, [zv] | // ZVAL_DEREF(z); | ZVAL_DEREF r1, info | // if (Z_COLLECTABLE_P(z)) -| test byte [r1 + 9], IS_TYPE_COLLECTABLE -| je >3 +| IF_NOT_Z_FLAGS r1, IS_TYPE_COLLECTABLE, >3 | // if (UNEXPECTED(!Z_GC_INFO_P(z))) -| mov FCARG1a, aword [r1] -| cmp word [FCARG1a + 6], 0 -| jne >3 +| GET_Z_PTR FCARG1a, r1 +| IF_GC_INFO FCARG1a, >3 || } else { | // if (Z_COLLECTABLE_P(z)) -| test byte [base + offset + 9], IS_TYPE_COLLECTABLE -| je >3 +| IF_NOT_Z_FLAGS zv, IS_TYPE_COLLECTABLE, >3 | // if (UNEXPECTED(!Z_GC_INFO_P(z))) -| mov FCARG1a, aword [base + offset] -| cmp word [FCARG1a + 6], 0 -| jne >3 +| GET_Z_PTR FCARG1a, zv +| IF_GC_INFO FCARG1a, >3 || } | // gc_possible_root(Z_COUNTED_P(z)) | EXT_CALL gc_possible_root, r0 @@ -599,7 +679,7 @@ static void* dasm_labels[zend_lb_MAX]; |.macro FREE_OP, op_type, op, op_info, op_array, lineno ||if ((op_type & (IS_VAR|IS_TMP_VAR)) && || (op_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { -| ZVAL_PTR_DTOR_NOGC FP, op.var, (op_array->filename ? ZSTR_VAL(op_array->filename) : NULL), lineno +| ZVAL_PTR_DTOR_NOGC FP + op.var, (op_array->filename ? ZSTR_VAL(op_array->filename) : NULL), lineno ||} |.endmacro @@ -622,8 +702,7 @@ static void* dasm_labels[zend_lb_MAX]; |.endmacro |.macro OBJ_RELEASE, reg, exit_label -| // if (--GC_REFCOUNT(obj) == 0) { -| dec dword [reg] +| GC_DELREF reg | jne >1 | // zend_objects_store_del(obj); |.if X64 @@ -635,11 +714,10 @@ static void* dasm_labels[zend_lb_MAX]; | EXT_CALL zend_objects_store_del, r0 | add r4, 16 |.endif -| jmp >exit_label +| jmp exit_label |1: | // if (UNEXPECTED(!GC_INFO(obj))) { -| cmp word [reg + 6], 0 -| jne >exit_label +| IF_GC_INFO reg, exit_label | // gc_possible_root(obj) | mov FCARG1a, reg | EXT_CALL gc_possible_root, r0 @@ -792,7 +870,7 @@ static int zend_jit_leave_throw_stub(dasm_State **Dst) | mov r0, OP:IP->result.var |.endif | add r0, FP - | ZVAL_PTR_DTOR r0, 0, MAY_BE_ANY, 0, NULL, 0 + | ZVAL_PTR_DTOR r0, MAY_BE_ANY, 0, NULL, 0 |5: | // EX(opline) = EG(exception_op); | LOAD_ADDR IP, &EG(exception_op) @@ -1006,12 +1084,11 @@ static int zend_jit_inc_dec(dasm_State **Dst, zend_op *opline, zend_op_array *op } if (op1_info & (MAY_BE_ANY-MAY_BE_LONG)) { - | cmp dword [FP + opline->op1.var + 8], IS_LONG - | jne >2 + | IF_NOT_Z_TYPE FP + opline->op1.var, IS_LONG, >2 } || if ((opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) && || opline->result_type != IS_UNUSED) { - | ZVAL_COPY_VALUE FP, opline->result.var, FP, opline->op1.var, MAY_BE_LONG, r0, eax, r1 + | ZVAL_COPY_VALUE FP + opline->result.var, FP + opline->op1.var, MAY_BE_LONG, r0, eax, r1 || } if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { | inc aword [FP + opline->op1.var] @@ -1023,50 +1100,49 @@ static int zend_jit_inc_dec(dasm_State **Dst, zend_op *opline, zend_op_array *op | jo >1 || if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && || opline->result_type != IS_UNUSED) { - | ZVAL_COPY_VALUE FP, opline->result.var, FP, opline->op1.var, MAY_BE_LONG, r0, eax, r1 + | ZVAL_COPY_VALUE FP + opline->result.var, FP + opline->op1.var, MAY_BE_LONG, r0, eax, r1 || } |.cold_code |1: if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { |.if X64 | mov64 rax, 0x43e0000000000000 - | mov qword [FP + opline->op1.var], rax + | SET_Z_LVAL FP + opline->op1.var, rax |.else - | mov dword [FP + opline->op1.var], 0 - | mov dword [FP + opline->op1.var + 4], 0x41e00000 + | SET_Z_LVAL FP + opline->op1.var, 0 + | SET_Z_W2 FP + opline->op1.var, 0x41e00000 |.endif } else { |.if X64 | mov64 rax, 0xc3e0000000000000 - | mov qword [FP + opline->op1.var], rax + | SET_Z_LVAL FP + opline->op1.var, rax |.else - | mov dword [FP + opline->op1.var], 0x00200000 - | mov dword [FP + opline->op1.var + 4], 0xc1e00000 + | SET_Z_LVAL FP + opline->op1.var, 0x00200000 + | SET_Z_W2 FP + opline->op1.var, 0xc1e00000 |.endif } - | SET_TYPE_INFO FP, opline->op1.var, IS_DOUBLE + | SET_Z_TYPE_INFO FP + opline->op1.var, IS_DOUBLE || if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && || opline->result_type != IS_UNUSED) { - | ZVAL_COPY_VALUE FP, opline->result.var, FP, opline->op1.var, MAY_BE_DOUBLE, r0, eax, r1 + | ZVAL_COPY_VALUE FP + opline->result.var, FP + opline->op1.var, MAY_BE_DOUBLE, r0, eax, r1 || } | jmp >3 |.code } else { || if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && || opline->result_type != IS_UNUSED) { - | ZVAL_COPY_VALUE FP, opline->result.var, FP, opline->op1.var, MAY_BE_LONG, r0, eax, r1 + | ZVAL_COPY_VALUE FP + opline->result.var, FP + opline->op1.var, MAY_BE_LONG, r0, eax, r1 || } } if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { |.cold_code |2: || if (op1_info & MAY_BE_UNDEF) { - | cmp byte [FP + opline->op1.var + 8], IS_UNDEF - | jne >2 + | IF_NOT_Z_TYPE FP + opline->op1.var, IS_UNDEF, >2 | // zend_error(E_NOTICE, "Undefined variable: %s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); | mov FCARG1d, opline->op1.var | EXT_CALL zend_jit_undefined_op_helper, r0 - | mov dword [FP + opline->op1.var + 8], IS_NULL + | SET_Z_TYPE_INFO FP + opline->op1.var, IS_NULL || } |2: | lea FCARG1a, [FP + opline->op1.var] @@ -1074,15 +1150,13 @@ static int zend_jit_inc_dec(dasm_State **Dst, zend_op *opline, zend_op_array *op | ZVAL_DEREF FCARG1a, op1_info || if ((opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC)) { || if (opline->result_type != IS_UNUSED) { - | ZVAL_COPY_VALUE FP, opline->result.var, FCARG1a, 0, op1_info, r0, eax, r2 + | ZVAL_COPY_VALUE FP + opline->result.var, FCARG1a, op1_info, r0, eax, r2 | //ZVAL_COPY_CTOR op1_info, ah, r2, (op_array->filename ? ZSTR_VAL(op_array->filename) : NULL), opline->lineno || if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { || if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY)) { - | test ah, IS_TYPE_REFCOUNTED + IS_TYPE_IMMUTABLE - | jz >2 - | test ah, IS_TYPE_COPYABLE + IS_TYPE_IMMUTABLE - | jnz >1 - | inc dword [r2] + | IF_NOT_FLAGS ah, IS_TYPE_REFCOUNTED + IS_TYPE_IMMUTABLE, >2 + | IF_FLAGS ah, IS_TYPE_COPYABLE + IS_TYPE_IMMUTABLE, >1 + | GC_ADDREF r2 | jmp >2 |1: | mov aword [r4], FCARG1a // save @@ -1097,14 +1171,12 @@ static int zend_jit_inc_dec(dasm_State **Dst, zend_op *opline, zend_op_array *op || } else { || if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY)) { | // SEPARATE_ZVAL_NOREF(var_ptr); - | test byte [FCARG1a + 9], IS_TYPE_COPYABLE + IS_TYPE_IMMUTABLE - | jz >2 - | mov r0, [FCARG1a] - | cmp dword [r0], 1 + | IF_NOT_Z_FLAGS FCARG1a, IS_TYPE_COPYABLE + IS_TYPE_IMMUTABLE, >2 + | GET_Z_PTR r0, FCARG1a + | cmp dword [r0], 1 // if (GC_REFCOUNTED() > 1) | jbe >2 - | test byte [FCARG1a + 9], IS_TYPE_IMMUTABLE - | jne >1 - | dec dword [r0] + | IF_Z_FLAGS FCARG1a, IS_TYPE_IMMUTABLE, >1 + | GC_DELREF r0 |1: | mov aword [r4], FCARG1a // save | ZVAL_COPY_CTOR_FUNC (op_array->filename ? ZSTR_VAL(op_array->filename) : NULL), opline->lineno @@ -1119,7 +1191,7 @@ static int zend_jit_inc_dec(dasm_State **Dst, zend_op *opline, zend_op_array *op || } || if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && || opline->result_type != IS_UNUSED) { - | ZVAL_COPY_VALUE FP, opline->result.var, FP, opline->op1.var, op1_def_info, r0, eax, r1 + | ZVAL_COPY_VALUE FP + opline->result.var, FP + opline->op1.var, op1_def_info, r0, eax, r1 | TRY_ADDREF op1_def_info, ah, r1 || } | jmp >3 @@ -1147,61 +1219,61 @@ static int zend_jit_math_long_long(dasm_State **Dst, zend_op *opline, zend_op_ar IS_32BIT(Z_LVAL_P(RT_CONSTANT(op_array, opline->op1))) && is_power_of_two(Z_LVAL_P(RT_CONSTANT(op_array, opline->op1)))))) { if (opline->op2_type == IS_CONST) { - | LONG_LOAD opline->op1_type, opline->op1, r0 + | LONG_LOAD r0, opline->op1_type, opline->op1 | shl r0, floor_log2(Z_LVAL_P(RT_CONSTANT(op_array, opline->op2))) } else { - | LONG_LOAD opline->op2_type, opline->op2, r0 + | LONG_LOAD r0, opline->op2_type, opline->op2 | shl r0, floor_log2(Z_LVAL_P(RT_CONSTANT(op_array, opline->op1))) } } else if (opline->opcode == ZEND_DIV && (opline->op2_type == IS_CONST && is_power_of_two(Z_LVAL_P(RT_CONSTANT(op_array, opline->op2))))) { - | LONG_LOAD opline->op1_type, opline->op1, r0 + | LONG_LOAD r0, opline->op1_type, opline->op1 | shr r0, floor_log2(Z_LVAL_P(RT_CONSTANT(op_array, opline->op2))) } else { - | LONG_LOAD opline->op1_type, opline->op1, r0 + | LONG_LOAD r0, opline->op1_type, opline->op1 if (same_ops && opline->opcode != ZEND_DIV) { | LONG_MATH2 opline->opcode, r0, r0 } else { - | LONG_MATH opline->opcode, opline->op2_type, opline->op2, r0 + | LONG_MATH opline->opcode, r0, opline->op2_type, opline->op2 } } if ((res_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) { if (res_forward) { - | LONG_STORE r1, res_forward, r0 - | SET_TYPE_INFO r1, res_forward, IS_LONG + | SET_Z_LVAL r1 + res_forward, r0 + | SET_Z_TYPE_INFO r1 + res_forward, IS_LONG } else { - | LONG_STORE FP, opline->result.var, r0 - | SET_TYPE_INFO FP, opline->result.var, IS_LONG + | SET_Z_LVAL FP + opline->result.var, r0 + | SET_Z_TYPE_INFO FP + opline->result.var, IS_LONG } } else { | jo >1 |.cold_code |1: |.if X64 or SSE - | SSE_LOAD_LONG opline->op1_type, opline->op1, xmm0 - | SSE_LOAD_LONG opline->op2_type, opline->op2, xmm1 - | SSE_MATH2 opline->opcode, xmm1, xmm0 + | SSE_LOAD_LONG xmm0, opline->op1_type, opline->op1 + | SSE_LOAD_LONG xmm1, opline->op2_type, opline->op2 + | SSE_MATH2 opline->opcode, xmm0, xmm1 |.else | FPU_LONG_LOAD opline->op2_type, opline->op2 | FPU_LONG_LOAD opline->op1_type, opline->op1 | FPU_MATH2 opline->opcode |.endif if (res_forward) { - | DOUBLE_STORE r1, res_forward, xmm0 - | SET_TYPE_INFO r1, res_forward, IS_DOUBLE + | DOUBLE_STORE r1 + res_forward, xmm0 + | SET_Z_TYPE_INFO r1 + res_forward, IS_DOUBLE } else { - | DOUBLE_STORE FP, opline->result.var, xmm0 - | SET_TYPE_INFO FP, opline->result.var, IS_DOUBLE + | DOUBLE_STORE FP + opline->result.var, xmm0 + | SET_Z_TYPE_INFO FP + opline->result.var, IS_DOUBLE } | jmp >2 |.code if (res_forward) { - | LONG_STORE r1, res_forward, r0 - | SET_TYPE_INFO r1, res_forward, IS_LONG + | SET_Z_LVAL r1 + res_forward, r0 + | SET_Z_TYPE_INFO r1 + res_forward, IS_LONG } else { - | LONG_STORE FP, opline->result.var, r0 - | SET_TYPE_INFO FP, opline->result.var, IS_LONG + | SET_Z_LVAL FP + opline->result.var, r0 + | SET_Z_TYPE_INFO FP + opline->result.var, IS_LONG } |2: } @@ -1212,18 +1284,18 @@ static int zend_jit_math_long_long(dasm_State **Dst, zend_op *opline, zend_op_ar static int zend_jit_math_long_double(dasm_State **Dst, zend_op *opline, uint32_t res_forward) { |.if X64 or SSE - | SSE_LOAD_LONG opline->op1_type, opline->op1, xmm0 - | SSE_MATH opline->opcode, opline->op2_type, opline->op2, xmm0 + | SSE_LOAD_LONG xmm0, opline->op1_type, opline->op1 + | SSE_MATH opline->opcode, xmm0, opline->op2_type, opline->op2 |.else | FPU_LONG_LOAD opline->op1_type, opline->op1 | FPU_MATH opline->opcode, opline->op2_type, opline->op2 |.endif if (res_forward) { - | DOUBLE_STORE r1, res_forward, xmm0 - | SET_TYPE_INFO r1, res_forward, IS_DOUBLE + | DOUBLE_STORE r1 + res_forward, xmm0 + | SET_Z_TYPE_INFO r1 + res_forward, IS_DOUBLE } else { - | DOUBLE_STORE FP, opline->result.var, xmm0 - | SET_TYPE_INFO FP, opline->result.var, IS_DOUBLE + | DOUBLE_STORE FP + opline->result.var, xmm0 + | SET_Z_TYPE_INFO FP + opline->result.var, IS_DOUBLE } return 1; @@ -1233,12 +1305,12 @@ static int zend_jit_math_double_long(dasm_State **Dst, zend_op *opline, uint32_t { |.if X64 or SSE || if (opline->opcode == ZEND_ADD || opline->opcode == ZEND_MUL) { - | SSE_LOAD_LONG opline->op2_type, opline->op2, xmm0 - | SSE_MATH opline->opcode, opline->op1_type, opline->op1, xmm0 + | SSE_LOAD_LONG xmm0, opline->op2_type, opline->op2 + | SSE_MATH opline->opcode, xmm0, opline->op1_type, opline->op1 || } else { - | SSE_LOAD opline->op1_type, opline->op1, xmm0 - | SSE_LOAD_LONG opline->op2_type, opline->op2, xmm1 - | SSE_MATH2 opline->opcode, xmm1, xmm0 + | SSE_LOAD xmm0, opline->op1_type, opline->op1 + | SSE_LOAD_LONG xmm1, opline->op2_type, opline->op2 + | SSE_MATH2 opline->opcode, xmm0, xmm1 || } |.else | FPU_LONG_LOAD opline->op2_type, opline->op2 @@ -1250,11 +1322,11 @@ static int zend_jit_math_double_long(dasm_State **Dst, zend_op *opline, uint32_t || } |.endif if (res_forward) { - | DOUBLE_STORE r1, res_forward, xmm0 - | SET_TYPE_INFO r1, res_forward, IS_DOUBLE + | DOUBLE_STORE r1 + res_forward, xmm0 + | SET_Z_TYPE_INFO r1 + res_forward, IS_DOUBLE } else { - | DOUBLE_STORE FP, opline->result.var, xmm0 - | SET_TYPE_INFO FP, opline->result.var, IS_DOUBLE + | DOUBLE_STORE FP + opline->result.var, xmm0 + | SET_Z_TYPE_INFO FP + opline->result.var, IS_DOUBLE } return 1; @@ -1265,22 +1337,22 @@ static int zend_jit_math_double_double(dasm_State **Dst, zend_op *opline, uint32 zend_bool same_ops = (opline->op1_type == opline->op2_type) && (opline->op1.var == opline->op2.var); |.if X64 or SSE - | SSE_LOAD opline->op1_type, opline->op1, xmm0 + | SSE_LOAD xmm0, opline->op1_type, opline->op1 if (same_ops) { | SSE_MATH2 opline->opcode, xmm0, xmm0 } else { - | SSE_MATH opline->opcode, opline->op2_type, opline->op2, xmm0 + | SSE_MATH opline->opcode, xmm0, opline->op2_type, opline->op2 } |.else | FPU_LOAD opline->op1_type, opline->op1 | FPU_MATH opline->opcode, opline->op2_type, opline->op2 |.endif if (res_forward) { - | DOUBLE_STORE r1, res_forward, xmm0 - | SET_TYPE_INFO r1, res_forward, IS_DOUBLE + | DOUBLE_STORE r1 + res_forward, xmm0 + | SET_Z_TYPE_INFO r1 + res_forward, IS_DOUBLE } else { - | DOUBLE_STORE FP, opline->result.var, xmm0 - | SET_TYPE_INFO FP, opline->result.var, IS_DOUBLE + | DOUBLE_STORE FP + opline->result.var, xmm0 + | SET_Z_TYPE_INFO FP + opline->result.var, IS_DOUBLE } return 1; @@ -1291,7 +1363,7 @@ static int zend_jit_shift(dasm_State **Dst, zend_op *opline, zend_op_array *op_a { zval *op2; uint32_t op1_info; - zend_bool has_slow = 0; + zend_bool has_slow; if (!ssa->ops || !ssa->var_info) { goto fallback; @@ -1306,20 +1378,21 @@ static int zend_jit_shift(dasm_State **Dst, zend_op *opline, zend_op_array *op_a goto fallback; } + has_slow = (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_LONG)) != 0; + op2 = RT_CONSTANT(op_array, opline->op2); if (op1_info & (MAY_BE_ANY-MAY_BE_LONG)) { - | cmp dword [FP + opline->op1.var + 8], IS_LONG - | JNE_SLOW + | IF_NOT_Z_TYPE FP + opline->op1.var, IS_LONG, >9 } - | LONG_LOAD opline->op1_type, opline->op1, r0 + | LONG_LOAD r0, opline->op1_type, opline->op1 if (opline->opcode == ZEND_SR) { | shr, r0, Z_LVAL_P(op2) } else { | shl, r0, Z_LVAL_P(op2) } - | LONG_STORE FP, opline->result.var, r0 - | SET_TYPE_INFO FP, opline->result.var, IS_LONG + | SET_Z_LVAL FP + opline->result.var, r0 + | SET_Z_TYPE_INFO FP + opline->result.var, IS_LONG if (has_slow) { |2: } @@ -1359,7 +1432,7 @@ static int zend_jit_math(dasm_State **Dst, zend_op *opline, int *opnum, zend_op_ { uint32_t op1_info, op2_info; zend_bool same_ops = (opline->op1_type == opline->op2_type) && (opline->op1.var == opline->op2.var); - zend_bool has_slow = 0; + zend_bool has_slow; uint32_t res_forward = 0; if (!ssa->ops || !ssa->var_info) { @@ -1378,6 +1451,12 @@ static int zend_jit_math(dasm_State **Dst, zend_op *opline, int *opnum, zend_op_ goto fallback; } + has_slow = + (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && + (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && + ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) || + (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))); + if (opline->result_type == IS_TMP_VAR && (opline+1)->opcode == ZEND_SEND_VAL && (opline+1)->op1_type == IS_TMP_VAR && @@ -1390,22 +1469,19 @@ static int zend_jit_math(dasm_State **Dst, zend_op *opline, int *opnum, zend_op_ if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)) { if (op1_info & (MAY_BE_ANY-MAY_BE_LONG)) { - | cmp dword [FP + opline->op1.var + 8], IS_LONG if (op1_info & MAY_BE_DOUBLE) { - | jne >4 + | IF_NOT_Z_TYPE FP + opline->op1.var, IS_LONG, >4 } else { - | JNE_SLOW + | IF_NOT_Z_TYPE FP + opline->op1.var, IS_LONG, >9 } } if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_LONG))) { - | cmp dword [FP + opline->op2.var + 8], IS_LONG if (op2_info & MAY_BE_DOUBLE) { - | jne >3 + | IF_NOT_Z_TYPE FP + opline->op2.var, IS_LONG, >3 |.cold_code |3: if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { - | cmp dword [FP + opline->op2.var + 8], IS_DOUBLE - | JNE_SLOW + | IF_NOT_Z_TYPE FP + opline->op2.var, IS_DOUBLE, >9 } if (!zend_jit_math_long_double(Dst, opline, res_forward)) { return 0; @@ -1413,7 +1489,7 @@ static int zend_jit_math(dasm_State **Dst, zend_op *opline, int *opnum, zend_op_ | jmp >6 |.code } else { - | JNE_SLOW + | IF_NOT_Z_TYPE FP + opline->op2.var, IS_LONG, >9 } } if (!zend_jit_math_long_long(Dst, opline, op_array, ssa, res_forward)) { @@ -1423,16 +1499,14 @@ static int zend_jit_math(dasm_State **Dst, zend_op *opline, int *opnum, zend_op_ |.cold_code |4: if (op1_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { - | cmp dword [FP + opline->op1.var + 8], IS_DOUBLE - | JNE_SLOW + | IF_NOT_Z_TYPE FP + opline->op1.var, IS_DOUBLE, >9 } if (op2_info & MAY_BE_DOUBLE) { if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { - | cmp dword [FP + opline->op2.var + 8], IS_DOUBLE if (!same_ops) { - | jne >5 + | IF_NOT_Z_TYPE, FP + opline->op2.var, IS_DOUBLE, >5 } else { - | JNE_SLOW + | IF_NOT_Z_TYPE, FP + opline->op2.var, IS_DOUBLE, >9 } } if (!zend_jit_math_double_double(Dst, opline, res_forward)) { @@ -1443,8 +1517,7 @@ static int zend_jit_math(dasm_State **Dst, zend_op *opline, int *opnum, zend_op_ if (!same_ops) { |5: if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { - | cmp dword [FP + opline->op2.var + 8], IS_LONG - | JNE_SLOW + | IF_NOT_Z_TYPE FP + opline->op2.var, IS_LONG, >9 } if (!zend_jit_math_double_long(Dst, opline, res_forward)) { return 0; @@ -1457,16 +1530,14 @@ static int zend_jit_math(dasm_State **Dst, zend_op *opline, int *opnum, zend_op_ !(op1_info & MAY_BE_LONG) && (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { if (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) { - | cmp dword [FP + opline->op1.var + 8], IS_DOUBLE - | JNE_SLOW + | IF_NOT_Z_TYPE FP + opline->op1.var, IS_DOUBLE, >9 } if (op2_info & MAY_BE_DOUBLE) { if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { - | cmp dword [FP + opline->op2.var + 8], IS_DOUBLE if (!same_ops && (op2_info & MAY_BE_LONG)) { - | jne >3 + | IF_NOT_Z_TYPE FP + opline->op2.var, IS_DOUBLE, >3 } else { - | JNE_SLOW + | IF_NOT_Z_TYPE FP + opline->op2.var, IS_DOUBLE, >9 } } if (!zend_jit_math_double_double(Dst, opline, res_forward)) { @@ -1479,8 +1550,7 @@ static int zend_jit_math(dasm_State **Dst, zend_op *opline, int *opnum, zend_op_ } |3: if (op2_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) { - | cmp dword [FP + opline->op2.var + 8], IS_LONG - | JNE_SLOW + | IF_NOT_Z_TYPE FP + opline->op2.var, IS_LONG, >9 } if (!zend_jit_math_double_long(Dst, opline, res_forward)) { return 0; @@ -1494,16 +1564,14 @@ static int zend_jit_math(dasm_State **Dst, zend_op *opline, int *opnum, zend_op_ !(op2_info & MAY_BE_LONG) && (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { if (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) { - | cmp dword [FP + opline->op2.var + 8], IS_DOUBLE - | JNE_SLOW + | IF_NOT_Z_TYPE FP + opline->op2.var, IS_DOUBLE, >9 } if (op1_info & MAY_BE_DOUBLE) { if (!same_ops && (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { - | cmp dword [FP + opline->op1.var + 8], IS_DOUBLE if (!same_ops && (op1_info & MAY_BE_LONG)) { - | jne >3 + | IF_NOT_Z_TYPE FP + opline->op1.var, IS_DOUBLE, >3 } else { - | JNE_SLOW + | IF_NOT_Z_TYPE FP + opline->op1.var, IS_DOUBLE, >9 } } if (!zend_jit_math_double_double(Dst, opline, res_forward)) { @@ -1516,8 +1584,7 @@ static int zend_jit_math(dasm_State **Dst, zend_op *opline, int *opnum, zend_op_ } |3: if (op1_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) { - | cmp dword [FP + opline->op1.var + 8], IS_LONG - | JNE_SLOW + | IF_NOT_Z_TYPE FP + opline->op1.var, IS_LONG, >9 } if (!zend_jit_math_long_double(Dst, opline, res_forward)) { return 0; @@ -1582,8 +1649,8 @@ static int zend_jit_cmp_long_long(dasm_State **Dst, zend_op *opline, int b, zend { unsigned int target_label; - | LONG_LOAD opline->op1_type, opline->op1, r0 - | LONG_OP cmp, opline->op2_type, opline->op2, r0 + | LONG_LOAD r0, opline->op1_type, opline->op1 + | LONG_OP cmp, r0, opline->op2_type, opline->op2 if ((opline+1)->opcode == ZEND_JMPZ && (opline+1)->op1_type == IS_TMP_VAR && (opline+1)->op1.var == opline->result.var) { @@ -1646,7 +1713,7 @@ static int zend_jit_cmp_long_long(dasm_State **Dst, zend_op *opline, int b, zend | add eax, 2 break; } - | SET_TYPE_INFO FP, opline->result.var, eax + | SET_Z_TYPE_INFO FP + opline->result.var, eax } return 1; @@ -1732,7 +1799,7 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, zend_op *opline, int b, | add eax, 2 break; } - | SET_TYPE_INFO FP, opline->result.var, eax + | SET_Z_TYPE_INFO FP + opline->result.var, eax } return 1; @@ -1741,8 +1808,8 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, zend_op *opline, int b, static int zend_jit_cmp_long_double(dasm_State **Dst, zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa) { |.if X64 or SSE - | SSE_LOAD_LONG opline->op1_type, opline->op1, xmm0 - | SSE_OP ucomisd, opline->op2_type, opline->op2, xmm0 + | SSE_LOAD_LONG xmm0, opline->op1_type, opline->op1 + | SSE_OP ucomisd, xmm0, opline->op2_type, opline->op2 |.else | FPU_LOAD opline->op2_type, opline->op2 | FPU_LONG_LOAD opline->op1_type, opline->op1 @@ -1756,8 +1823,8 @@ static int zend_jit_cmp_long_double(dasm_State **Dst, zend_op *opline, int b, ze static int zend_jit_cmp_double_long(dasm_State **Dst, zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa) { |.if X64 or SSE - | SSE_LOAD opline->op1_type, opline->op1, xmm0 - | SSE_LOAD_LONG opline->op2_type, opline->op2, xmm1 + | SSE_LOAD xmm0, opline->op1_type, opline->op1 + | SSE_LOAD_LONG xmm1, opline->op2_type, opline->op2 | ucomisd xmm0, xmm1 |.else | FPU_LONG_LOAD opline->op2_type, opline->op2 @@ -1772,8 +1839,8 @@ static int zend_jit_cmp_double_long(dasm_State **Dst, zend_op *opline, int b, ze static int zend_jit_cmp_double_double(dasm_State **Dst, zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa) { |.if X64 or SSE - | SSE_LOAD opline->op1_type, opline->op1, xmm0 - | SSE_OP ucomisd, opline->op2_type, opline->op2, xmm0 + | SSE_LOAD xmm0, opline->op1_type, opline->op1 + | SSE_OP ucomisd, xmm0, opline->op2_type, opline->op2 |.else | FPU_LOAD opline->op2_type, opline->op2 | FPU_LOAD opline->op1_type, opline->op1 @@ -1851,7 +1918,7 @@ static int zend_jit_cmp_slow(dasm_State **Dst, zend_op *opline, int b, zend_op_a | add eax, 2 break; } - | SET_TYPE_INFO FP, opline->result.var, eax + | SET_Z_TYPE_INFO FP + opline->result.var, eax } return 1; @@ -1861,7 +1928,7 @@ static int zend_jit_cmp(dasm_State **Dst, zend_op *opline, int b, int *opnum, ze { uint32_t op1_info, op2_info; zend_bool same_ops = (opline->op1_type == opline->op2_type) && (opline->op1.var == opline->op2.var); - zend_bool has_slow = 0; + zend_bool has_slow; if (!ssa->ops || !ssa->var_info) { goto fallback; @@ -1874,24 +1941,27 @@ static int zend_jit_cmp(dasm_State **Dst, zend_op *opline, int b, int *opnum, ze goto fallback; } + has_slow = + (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && + (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && + ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) || + (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))); + if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)) { if (op1_info & (MAY_BE_ANY-MAY_BE_LONG)) { - | cmp dword [FP + opline->op1.var + 8], IS_LONG if (op1_info & MAY_BE_DOUBLE) { - | jne >4 + | IF_NOT_Z_TYPE FP + opline->op1.var, IS_LONG, >4 } else { - | JNE_SLOW + | IF_NOT_Z_TYPE FP + opline->op1.var, IS_LONG, >9 } } if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_LONG))) { - | cmp dword [FP + opline->op2.var + 8], IS_LONG if (op2_info & MAY_BE_DOUBLE) { - | jne >3 + | IF_NOT_Z_TYPE FP + opline->op2.var, IS_LONG, >3 |.cold_code |3: if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { - | cmp dword [FP + opline->op2.var + 8], IS_DOUBLE - | JNE_SLOW + | IF_NOT_Z_TYPE FP + opline->op2.var, IS_DOUBLE, >9 } if (!zend_jit_cmp_long_double(Dst, opline, b, op_array, ssa)) { return 0; @@ -1899,7 +1969,7 @@ static int zend_jit_cmp(dasm_State **Dst, zend_op *opline, int b, int *opnum, ze | jmp >6 |.code } else { - | JNE_SLOW + | IF_NOT_Z_TYPE FP + opline->op2.var, IS_LONG, >9 } } if (!zend_jit_cmp_long_long(Dst, opline, b, op_array, ssa)) { @@ -1909,16 +1979,14 @@ static int zend_jit_cmp(dasm_State **Dst, zend_op *opline, int b, int *opnum, ze |.cold_code |4: if (op1_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { - | cmp dword [FP + opline->op1.var + 8], IS_DOUBLE - | JNE_SLOW + | IF_NOT_Z_TYPE FP + opline->op1.var, IS_DOUBLE, >9 } if (op2_info & MAY_BE_DOUBLE) { if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { - | cmp dword [FP + opline->op2.var + 8], IS_DOUBLE if (!same_ops) { - | jne >5 + | IF_NOT_Z_TYPE FP + opline->op2.var, IS_DOUBLE, >5 } else { - | JNE_SLOW + | IF_NOT_Z_TYPE FP + opline->op2.var, IS_DOUBLE, >9 } } if (!zend_jit_cmp_double_double(Dst, opline, b, op_array, ssa)) { @@ -1929,8 +1997,7 @@ static int zend_jit_cmp(dasm_State **Dst, zend_op *opline, int b, int *opnum, ze if (!same_ops) { |5: if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { - | cmp dword [FP + opline->op2.var + 8], IS_LONG - | JNE_SLOW + | IF_NOT_Z_TYPE FP + opline->op2.var, IS_LONG, >9 } if (!zend_jit_cmp_double_long(Dst, opline, b, op_array, ssa)) { return 0; @@ -1943,16 +2010,14 @@ static int zend_jit_cmp(dasm_State **Dst, zend_op *opline, int b, int *opnum, ze !(op1_info & MAY_BE_LONG) && (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { if (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) { - | cmp dword [FP + opline->op1.var + 8], IS_DOUBLE - | JNE_SLOW + | IF_NOT_Z_TYPE FP + opline->op1.var, IS_DOUBLE, >9 } if (op2_info & MAY_BE_DOUBLE) { if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { - | cmp dword [FP + opline->op2.var + 8], IS_DOUBLE if (!same_ops && (op2_info & MAY_BE_LONG)) { - | jne >3 + | IF_NOT_Z_TYPE FP + opline->op2.var, IS_DOUBLE, >3 } else { - | JNE_SLOW + | IF_NOT_Z_TYPE FP + opline->op2.var, IS_DOUBLE, >9 } } if (!zend_jit_cmp_double_double(Dst, opline, b, op_array, ssa)) { @@ -1965,8 +2030,7 @@ static int zend_jit_cmp(dasm_State **Dst, zend_op *opline, int b, int *opnum, ze } |3: if (op2_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) { - | cmp dword [FP + opline->op2.var + 8], IS_LONG - | JNE_SLOW + | IF_NOT_Z_TYPE FP + opline->op2.var, IS_LONG, >9 } if (!zend_jit_cmp_double_long(Dst, opline, b, op_array, ssa)) { return 0; @@ -1980,16 +2044,14 @@ static int zend_jit_cmp(dasm_State **Dst, zend_op *opline, int b, int *opnum, ze !(op2_info & MAY_BE_LONG) && (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { if (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) { - | cmp dword [FP + opline->op2.var + 8], IS_DOUBLE - | JNE_SLOW + | IF_NOT_Z_TYPE FP + opline->op2.var, IS_DOUBLE, >9 } if (op1_info & MAY_BE_DOUBLE) { if (!same_ops && (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { - | cmp dword [FP + opline->op1.var + 8], IS_DOUBLE if (!same_ops && (op1_info & MAY_BE_LONG)) { - | jne >3 + | IF_NOT_Z_TYPE FP + opline->op1.var, IS_DOUBLE, >3 } else { - | JNE_SLOW + | IF_NOT_Z_TYPE FP + opline->op1.var, IS_DOUBLE, >9 } } if (!zend_jit_cmp_double_double(Dst, opline, b, op_array, ssa)) { @@ -2002,8 +2064,7 @@ static int zend_jit_cmp(dasm_State **Dst, zend_op *opline, int b, int *opnum, ze } |3: if (op1_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) { - | cmp dword [FP + opline->op1.var + 8], IS_LONG - | JNE_SLOW + | IF_NOT_Z_TYPE FP + opline->op1.var, IS_LONG, >9 } if (!zend_jit_cmp_long_double(Dst, opline, b, op_array, ssa)) { return 0; @@ -2016,16 +2077,15 @@ static int zend_jit_cmp(dasm_State **Dst, zend_op *opline, int b, int *opnum, ze } if (has_slow || - !(op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) || - !(op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { + (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) || + (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { if (has_slow) { |.cold_code |9: } | mov EX->opline, IP if (opline->op1_type == IS_CV && (op1_info & MAY_BE_UNDEF)) { - | cmp dword [FP + opline->op1.var + 8], 0 - | jnz >1 + | IF_NOT_Z_TYPE FP + opline->op1.var, IS_UNDEF, >1 |1: } | lea FCARG1a, [FP + opline->result.var] @@ -2073,53 +2133,48 @@ static int zend_jit_simple_assign(dasm_State **Dst, zend_op *opline, zend_op_arr { if (val_type == IS_CONST) { zval *zv = RT_CONSTANT(op_array, val); - | ZVAL_COPY_CONST FP, var, var_info, zv, r0 + | ZVAL_COPY_CONST FP + var, var_info, zv, r0 || if (Z_REFCOUNTED_P(zv)) { | ADDREF_CONST zv, r0 || } } else { if (val_info & MAY_BE_UNDEF) { - | cmp byte [FP + val.var + 8], IS_UNDEF - | jz >1 + | IF_Z_TYPE FP + val.var, IS_UNDEF, >1 |.cold_code |1: | // zend_error(E_NOTICE, "Undefined variable: %s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); | mov FCARG1d, opline->op1.var | EXT_CALL zend_jit_undefined_op_helper, r0 - | mov dword[FP + var + 8], IS_NULL + | SET_Z_TYPE_INFO FP + var, IS_NULL | jmp >3 |.code } if (val_info & MAY_BE_REF) { - | lea r2, [FP + val.var] if (val_type == IS_CV) { + | lea r2, [FP + val.var] | ZVAL_DEREF r2, val_info - | ZVAL_COPY_VALUE FP, var, r2, 0, val_info, r0, eax, r1 + | ZVAL_COPY_VALUE FP + var, r2, val_info, r0, eax, r1 } else { - | cmp byte [FP + val.var + 8], IS_REFERENCE - | je >1 + | IF_Z_TYPE FP + val.var, IS_REFERENCE, >1 |.cold_code |1: | // zend_refcounted *ref = Z_COUNTED_P(retval_ptr); - | mov r0, aword [FP + opline->op1.var] + | GET_Z_PTR r0, FP + opline->op1.var | // ZVAL_COPY_VALUE(return_value, &ref->value); - | ZVAL_COPY_VALUE FP, var, r0, 8, val_info, r2, edx, r1 - | dec dword [r0] + | ZVAL_COPY_VALUE FP + var, r0 + 8, val_info, r2, edx, r1 + | GC_DELREF r0 | je >2 - | // if (IS_REFCOUNTED()) - | test dh, IS_TYPE_REFCOUNTED - | jz >3 - | // ADDREF - | inc dword [r1] + | IF_NOT_REFCOUNTED dh, >3 + | GC_ADDREF r1 | jmp >3 |2: | EFREE_SIZE r0, sizeof(zend_reference), op_array, opline | jmp >3 |.code - | ZVAL_COPY_VALUE FP, var, FP, val.var, val_info, r0, eax, r1 + | ZVAL_COPY_VALUE FP + var, FP + val.var, val_info, r0, eax, r1 } } else { - | ZVAL_COPY_VALUE FP, var, FP, val.var, val_info, r0, eax, r1 + | ZVAL_COPY_VALUE FP + var, FP + val.var, val_info, r0, eax, r1 } || if (val_type == IS_CV) { | TRY_ADDREF val_info, ah, r1 @@ -2304,7 +2359,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, zend_op *opline, zend_op_array *o | mov aword EX:r1->call, 0 || if (RETURN_VALUE_USED(opline)) { | // ZVAL_NULL(EX_VAR(opline->result.var)); - | SET_TYPE_INFO FP, opline->result.var, IS_NULL + | SET_Z_TYPE_INFO FP + opline->result.var, IS_NULL | // EX(return_value) = EX_VAR(opline->result.var); | lea r0, aword [FP + opline->result.var] | mov aword EX:r1->return_value, r0 @@ -2315,7 +2370,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, zend_op *opline, zend_op_array *o | || for (i = call_info->num_args; i < func->op_array.last_var; i++) { || uint32_t n = (uint32_t)(uintptr_t)ZEND_CALL_VAR_NUM(NULL, i); - | SET_TYPE_INFO r1, n, IS_UNDEF + | SET_Z_TYPE_INFO r1 + n, IS_UNDEF || } | | //EX_LOAD_RUN_TIME_CACHE(op_array); @@ -2390,7 +2445,7 @@ static int zend_jit_send_val(dasm_State **Dst, zend_op *opline, zend_op_array *o } |.cold_code |1: - | SET_TYPE_INFO r1, opline->result.var, IS_UNDEF + | SET_Z_TYPE_INFO r1 + opline->result.var, IS_UNDEF | mov EX->opline, IP |.if X64 | mov CARG1, 0 @@ -2413,12 +2468,12 @@ static int zend_jit_send_val(dasm_State **Dst, zend_op *opline, zend_op_array *o if (opline->op1_type == IS_CONST) { zval *zv = RT_CONSTANT(op_array, opline->op1); - | ZVAL_COPY_CONST r1, opline->result.var, -1, zv, r0 + | ZVAL_COPY_CONST r1 + opline->result.var, -1, zv, r0 || if (Z_REFCOUNTED_P(zv)) { | ADDREF_CONST zv, r0 || } } else { - | ZVAL_COPY_VALUE r1, opline->result.var, FP, opline->op1.var, op1_info, r0, eax, r2 + | ZVAL_COPY_VALUE r1 + opline->result.var, FP + opline->op1.var, op1_info, r0, eax, r2 } valid_opline_offset++; @@ -2444,7 +2499,7 @@ static int zend_jit_send_var(dasm_State **Dst, zend_op *opline, zend_op_array *o op1_info = OP1_INFO(); | mov r1, EX->call - | ZVAL_COPY_VALUE r1, opline->result.var, FP, opline->op1.var, op1_info, r0, eax, r2 + | ZVAL_COPY_VALUE r1 + opline->result.var, FP + opline->op1.var, op1_info, r0, eax, r2 || if (opline->op1_type == IS_CV) { | TRY_ADDREF op1_info, ah, r2 || } @@ -2488,7 +2543,7 @@ static int zend_jit_type_check(dasm_State **Dst, zend_op *opline, int b, int *op target_label = ssa->cfg.blocks[b].successors[0]; | jmp =>target_label } else { - | SET_TYPE_INFO FP, opline->result.var, IS_TRUE + | SET_Z_TYPE_INFO FP + opline->result.var, IS_TRUE valid_opline_offset++; } } else if (!(op1_info & mask)) { @@ -2504,7 +2559,7 @@ static int zend_jit_type_check(dasm_State **Dst, zend_op *opline, int b, int *op (*opnum)++; valid_opline_offset += 2; } else { - | SET_TYPE_INFO FP, opline->result.var, IS_FALSE + | SET_Z_TYPE_INFO FP + opline->result.var, IS_FALSE valid_opline_offset++; } } else { @@ -2537,7 +2592,7 @@ static int zend_jit_free_compiled_variables(dasm_State **Dst, zend_op *opline, z if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { uint32_t offset = (uint32_t)(uintptr_t)ZEND_CALL_VAR_NUM(NULL, i); - | ZVAL_PTR_DTOR FP, offset, info, 1, (op_array->filename ? ZSTR_VAL(op_array->filename) : NULL), opline->lineno + | ZVAL_PTR_DTOR FP + offset, info, 1, (op_array->filename ? ZSTR_VAL(op_array->filename) : NULL), opline->lineno } } return 1; @@ -2588,7 +2643,7 @@ static int zend_jit_leave_func(dasm_State **Dst, zend_op *opline, zend_op_array | test FCARG1d, ZEND_CALL_CTOR | jz >5 | // GC_REFCOUNT(object)--; - | dec dword [r0] + | GC_DELREF r0 | // zend_object_store_ctor_failed(object); |.if X64 | mov CARG1, r0 @@ -2606,7 +2661,7 @@ static int zend_jit_leave_func(dasm_State **Dst, zend_op *opline, zend_op_array |5: } | // OBJ_RELEASE(object); - | OBJ_RELEASE r0, 4 + | OBJ_RELEASE r0, >4 | jmp >4 if (op_array->fn_flags & ZEND_ACC_STATIC) { |.code @@ -2621,7 +2676,7 @@ static int zend_jit_leave_func(dasm_State **Dst, zend_op *opline, zend_op_array | // OBJ_RELEASE((zend_object*)execute_data->func->op_array.prototype); | mov r0, EX->func | mov r0, aword [r0 + offsetof(zend_op_array, prototype)] - | OBJ_RELEASE r0, 4 + | OBJ_RELEASE r0, >4 | jmp >4 |.code |4: @@ -2665,14 +2720,13 @@ static int zend_jit_return(dasm_State **Dst, zend_op *opline, zend_op_array *op_ | jz >1 |.cold_code |1: - | test byte [FP + opline->op1.var + 9], IS_TYPE_REFCOUNTED || if (jit_return_label >= 0) { - | jz =>jit_return_label + | IF_NOT_Z_REFCOUNTED FP + opline->op1.var, =>jit_return_label || } else { - | jz >9 + | IF_NOT_Z_REFCOUNTED FP + opline->op1.var, >9 || } - | mov FCARG1a, aword [FP + opline->op1.var] - | dec dword [FCARG1a] + | GET_Z_PTR FCARG1a, FP + opline->op1.var + | GC_DELREF FCARG1a || if (jit_return_label >= 0) { | jnz =>jit_return_label || } else { @@ -2698,83 +2752,74 @@ static int zend_jit_return(dasm_State **Dst, zend_op *opline, zend_op_array *op_ if (opline->op1_type == IS_CONST) { zval *zv = RT_CONSTANT(op_array, opline->op1); - | ZVAL_COPY_CONST r1, 0, -1, zv, r0 + | ZVAL_COPY_CONST r1, -1, zv, r0 if (Z_REFCOUNTED_P(zv)) { | ADDREF_CONST zv, r0 } } else if (opline->op1_type == IS_TMP_VAR) { - | ZVAL_COPY_VALUE r1, 0, FP, opline->op1.var, op1_info, r0, eax, r2 + | ZVAL_COPY_VALUE r1, FP + opline->op1.var, op1_info, r0, eax, r2 } else if (opline->op1_type == IS_CV) { if (op1_info & MAY_BE_REF) { - | cmp byte [FP + opline->op1.var + 8], IS_REFERENCE - | je >1 + | IF_Z_TYPE FP + opline->op1.var, IS_REFERENCE, >1 |.cold_code |1: | // retval_ptr = Z_REFVAL_P(retval_ptr); - | mov r0, [FP + opline->op1.var] + | GET_Z_PTR r0, FP + opline->op1.var | // ZVAL_COPY(return_value, retval_ptr); - | mov edx, dword [r0 + 16] - | mov dword [r1 + 8], edx - | test dh, IS_TYPE_REFCOUNTED - | jnz >2 + | GET_Z_TYPE_INFO edx, r0 + offsetof(zend_reference, val) + | SET_Z_TYPE_INFO r1, edx + | IF_REFCOUNTED dh, >2 |.if X64 - | mov r2, aword [r0 + 8] - | mov aword [r1], r2 + | GET_Z_PTR r2, r0 + offsetof(zend_reference, val) + | SET_Z_PTR r1, r2 |.else - | mov r2, aword [r0 + 8] - | mov aword [r1], r2 - | mov r2, aword [r0 + 12] - | mov aword [r1 + 4], r2 + | GET_Z_PTR r2, r0 + offsetof(zend_reference, val) + | SET_Z_PTR r1, r2 + | GET_Z_W2 r2, r0 + offsetof(zend_reference, val) + | SET_Z_W2 r1, r2 |.endif | jmp >3 |2: - |.if X64 - | mov r2, aword [r0 + 8] - | mov aword [r1], r2 - |.else - | mov r2, aword [r0 + 8] - | mov aword [r1], r2 - |.endif - | inc dword [r2] + | GET_Z_PTR r2, r0 + offsetof(zend_reference, val) + | SET_Z_PTR r1, r2 + | GC_ADDREF r2 | jmp >3 |.code } - | ZVAL_COPY_VALUE r1, 0, FP, opline->op1.var, op1_info, r0, eax, r2 + | ZVAL_COPY_VALUE r1, FP + opline->op1.var, op1_info, r0, eax, r2 | // TODO: JIT: if (EXPECTED(!(EX_CALL_INFO() & ZEND_CALL_CODE))) ZVAL_NULL(retval_ptr); ??? | TRY_ADDREF op1_info, ah, r2 |3: } else { if (op1_info & MAY_BE_REF) { - | cmp byte [FP + opline->op1.var + 8], IS_REFERENCE - | je >1 + | IF_Z_TYPE FP + opline->op1.var, IS_REFERENCE, >1 |.cold_code |1: | // zend_refcounted *ref = Z_COUNTED_P(retval_ptr); - | mov r0, aword [FP + opline->op1.var] + | GET_Z_PTR r0, FP + opline->op1.var | // ZVAL_COPY_VALUE(return_value, &ref->value); |.if X64 - | mov r2, aword [r0 + 8] - | mov aword [r1], r2 + | GET_Z_PTR r2, r0 + offsetof(zend_reference, val) + | SET_Z_PTR r1, r2 |.else - | mov r2, aword [r0 + 8] - | mov aword [r1], r2 - | mov r2, aword [r0 + 12] - | mov aword [r1 + 4], r2 + | GET_Z_PTR r2, r0 + offsetof(zend_reference, val) + | SET_Z_PTR r1, r2 + | GET_Z_W2 r2, r0 + offsetof(zend_reference, val) + | SET_Z_W2 r1 + 4, r2 |.endif - | mov edx, dword [r0 + 16] - | mov dword [r1 + 8], edx - | dec dword [r0] + | GET_Z_TYPE_INFO edx, r0 + offsetof(zend_reference, val) + | SET_Z_TYPE_INFO r1, edx + | GC_DELREF r0 | je >2 | // if (IS_REFCOUNTED()) - | test dh, IS_TYPE_REFCOUNTED || if (jit_return_label >= 0) { - | je =>jit_return_label + | IF_NOT_REFCOUNTED dh, =>jit_return_label || } else { - | je >9 + | IF_NOT_REFCOUNTED dh, >9 || } | // ADDREF - | mov r2, aword [r1] - | inc dword [r2] + | GET_Z_PTR r2, r1 + | GC_ADDREF r2 || if (jit_return_label >= 0) { | jmp =>jit_return_label || } else { @@ -2789,7 +2834,7 @@ static int zend_jit_return(dasm_State **Dst, zend_op *opline, zend_op_array *op_ || } |.code } - | ZVAL_COPY_VALUE r1, 0, FP, opline->op1.var, op1_info, r0, eax, r2 + | ZVAL_COPY_VALUE r1, FP + opline->op1.var, op1_info, r0, eax, r2 } |9: @@ -2811,34 +2856,26 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, zend_op *opline, zend_op_ar if (op1_info & MAY_BE_REF) { | lea FCARG1a, [FP + opline->op1.var] - | cmp byte [FCARG1a + 8], IS_REFERENCE - | jne >1 - | mov FCARG1a, aword [FCARG1a] - | add FCARG1a, 8 - |1: + | ZVAL_DEREF FCARG1a, op1_info } if (op1_info & MAY_BE_ARRAY) { if (op1_info & MAY_BE_REF) { - | cmp byte [FCARG1a + 8], IS_ARRAY - | jne >7 // NOT_ARRAY - | mov FCARG1a, aword [FCARG1a] + | IF_NOT_Z_TYPE FCARG1a, IS_ARRAY, >7 + | GET_Z_PTR FCARG1a, FCARG1a } else { if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { - | // if (EXPECTED(Z_TYPE_P(container) == IS_ARRAY)) - | cmp byte [FP + opline->op1.var + 8], IS_ARRAY - | jne >7 // NOT_ARRAY + | IF_NOT_Z_TYPE FP + opline->op1.var, IS_ARRAY, >7 } - | LONG_LOAD opline->op1_type, opline->op1, FCARG1a + | LONG_LOAD FCARG1a, opline->op1_type, opline->op1 } if (op2_info & MAY_BE_LONG) { if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_LONG)) { | // if (EXPECTED(Z_TYPE_P(dim) == IS_LONG)) - | cmp byte [FP + opline->op2.var + 8], IS_LONG - | jne >3 // NOT_LONG + | IF_NOT_Z_TYPE FP + opline->op2.var, IS_LONG, >3 } | // hval = Z_LVAL_P(dim); - | LONG_LOAD opline->op2_type, opline->op2, FCARG2a + | LONG_LOAD FCARG2a, opline->op2_type, opline->op2 if (opline->op2_type == IS_CONST) { zend_long val = Z_LVAL_P(RT_CONSTANT(op_array, opline->op2)); @@ -2857,8 +2894,7 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, zend_op *opline, zend_op_ar | // _ret = &_ht->arData[_h].val; | mov r0, aword [FCARG1a + offsetof(zend_array, arData)] | add r0, val * sizeof(Bucket) - | cmp dword [r0 + 8], IS_UNDEF - | jne >8 // FOUND + | IF_NOT_Z_TYPE r0, IS_UNDEF, >8 //FOUND | jmp >2 // NOT_FOUND } } else { @@ -2881,8 +2917,7 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, zend_op *opline, zend_op_ar |.endif | mov r0, aword [FCARG1a + offsetof(zend_array, arData)] | add r0, FCARG2a - | cmp dword [r0 + 8], IS_UNDEF - | jne >8 // FOUND + | IF_NOT_Z_TYPE r0, IS_UNDEF, >8 // FOUND | jmp >2 // NOT_FOUND } |1: @@ -2896,11 +2931,11 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, zend_op *opline, zend_op_ar |.if X64 | mov CARG1, E_NOTICE | LOAD_ADDR CARG2, "Undefined offset: " ZEND_LONG_FMT - | LONG_LOAD opline->op2_type, opline->op2, CARG3 + | LONG_LOAD CARG3, opline->op2_type, opline->op2 | EXT_CALL zend_error, r0 |.else | sub r4, 4 - | LONG_LOAD opline->op2_type, opline->op2, r0 + | LONG_LOAD r0, opline->op2_type, opline->op2 | push r0 | push "Undefined offset: " ZEND_LONG_FMT | push E_NOTICE @@ -2909,7 +2944,7 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, zend_op *opline, zend_op_ar |.endif } | // retval = &EG(uninitialized_zval); - | mov dword [FP + opline->result.var + 8], IS_NULL + | SET_Z_TYPE_INFO FP + opline->result.var, IS_NULL | jmp >9 // END |.code if (op2_info & (MAY_BE_ANY - MAY_BE_STRING)) { @@ -2922,11 +2957,10 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, zend_op *opline, zend_op_ar if (op2_info & MAY_BE_STRING) { if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) { | // if (EXPECTED(Z_TYPE_P(dim) == IS_STRING)) - | cmp byte [FP + opline->op2.var + 8], IS_STRING - | jne >3 // NOT_STRING + | IF_NOT_Z_TYPE FP + opline->op2.var, IS_STRING, >3 } | // offset_key = Z_STR_P(dim); - | LONG_LOAD opline->op2_type, opline->op2, FCARG2a + | LONG_LOAD FCARG2a, opline->op2_type, opline->op2 | // retval = zend_hash_find(ht, offset_key); if (opline->op2_type != IS_CONST) { | EXT_CALL zend_jit_symtable_find, r0 @@ -2936,26 +2970,24 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, zend_op *opline, zend_op_ar | test r0, r0 | jz >2 // NOT_FOUND | // if (UNEXPECTED(Z_TYPE_P(retval) == IS_INDIRECT)) - | cmp dword [r0 + 8], IS_INDIRECT - | je >1 // SLOW + | IF_Z_TYPE r0, IS_INDIRECT, >1 // SLOW |.cold_code |1: | // retval = Z_INDIRECT_P(retval); - | mov r0, aword [r0] - | cmp dword [r0 + 8], IS_UNDEF - | jne >8 // COPY + | GET_Z_PTR r0, r0 + | IF_NOT_Z_TYPE r0, IS_UNDEF, >8 // COPY |2: if (opline->opcode == ZEND_FETCH_DIM_R) { // zend_error(E_NOTICE, "Undefined index: %s", ZSTR_VAL(offset_key)); |.if X64 | mov CARG1, E_NOTICE | LOAD_ADDR CARG2, "Undefined index: %s" - | LONG_LOAD opline->op2_type, opline->op2, CARG3 + | LONG_LOAD CARG3, opline->op2_type, opline->op2 | add CARG3, offsetof(zend_string, val) | EXT_CALL zend_error, r0 |.else | sub r4, 4 - | LONG_LOAD opline->op2_type, opline->op2, r0 + | LONG_LOAD r0, opline->op2_type, opline->op2 | add r0, offsetof(zend_string, val) | push r0 | push "Undefined index: %s" @@ -2964,7 +2996,7 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, zend_op *opline, zend_op_ar | add r4, 16 |.endif } - | mov dword [FP +opline->result.var + 8], IS_NULL + | SET_Z_TYPE_INFO FP + opline->result.var, IS_NULL | jmp >9 // END |.code if (op2_info & (MAY_BE_ANY - (MAY_BE_LONG|MAY_BE_STRING))) { @@ -3007,11 +3039,10 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, zend_op *opline, zend_op_ar if (op1_info & MAY_BE_STRING) { if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_STRING))) { if (op1_info & MAY_BE_REF) { - | cmp byte [FCARG1a + 8], IS_STRING + | IF_NOT_Z_TYPE FCARG1a, IS_STRING, >6 } else { - | cmp byte [FP + opline->op1.var + 8], IS_STRING + | IF_NOT_Z_TYPE FP + opline->op1.var, IS_STRING, >6 } - | jne >6 } | SAVE_VALID_OPLINE if (!(op1_info & MAY_BE_REF)) { @@ -3038,11 +3069,10 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, zend_op *opline, zend_op_ar if (op1_info & MAY_BE_OBJECT) { if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_STRING|MAY_BE_OBJECT))) { if (op1_info & MAY_BE_REF) { - | cmp byte [FCARG1a + 8], IS_OBJECT + | IF_NOT_Z_TYPE FCARG1a, IS_OBJECT, >6 } else { - | cmp byte [FP + opline->op1.var + 8], IS_OBJECT + | IF_NOT_Z_TYPE FP + opline->op1.var, IS_OBJECT, >6 } - | jne >6 } | SAVE_VALID_OPLINE if (!(op1_info & MAY_BE_REF)) { @@ -3068,24 +3098,22 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, zend_op *opline, zend_op_ar if (opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) { if (op1_info & MAY_BE_REF) { - | cmp byte [FCARG1a + 8], IS_UNDEF + | IF_NOT_Z_TYPE FCARG1a, IS_UNDEF, >1 } else { - | cmp byte [FP + opline->op1.var + 8], IS_UNDEF + | IF_NOT_Z_TYPE FP + opline->op1.var, IS_UNDEF, >1 } - | jne >1 | // zend_error(E_NOTICE, "Undefined variable: %s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); | mov FCARG1d, opline->op1.var | EXT_CALL zend_jit_undefined_op_helper, r0 |1: } if (op2_info & MAY_BE_UNDEF) { - | cmp byte [FP + opline->op2.var + 8], IS_UNDEF - | jne >1 + | IF_NOT_Z_TYPE FP + opline->op2.var, IS_UNDEF, >1 | mov FCARG1d, opline->op2.var | EXT_CALL zend_jit_undefined_op_helper, r0 |1: } - | mov dword [FP + opline->result.var + 8], IS_NULL + | SET_Z_TYPE_INFO FP + opline->result.var, IS_NULL | jmp >9 // END if (op1_info & MAY_BE_ARRAY) { |.code @@ -3095,11 +3123,9 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, zend_op *opline, zend_op_ar |8: if (res_info & MAY_BE_REF) { | // ZVAL_COPY_UNREF - | test byte [r0 + 9], IS_TYPE_REFCOUNTED - | jz >2 - | mov r1, aword [r0] - | cmp byte [r0 + 8], IS_REFERENCE - | jne >1 + | IF_NOT_Z_REFCOUNTED r0, >2 + | GET_Z_PTR r1, r0 + | IF_NOT_Z_TYPE r0, IS_REFERENCE, >1 | cmp dword [r1], 1 | jne >1 | lea FCARG1a, [FP + opline->result.var] @@ -3107,12 +3133,12 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, zend_op *opline, zend_op_ar | EXT_CALL zend_jit_zval_copy_unref_helper, r0 | jmp >9 |1: - | inc dword [r1] + | GC_ADDREF r1 |2: - | ZVAL_COPY_VALUE FP, opline->result.var, r0, 0, MAY_BE_ANY, r1, ecx, r2 + | ZVAL_COPY_VALUE FP + opline->result.var, r0, MAY_BE_ANY, r1, ecx, r2 } else { | // ZVAL_COPY - | ZVAL_COPY_VALUE FP, opline->result.var, r0, 0, MAY_BE_ANY, r1, ecx, r2 + | ZVAL_COPY_VALUE FP + opline->result.var, r0, MAY_BE_ANY, r1, ecx, r2 | TRY_ADDREF res_info, ch, r2 } |9: // END @@ -3165,9 +3191,7 @@ static int zend_jit_bind_global(dasm_State **Dst, zend_op *opline, zend_op_array |.else | add r0, [&EG(symbol_table).arData] |.endif - //if (EXPECTED(Z_TYPE(p->val) != IS_UNDEF) - | cmp byte [r0 + 8], IS_UNDEF - | je >9 + | IF_Z_TYPE r0, IS_UNDEF, >9 // (EXPECTED(p->key == Z_STR_P(varname)) | LOAD_ADDR r1, Z_PTR_P(varname) | cmp [r0 + offsetof(Bucket, key)], r1 @@ -3222,7 +3246,7 @@ static int zend_jit_bind_global(dasm_State **Dst, zend_op *opline, zend_op_array | mov cl, byte [r0 + 8] | cmp cl, IS_UNDEF | jne >2 - | SET_TYPE_INFO r0, 0, IS_NULL + | SET_Z_TYPE_INFO r0, IS_NULL | jmp >8 |.code |2: @@ -3233,13 +3257,12 @@ static int zend_jit_bind_global(dasm_State **Dst, zend_op *opline, zend_op_array //stash this for later use | mov r2, r0 } - | mov r0, [r0] - | inc dword [r0] + | GET_Z_PTR r0, r0 + | GC_ADDREF r0 //if (UNEXPECTED(Z_REFCOUNTED_P(variable_ptr))) if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { if (op1_info & (MAY_BE_ANY - (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { - | test byte [FP + opline->op1.var + 9], IS_TYPE_REFCOUNTED - | jnz >2 + | IF_Z_REFCOUNTED FP + opline->op1.var, >2 } |.cold_code //zval_dtor_func(Z_COUNTED_P(variable_ptr)) @@ -3248,37 +3271,35 @@ static int zend_jit_bind_global(dasm_State **Dst, zend_op *opline, zend_op_array | lea FCARG1a, aword [FP + opline->op1.var] | cmp FCARG1a, r2 | je >4 - | mov FCARG1a, [FCARG1a] - | dec dword [FCARG1a] + | GET_Z_PTR FCARG1a, FCARG1a + | GC_DELREF FCARG1a if (op1_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) { | jnz >3 } - | mov [r4], r0 + | mov aword [r4], r0 // save | ZVAL_DTOR_FUNC (op_array->filename ? ZSTR_VAL(op_array->filename) : NULL), opline->lineno - | mov r0, aword [r4] + | mov r0, aword [r4] // restore | jmp >5 if (op1_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) { |3: // GC_ZVAL_CHECK_POSSIBLE_ROOT(variable_ptr) - | test byte [FP + opline->op1.var + 9], IS_TYPE_COLLECTABLE - | jz >5 - | cmp word [FCARG1a + 6], 0 - | jne >5 - | mov [r4], r0 + | IF_NOT_Z_REFCOUNTED FP + opline->op1.var, >5 + | IF_GC_INFO FCARG1a, >5 + | mov aword [r4], r0 //save | EXT_CALL gc_possible_root, r1 - | mov r0, aword [r4] + | mov r0, aword [r4] // restore | jmp >5 } |4: - | mov FCARG1a, [FCARG1a] - | dec dword [FCARG1a] + | GET_Z_PTR FCARG1a, FCARG1a + | GC_DELREF FCARG1a | jmp >5 |.code } |5: //ZVAL_REF(variable_ptr, ref) - | mov aword [FP + opline->op1.var], r0 - | mov dword [FP + opline->op1.var + 8], IS_REFERENCE_EX + | SET_Z_PTR FP + opline->op1.var, r0 + | SET_Z_TYPE_INFO FP + opline->op1.var, IS_REFERENCE_EX //END of handler |.cold_code From 63d402cbf3901679cb6ce30467ce05f029dd8630 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 20 Sep 2016 10:30:20 +0300 Subject: [PATCH 215/569] Improved JIT for ASSIGN --- ext/opcache/jit/zend_jit_x86.dasc | 254 +++++++++++++++++++++++++++--- 1 file changed, 229 insertions(+), 25 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 18b49a5281cf7..c9cd558cd2354 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -432,6 +432,62 @@ static void* dasm_labels[zend_lb_MAX]; ||} |.endmacro +|.macro ZVAL_COPY_CONST_2, dst, dst2, dst_info, zv, tmp_reg +||if (Z_TYPE_P(zv) > IS_TRUE) { +|| if (Z_TYPE_P(zv) == IS_DOUBLE) { +|.if X64 or SSE +|| if (Z_DVAL_P(zv) == 0.0 && !is_signed(Z_DVAL_P(zv))) { +| xorps xmm0, xmm0 +| .if X64 +|| } else if (!IS_32BIT(zv)) { +| mov64 tmp_reg, ((uintptr_t)zv) +| movsd xmm0, qword [tmp_reg] +| .endif +|| } else { +| movsd xmm0, qword [((uint32_t)(uintptr_t)zv)] +|| } +| movsd qword [dst], xmm0 +| movsd qword [dst2], xmm0 +|.else +|| if (Z_DVAL_P(zv) == 0.0 && !is_signed(Z_DVAL_P(zv))) { +| fldz +|| } else if (Z_DVAL_P(zv) == 1.0) { +| fld1 +|| } else { +| fld qword [zv] +|| } +| fstp qword [dst] +|| if (Z_DVAL_P(zv) == 0.0 && !is_signed(Z_DVAL_P(zv))) { +| fldz +|| } else if (Z_DVAL_P(zv) == 1.0) { +| fld1 +|| } else { +| fld qword [zv] +|| } +| fstp qword [dst2] +|.endif +|| } else { +|.if X64 +|| if (!IS_32BIT(Z_LVAL_P(zv))) { +| mov64 tmp_reg, ((uintptr_t)Z_LVAL_P(zv)) +| SET_Z_LVAL dst, tmp_reg +| SET_Z_LVAL dst2, tmp_reg +|| } else { +| SET_Z_LVAL dst, Z_LVAL_P(zv) +| SET_Z_LVAL dst2, Z_LVAL_P(zv) +|| } +|.else +| SET_Z_LVAL dst, Z_LVAL_P(zv) +| SET_Z_LVAL dst2, Z_LVAL_P(zv) +|.endif +|| } +||} +||if ((dst_info & MAY_BE_ANY) != (1<1 +|| } +| add dword [value_ptr_reg], 2 +|1: +||} +|.endmacro + // zval should be in FCARG1a |.macro ZVAL_COPY_CTOR_FUNC, filename, lineno // arg1 must be in FCARG1a || if (ZEND_DEBUG) { @@ -2129,56 +2239,110 @@ fallback: return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); } -static int zend_jit_simple_assign(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, uint32_t var, uint32_t var_info, zend_uchar val_type, znode_op val, uint32_t val_info) +static int zend_jit_simple_assign(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, uint32_t var, uint32_t var_info, zend_uchar val_type, znode_op val, uint32_t val_info, int in_cold, uint32_t var2) { if (val_type == IS_CONST) { zval *zv = RT_CONSTANT(op_array, val); - | ZVAL_COPY_CONST FP + var, var_info, zv, r0 + if (var2 == (uint32_t)-1) { + | ZVAL_COPY_CONST FP + var, var_info, zv, r0 + } else { + | ZVAL_COPY_CONST_2 FP + var, FP + var2, var_info, zv, r0 + } || if (Z_REFCOUNTED_P(zv)) { +// TODO: += 2 | ADDREF_CONST zv, r0 || } } else { if (val_info & MAY_BE_UNDEF) { - | IF_Z_TYPE FP + val.var, IS_UNDEF, >1 - |.cold_code - |1: + if (in_cold) { + | IF_NOT_Z_TYPE FP + val.var, IS_UNDEF, >2 + } else { + | IF_Z_TYPE FP + val.var, IS_UNDEF, >1 + |.cold_code + |1: + } | // zend_error(E_NOTICE, "Undefined variable: %s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); - | mov FCARG1d, opline->op1.var + | mov FCARG1d, val.var | EXT_CALL zend_jit_undefined_op_helper, r0 - | SET_Z_TYPE_INFO FP + var, IS_NULL + | SET_Z_TYPE_INFO FP + val.var, IS_NULL | jmp >3 - |.code + if (in_cold) { + |2: + } else { + |.code + } } if (val_info & MAY_BE_REF) { if (val_type == IS_CV) { | lea r2, [FP + val.var] | ZVAL_DEREF r2, val_info - | ZVAL_COPY_VALUE FP + var, r2, val_info, r0, eax, r1 + if (var2 == (uint32_t)-1) { + | ZVAL_COPY_VALUE FP + var, r2, val_info, r0, eax, r1 + } else { + | ZVAL_COPY_VALUE_2 FP + var, FP + var2, r2, val_info, r0, eax, r1 + } } else { - | IF_Z_TYPE FP + val.var, IS_REFERENCE, >1 - |.cold_code - |1: + if (in_cold) { + | IF_NOT_Z_TYPE FP + val.var, IS_REFERENCE, >1 + } else { + | IF_Z_TYPE FP + val.var, IS_REFERENCE, >1 + |.cold_code + |1: + } | // zend_refcounted *ref = Z_COUNTED_P(retval_ptr); - | GET_Z_PTR r0, FP + opline->op1.var + | GET_Z_PTR r0, FP + val.var | // ZVAL_COPY_VALUE(return_value, &ref->value); - | ZVAL_COPY_VALUE FP + var, r0 + 8, val_info, r2, edx, r1 + if (var2 == (uint32_t)-1) { + | ZVAL_COPY_VALUE FP + var, r0 + 8, val_info, r2, edx, r1 + } else { + | ZVAL_COPY_VALUE_2 FP + var, FP + var2, r0 + 8, val_info, r2, edx, r1 + } | GC_DELREF r0 | je >2 | IF_NOT_REFCOUNTED dh, >3 - | GC_ADDREF r1 + if (var2 == (uint32_t)-1) { + | GC_ADDREF r1 + } else { + | add dword [r1], 2 + } | jmp >3 |2: + if (var2 != (uint32_t)-1) { + | IF_NOT_REFCOUNTED dh, >2 + | GC_ADDREF r1 + |2: + } | EFREE_SIZE r0, sizeof(zend_reference), op_array, opline | jmp >3 - |.code + if (in_cold) { + |1: + } else { + |.code + } + if (var2 == (uint32_t)-1) { + | ZVAL_COPY_VALUE FP + var, FP + val.var, val_info, r0, eax, r1 + } else { + | ZVAL_COPY_VALUE_2 FP + var, FP + var2, FP + val.var, val_info, r0, eax, r1 + } + } + } else { + if (var2 == (uint32_t)-1) { | ZVAL_COPY_VALUE FP + var, FP + val.var, val_info, r0, eax, r1 + } else { + | ZVAL_COPY_VALUE_2 FP + var, FP + var2, FP + val.var, val_info, r0, eax, r1 + } + } + if (val_type == IS_CV) { + if (var2 == (uint32_t)-1) { + | TRY_ADDREF val_info, ah, r1 + } else { + | TRY_ADDREF_2 val_info, ah, r1 } } else { - | ZVAL_COPY_VALUE FP + var, FP + val.var, val_info, r0, eax, r1 + if (var2 != (uint32_t)-1) { + | TRY_ADDREF val_info, ah, r1 + } } - || if (val_type == IS_CV) { - | TRY_ADDREF val_info, ah, r1 - || } |3: } return 1; @@ -2189,24 +2353,64 @@ static int zend_jit_qm_assign(dasm_State **Dst, zend_op *opline, zend_op_array * uint32_t op1_info = OP1_INFO(); valid_opline_offset++; - return zend_jit_simple_assign(Dst, opline, op_array, ssa, opline->result.var, -1, opline->op1_type, opline->op1, op1_info); + return zend_jit_simple_assign(Dst, opline, op_array, ssa, opline->result.var, -1, opline->op1_type, opline->op1, op1_info, 0, -1); } static int zend_jit_assign(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) { uint32_t op1_info, op2_info; - if (opline->op1_type != IS_CV || opline->result_type != IS_UNUSED || !ssa->ops || !ssa->var_info) { + if (opline->op1_type != IS_CV || !ssa->ops || !ssa->var_info) { goto fallback; } op1_info = OP1_INFO(); op2_info = OP2_INFO(); - if (!(op1_info & (MAY_BE_STRING|MAY_BE_RESOURCE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_REF))) { - valid_opline_offset++; - return zend_jit_simple_assign(Dst, opline, op_array, ssa, opline->op1.var, op1_info, opline->op2_type, opline->op2, op2_info); + if (op1_info & MAY_BE_REF) { + goto fallback; + } + + if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { + | IF_Z_REFCOUNTED FP + opline->op1.var, >1 + |.cold_code + |1: + } + | // TODO: support for object->set + | // TODO: support for assignment to itself + | GET_Z_PTR r0, FP + opline->op1.var + | GC_DELREF r0 + | jnz >4 + | mov aword [r4], r0 // save + if (!zend_jit_simple_assign(Dst, opline, op_array, ssa, opline->op1.var, op1_info, opline->op2_type, opline->op2, op2_info, + (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) != 0, + opline->result_type == IS_UNUSED ? -1 : opline->result.var)) { + return 0; + } + | mov FCARG1a, aword [r4] // restore + | ZVAL_DTOR_FUNC (op_array->filename ? ZSTR_VAL(op_array->filename) : NULL), opline->lineno + | jmp >6 + |4: + | IF_NOT_Z_FLAGS FP + opline->op1.var, IS_TYPE_COLLECTABLE, >5 + | GET_Z_PTR FCARG1a, FP + opline->op1.var + | IF_GC_INFO FCARG1a, >5 + | EXT_CALL gc_possible_root, r0 + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { + | jmp >5 + |.code + } + |5: + } + + if (!zend_jit_simple_assign(Dst, opline, op_array, ssa, opline->op1.var, op1_info, opline->op2_type, opline->op2, op2_info, 0, + opline->result_type == IS_UNUSED ? -1 : opline->result.var)) { + return 0; } + |6: + + valid_opline_offset++; + return 1; fallback: /* fallback to subroutine threading */ From 4fa6f6f6ade6876b79ee4d028d74790591102e89 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 20 Sep 2016 11:20:30 +0300 Subject: [PATCH 216/569] Separate JIT for assignto_to_variable(). --- ext/opcache/jit/zend_jit_x86.dasc | 64 ++++++++++++++++++------------- 1 file changed, 37 insertions(+), 27 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index c9cd558cd2354..17d2fa7a377f6 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -2239,7 +2239,7 @@ fallback: return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); } -static int zend_jit_simple_assign(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, uint32_t var, uint32_t var_info, zend_uchar val_type, znode_op val, uint32_t val_info, int in_cold, uint32_t var2) +static int zend_jit_simple_assign(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, uint32_t var, uint32_t var_info, zend_uchar val_type, znode_op val, uint32_t val_info, uint32_t var2, int in_cold) { if (val_type == IS_CONST) { zval *zv = RT_CONSTANT(op_array, val); @@ -2353,62 +2353,72 @@ static int zend_jit_qm_assign(dasm_State **Dst, zend_op *opline, zend_op_array * uint32_t op1_info = OP1_INFO(); valid_opline_offset++; - return zend_jit_simple_assign(Dst, opline, op_array, ssa, opline->result.var, -1, opline->op1_type, opline->op1, op1_info, 0, -1); + return zend_jit_simple_assign(Dst, opline, op_array, ssa, opline->result.var, -1, opline->op1_type, opline->op1, op1_info, -1, 0); } -static int zend_jit_assign(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) +static int zend_jit_assign_to_variable(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, uint32_t var, uint32_t var_info, zend_uchar val_type, znode_op val, uint32_t val_info, uint32_t var2) { - uint32_t op1_info, op2_info; - - if (opline->op1_type != IS_CV || !ssa->ops || !ssa->var_info) { - goto fallback; - } + if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { + int in_cold = 0; - op1_info = OP1_INFO(); - op2_info = OP2_INFO(); - - if (op1_info & MAY_BE_REF) { - goto fallback; - } - - if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { - if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { - | IF_Z_REFCOUNTED FP + opline->op1.var, >1 + if (var_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { + | IF_Z_REFCOUNTED FP + var, >1 |.cold_code |1: + in_cold = 1; } | // TODO: support for object->set | // TODO: support for assignment to itself - | GET_Z_PTR r0, FP + opline->op1.var + | GET_Z_PTR r0, FP + var | GC_DELREF r0 | jnz >4 | mov aword [r4], r0 // save - if (!zend_jit_simple_assign(Dst, opline, op_array, ssa, opline->op1.var, op1_info, opline->op2_type, opline->op2, op2_info, - (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) != 0, - opline->result_type == IS_UNUSED ? -1 : opline->result.var)) { + if (!zend_jit_simple_assign(Dst, opline, op_array, ssa, var, var_info, val_type, val, val_info, var2, in_cold)) { return 0; } | mov FCARG1a, aword [r4] // restore | ZVAL_DTOR_FUNC (op_array->filename ? ZSTR_VAL(op_array->filename) : NULL), opline->lineno | jmp >6 |4: - | IF_NOT_Z_FLAGS FP + opline->op1.var, IS_TYPE_COLLECTABLE, >5 - | GET_Z_PTR FCARG1a, FP + opline->op1.var + | IF_NOT_Z_FLAGS FP + var, IS_TYPE_COLLECTABLE, >5 + | GET_Z_PTR FCARG1a, FP + var | IF_GC_INFO FCARG1a, >5 | EXT_CALL gc_possible_root, r0 - if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { + if (in_cold) { | jmp >5 |.code } |5: } - if (!zend_jit_simple_assign(Dst, opline, op_array, ssa, opline->op1.var, op1_info, opline->op2_type, opline->op2, op2_info, 0, - opline->result_type == IS_UNUSED ? -1 : opline->result.var)) { + if (!zend_jit_simple_assign(Dst, opline, op_array, ssa, var, var_info, val_type, val, val_info, var2, 0)) { return 0; } |6: + return 1; +} + +static int zend_jit_assign(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) +{ + uint32_t op1_info, op2_info; + + if (opline->op1_type != IS_CV || !ssa->ops || !ssa->var_info) { + goto fallback; + } + + op1_info = OP1_INFO(); + op2_info = OP2_INFO(); + + if (op1_info & MAY_BE_REF) { + goto fallback; + } + + if (!zend_jit_assign_to_variable(Dst, opline, op_array, ssa, opline->op1.var, op1_info, opline->op2_type, opline->op2, op2_info, + opline->result_type == IS_UNUSED ? -1 : opline->result.var)) { + return 0; + } + valid_opline_offset++; return 1; From 2dad50101b27d6e499ed7aa8e6e65953cab82109 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 20 Sep 2016 17:31:23 +0300 Subject: [PATCH 217/569] Implemented JIT for JMPZ/JMPNZ/JMPZNZ --- ext/opcache/jit/zend_jit.c | 42 +++- ext/opcache/jit/zend_jit_x86.dasc | 311 +++++++++++++++++++++++++++--- 2 files changed, 322 insertions(+), 31 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 760bcb3968cc1..1c5ffaf2788d3 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -850,6 +850,32 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) goto jit_failure; } break; + case ZEND_JMPZ: + case ZEND_JMPNZ: + case ZEND_JMPZNZ: + if ((opline-1)->opcode == ZEND_IS_EQUAL || + (opline-1)->opcode == ZEND_IS_NOT_EQUAL || + (opline-1)->opcode == ZEND_IS_SMALLER || + (opline-1)->opcode == ZEND_IS_SMALLER_OR_EQUAL || + (opline-1)->opcode == ZEND_CASE) { + /* skip */ + } else if ((opline-1)->opcode == ZEND_IS_IDENTICAL || + (opline-1)->opcode == ZEND_IS_NOT_IDENTICAL || + (opline-1)->opcode == ZEND_ISSET_ISEMPTY_VAR || + (opline-1)->opcode == ZEND_ISSET_ISEMPTY_STATIC_PROP || + (opline-1)->opcode == ZEND_ISSET_ISEMPTY_DIM_OBJ || + (opline-1)->opcode == ZEND_ISSET_ISEMPTY_PROP_OBJ || + (opline-1)->opcode == ZEND_INSTANCEOF || + (opline-1)->opcode == ZEND_TYPE_CHECK || + (opline-1)->opcode == ZEND_DEFINED) { + /* smart branch */ + if (!zend_jit_cond_jmp(&dasm_state, opline + 1, ssa->cfg.blocks[b].successors[0])) { + goto jit_failure; + } + } else if (!zend_jit_jmpznz(&dasm_state, opline, b, op_array, ssa)) { + goto jit_failure; + } + break; case ZEND_FETCH_DIM_R: case ZEND_FETCH_DIM_IS: if (!zend_jit_fetch_dim_read(&dasm_state, opline, op_array, ssa)) { @@ -920,6 +946,14 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) goto jit_failure; } break; +#if ZEND_JIT_LEVEL < ZEND_JIT_LEVEL_OPT_FUNC + case ZEND_JMPZNZ: + if (!zend_jit_handler(&dasm_state, opline, zend_may_throw(opline, op_array, ssa)) || + !zend_jit_cond_jmp(&dasm_state, OP_JMP_ADDR(opline, opline->op2), ssa->cfg.blocks[b].successors[1]) || + !zend_jit_jmp(&dasm_state, ssa->cfg.blocks[b].successors[0])) { + goto jit_failure; + } + break; case ZEND_JMPZ: case ZEND_JMPNZ: if (i != 0) { @@ -963,6 +997,7 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) } } /* break missing intentionally */ +#endif case ZEND_JMPZ_EX: case ZEND_JMPNZ_EX: case ZEND_JMP_SET: @@ -980,13 +1015,6 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) goto jit_failure; } break; - case ZEND_JMPZNZ: - if (!zend_jit_handler(&dasm_state, opline, zend_may_throw(opline, op_array, ssa)) || - !zend_jit_cond_jmp(&dasm_state, OP_JMP_ADDR(opline, opline->op2), ssa->cfg.blocks[b].successors[1]) || - !zend_jit_jmp(&dasm_state, ssa->cfg.blocks[b].successors[0])) { - goto jit_failure; - } - break; case ZEND_FE_FETCH_R: case ZEND_FE_FETCH_RW: case ZEND_DECLARE_ANON_CLASS: diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 17d2fa7a377f6..7cbb42dd0f779 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -78,7 +78,20 @@ static void* dasm_labels[zend_lb_MAX]; | mov64 reg, ((ptrdiff_t)addr) // 0x48 0xb8 ||} |.else -| mov reg, addr +| mov reg, ((ptrdiff_t)addr) +|.endif +|.endmacro + +|.macro PUSH_ADDR, addr, tmp_reg +|.if X64 +||if (((ptrdiff_t)addr) <= 0x7fffffff) { +| push ((ptrdiff_t)addr) // 0x48 0xc7 0xc0 +|| } else { +| mov64 tmp_reg, ((ptrdiff_t)addr) // 0x48 0xb8 +| push tmp_reg +||} +|.else +| push ((ptrdiff_t)addr) |.endif |.endmacro @@ -1564,7 +1577,7 @@ static int zend_jit_math(dasm_State **Dst, zend_op *opline, int *opnum, zend_op_ has_slow = (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && - ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) || + ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) || (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))); if (opline->result_type == IS_TMP_VAR && @@ -1799,6 +1812,27 @@ static int zend_jit_cmp_long_long(dasm_State **Dst, zend_op *opline, int b, zend | jle => target_label break; } + } else if ((opline+1)->opcode == ZEND_JMPZNZ && + (opline+1)->op1_type == IS_TMP_VAR && + (opline+1)->op1.var == opline->result.var) { + target_label = ssa->cfg.blocks[b].successors[0]; + switch (opline->opcode) { + case ZEND_IS_EQUAL: + case ZEND_CASE: + | jne => target_label + break; + case ZEND_IS_NOT_EQUAL: + | je => target_label + break; + case ZEND_IS_SMALLER: + | jge => target_label + break; + case ZEND_IS_SMALLER_OR_EQUAL: + | jg => target_label + break; + } + target_label = ssa->cfg.blocks[b].successors[1]; + | jmp => target_label } else { switch (opline->opcode) { case ZEND_IS_EQUAL: @@ -1879,6 +1913,31 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, zend_op *opline, int b, | jna => target_label break; } + } else if ((opline+1)->opcode == ZEND_JMPZNZ && + (opline+1)->op1_type == IS_TMP_VAR && + (opline+1)->op1.var == opline->result.var) { + target_label = ssa->cfg.blocks[b].successors[0]; + switch (opline->opcode) { + case ZEND_IS_EQUAL: + case ZEND_CASE: + | jp >1 + | jne => target_label + |1: + break; + case ZEND_IS_NOT_EQUAL: + | jp >1 + | je => target_label + |1: + break; + case ZEND_IS_SMALLER: + | jae => target_label + break; + case ZEND_IS_SMALLER_OR_EQUAL: + | ja => target_label + break; + } + target_label = ssa->cfg.blocks[b].successors[1]; + | jmp => target_label } else { switch (opline->opcode) { case ZEND_IS_EQUAL: @@ -2004,6 +2063,27 @@ static int zend_jit_cmp_slow(dasm_State **Dst, zend_op *opline, int b, zend_op_a | jle => target_label break; } + } else if ((opline+1)->opcode == ZEND_JMPZNZ && + (opline+1)->op1_type == IS_TMP_VAR && + (opline+1)->op1.var == opline->result.var) { + target_label = ssa->cfg.blocks[b].successors[0]; + switch (opline->opcode) { + case ZEND_IS_EQUAL: + case ZEND_CASE: + | jne => target_label + break; + case ZEND_IS_NOT_EQUAL: + | je => target_label + break; + case ZEND_IS_SMALLER: + | jge => target_label + break; + case ZEND_IS_SMALLER_OR_EQUAL: + | jg => target_label + break; + } + target_label = ssa->cfg.blocks[b].successors[1]; + | jmp => target_label } else { switch (opline->opcode) { case ZEND_IS_EQUAL: @@ -2040,21 +2120,13 @@ static int zend_jit_cmp(dasm_State **Dst, zend_op *opline, int b, int *opnum, ze zend_bool same_ops = (opline->op1_type == opline->op2_type) && (opline->op1.var == opline->op2.var); zend_bool has_slow; - if (!ssa->ops || !ssa->var_info) { - goto fallback; - } - op1_info = OP1_INFO(); op2_info = OP2_INFO(); - if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) { - goto fallback; - } - has_slow = (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && - ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) || + ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) || (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))); if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)) { @@ -2194,17 +2266,41 @@ static int zend_jit_cmp(dasm_State **Dst, zend_op *opline, int b, int *opnum, ze |9: } | mov EX->opline, IP + | lea FCARG1a, [FP + opline->result.var] + | LOAD_ZVAL_ADDR FCARG2a, opline->op1_type, opline->op1 if (opline->op1_type == IS_CV && (op1_info & MAY_BE_UNDEF)) { - | IF_NOT_Z_TYPE FP + opline->op1.var, IS_UNDEF, >1 + | IF_NOT_Z_TYPE FCARG2a, IS_UNDEF, >1 + | mov FCARG1a, opline->op1.var + | EXT_CALL zend_jit_undefined_op_helper, r0 + | LOAD_ADDR FCARG2a, &EG(uninitialized_zval) |1: } - | lea FCARG1a, [FP + opline->result.var] - | LOAD_ZVAL_ADDR FCARG2a, opline->op1_type, opline->op1 - |.if X64 - | LOAD_ZVAL_ADDR CARG3, opline->op2_type, opline->op2 - |.else - | PUSH_ZVAL_ADDR opline->op2_type, opline->op2, r0 - |.endif + if (opline->op2_type == IS_CV && (op2_info & MAY_BE_UNDEF)) { + | IF_NOT_Z_TYPE FP + opline->op2.var, IS_UNDEF, >1 + | mov [r4], FCARG2a // save + | mov FCARG1a, opline->op2.var + | EXT_CALL zend_jit_undefined_op_helper, r0 + | mov FCARG2a, [r4] // restore + |.if X64 + | LOAD_ADDR CARG3, &EG(uninitialized_zval) + |.else + | PUSH_ADDR &EG(uninitialized_zval), r0 + |.endif + | jmp >2 + |1: + |.if X64 + | LOAD_ZVAL_ADDR CARG3, opline->op2_type, opline->op2 + |.else + | PUSH_ZVAL_ADDR opline->op2_type, opline->op2, r0 + |.endif + |2: + } else { + |.if X64 + | LOAD_ZVAL_ADDR CARG3, opline->op2_type, opline->op2 + |.else + | PUSH_ZVAL_ADDR opline->op2_type, opline->op2, r0 + |.endif + } | EXT_CALL compare_function, r0 || if (opline->opcode != ZEND_CASE) { | FREE_OP opline->op1_type, opline->op1, op1_info, op_array, opline->lineno @@ -2223,7 +2319,9 @@ static int zend_jit_cmp(dasm_State **Dst, zend_op *opline, int b, int *opnum, ze } |6: - if (((opline+1)->opcode == ZEND_JMPZ || (opline+1)->opcode == ZEND_JMPNZ) && + if (((opline+1)->opcode == ZEND_JMPZ || + (opline+1)->opcode == ZEND_JMPNZ || + (opline+1)->opcode == ZEND_JMPZNZ) && (opline+1)->op1_type == IS_TMP_VAR && (opline+1)->op1.var == opline->result.var) { (*opnum)++; @@ -2233,10 +2331,175 @@ static int zend_jit_cmp(dasm_State **Dst, zend_op *opline, int b, int *opnum, ze } return 1; +} -fallback: - /* fallback to subroutine threading */ - return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); +static int zend_jit_jmpznz(dasm_State **Dst, zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa) +{ + uint32_t op1_info = OP1_INFO(); + uint32_t true_label, false_label; + + if (opline->opcode == ZEND_JMPZ) { + true_label = (uint32_t)-1; + false_label = ssa->cfg.blocks[b].successors[0]; + } else if (opline->opcode == ZEND_JMPNZ) { + true_label = ssa->cfg.blocks[b].successors[0]; + false_label = (uint32_t)-1; + } else if (opline->opcode == ZEND_JMPZNZ) { + true_label = ssa->cfg.blocks[b].successors[1]; + false_label = ssa->cfg.blocks[b].successors[0]; + } else { + ZEND_ASSERT(0); + } + + if ((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY)) == MAY_BE_TRUE) { + if (true_label != (uint32_t)-1) { + | jmp =>true_label; + } + } else if (!(op1_info & (MAY_BE_ANY - (MAY_BE_NULL|MAY_BE_FALSE)))) { + if (op1_info & MAY_BE_UNDEF) { + if (op1_info & MAY_BE_ANY) { + | IF_Z_TYPE FP + opline->op1.var, IS_UNDEF, >1 + |.cold_code + |1: + } + | mov FCARG1d, opline->op1.var + | SAVE_VALID_OPLINE + | EXT_CALL zend_jit_undefined_op_helper, r0 + if (zend_may_throw(opline, op_array, ssa)) { + if (!zend_jit_check_exception(Dst)) { + return 0; + } + } + if (op1_info & MAY_BE_ANY) { + if (false_label != (uint32_t)-1) { + | jmp =>false_label; + } else { + | jmp >9 + } + |.code + } + } + if (false_label != (uint32_t)-1) { + | jmp =>false_label; + } + } else { + | LOAD_ZVAL_ADDR FCARG1a, opline->op1_type, opline->op1 + | ZVAL_DEREF FCARG1a, op1_info + + if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE)) { + if (true_label != (uint32_t)-1) { + | IF_Z_TYPE FCARG1a, IS_TRUE, =>true_label + } else { + | IF_Z_TYPE FCARG1a, IS_TRUE, >9 + } + + if (op1_info & MAY_BE_UNDEF) { + if (op1_info & (MAY_BE_ANY - (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) { + | jg >2 + } + | IF_Z_TYPE FCARG1a, IS_UNDEF, >1 + if (false_label != (uint32_t)-1) { + | jmp =>false_label + } else if (op1_info & (MAY_BE_ANY - (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) { + | jmp >9 + } + |.cold_code + |1: + | mov FCARG1d, opline->op1.var + | SAVE_VALID_OPLINE + | EXT_CALL zend_jit_undefined_op_helper, r0 + if (zend_may_throw(opline, op_array, ssa)) { + if (!zend_jit_check_exception(Dst)) { + return 0; + } + } + if (false_label != (uint32_t)-1) { + | jmp =>false_label + } else { + | jmp >9 + } + |.code + } else { + if (false_label != (uint32_t)-1) { + | jl =>false_label + } else if (op1_info & (MAY_BE_ANY - (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) { + if (op1_info & MAY_BE_LONG) { + | jl >9 + } else { + | jg >2 + } + } + } + } + + if (op1_info & MAY_BE_LONG) { + |2: + if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG))) { + | IF_NOT_Z_TYPE FCARG1a, IS_LONG, >2 + } + | cmp aword [FCARG1a], 0 + if (true_label != (uint32_t)-1) { + | jne =>true_label + if (false_label != (uint32_t)-1) { + | jmp =>false_label + } + } else { + | je =>false_label + } + } + + if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG))) { + if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { + |.cold_code + |2: + } + | SAVE_VALID_OPLINE + | EXT_CALL zend_is_true, r0 + + if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && + (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { + | test byte [FP + opline->op1.var + 9], IS_TYPE_REFCOUNTED + | je >1 + | mov FCARG1a, aword [FP + opline->op1.var] + | dec dword [FCARG1a] + | jnz >1 + | mov aword [r4], r0 // save + | ZVAL_DTOR_FUNC (op_array->filename ? ZSTR_VAL(op_array->filename) : NULL), opline->lineno + | mov r0, aword [r4] // restore + |1: + } + if (zend_may_throw(opline, op_array, ssa)) { + if (!zend_jit_check_exception(Dst)) { + return 0; + } + } + + | test r0, r0 + if (true_label != (uint32_t)-1) { + | jne =>true_label + if (false_label != (uint32_t)-1) { + | jmp =>false_label + } else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { + | jmp >9 + } + } else { + | je =>false_label + if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { + | jmp >9 + } + } + + if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { + |.code + } + } + + |9: + } + + valid_opline_offset++; + + return 1; } static int zend_jit_simple_assign(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, uint32_t var, uint32_t var_info, zend_uchar val_type, znode_op val, uint32_t val_info, uint32_t var2, int in_cold) @@ -3489,7 +3752,7 @@ static int zend_jit_bind_global(dasm_State **Dst, zend_op *opline, zend_op_array | GC_DELREF FCARG1a if (op1_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) { | jnz >3 - } + } | mov aword [r4], r0 // save | ZVAL_DTOR_FUNC (op_array->filename ? ZSTR_VAL(op_array->filename) : NULL), opline->lineno | mov r0, aword [r4] // restore From 1ba74786fea1701f55ffeaee8170ca8fdbf9ef2a Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Wed, 21 Sep 2016 00:36:40 +0800 Subject: [PATCH 218/569] Fixed segfault(array and string maybe not refcounted) --- ext/opcache/jit/zend_jit_x86.dasc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 7cbb42dd0f779..2ef2e57da83d6 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -440,7 +440,7 @@ static void* dasm_labels[zend_lb_MAX]; |.endif || } ||} -||if ((dst_info & MAY_BE_ANY) != (1< Date: Wed, 21 Sep 2016 01:15:46 +0800 Subject: [PATCH 219/569] Revert "Fixed segfault(array and string maybe not refcounted)" This reverts commit 1ba74786fea1701f55ffeaee8170ca8fdbf9ef2a. --- ext/opcache/jit/zend_jit_x86.dasc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 2ef2e57da83d6..7cbb42dd0f779 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -440,7 +440,7 @@ static void* dasm_labels[zend_lb_MAX]; |.endif || } ||} -||if (((dst_info & MAY_BE_ANY) != (1< Date: Wed, 21 Sep 2016 11:39:23 +0800 Subject: [PATCH 220/569] Revert "Revert "Fixed segfault(array and string maybe not refcounted)"" This reverts commit 4bd731eab5abcba4e5f6a780f5ec970d15b98f1c. --- ext/opcache/jit/zend_jit_x86.dasc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 7cbb42dd0f779..2ef2e57da83d6 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -440,7 +440,7 @@ static void* dasm_labels[zend_lb_MAX]; |.endif || } ||} -||if ((dst_info & MAY_BE_ANY) != (1< Date: Wed, 21 Sep 2016 12:09:43 +0800 Subject: [PATCH 221/569] Fixed memleak in ext/spl/tests/SplFileObject_fputcsv_variation5.phpt --- ext/opcache/jit/zend_jit_x86.dasc | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 2ef2e57da83d6..2161a687ea3a3 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -2450,7 +2450,6 @@ static int zend_jit_jmpznz(dasm_State **Dst, zend_op *opline, int b, zend_op_arr if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG))) { if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { - |.cold_code |2: } | SAVE_VALID_OPLINE @@ -2459,7 +2458,7 @@ static int zend_jit_jmpznz(dasm_State **Dst, zend_op *opline, int b, zend_op_arr if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { | test byte [FP + opline->op1.var + 9], IS_TYPE_REFCOUNTED - | je >1 + | jz >1 | mov FCARG1a, aword [FP + opline->op1.var] | dec dword [FCARG1a] | jnz >1 @@ -2488,10 +2487,6 @@ static int zend_jit_jmpznz(dasm_State **Dst, zend_op *opline, int b, zend_op_arr | jmp >9 } } - - if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { - |.code - } } |9: From d1fd16d6743462a209b944f2011d18075d5f0510 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Wed, 21 Sep 2016 13:10:00 +0800 Subject: [PATCH 222/569] Implemented JIT for RECV_INIT --- ext/opcache/jit/zend_jit.c | 8 +- ext/opcache/jit/zend_jit_disasm_x86.c | 2 + ext/opcache/jit/zend_jit_helpers.c | 60 ++++++++++++++ ext/opcache/jit/zend_jit_x86.dasc | 111 +++++++++++++++++++++++++- 4 files changed, 173 insertions(+), 8 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 1c5ffaf2788d3..5ebc603f1d9aa 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -725,11 +725,7 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) } else if (ssa->cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_RECV_ENTRY)) { opline = op_array->opcodes + ssa->cfg.blocks[b].start; if (ssa->cfg.split_at_recv && opline->opcode == ZEND_RECV_INIT) { - if (opline > op_array->opcodes && - (opline-1)->opcode == ZEND_RECV_INIT) { - /* repeatable opcode */ - continue; - } else { + if (opline == op_array->opcodes || (opline-1)->opcode != ZEND_RECV_INIT) { for (i = 0; (opline+i)->opcode == ZEND_RECV_INIT; i++) { } zend_jit_jmp(&dasm_state, b + i); @@ -890,7 +886,7 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) #endif case ZEND_RECV_INIT: if (ssa->cfg.split_at_recv) { - if (!zend_jit_handler(&dasm_state, opline, zend_may_throw(opline, op_array, ssa))) { + if (!zend_jit_recv_init(&dasm_state, opline, op_array, (opline + 1)->opcode != ZEND_RECV_INIT)) { goto jit_failure; } break; diff --git a/ext/opcache/jit/zend_jit_disasm_x86.c b/ext/opcache/jit/zend_jit_disasm_x86.c index 39f28373a73d0..5093d1417f4ac 100644 --- a/ext/opcache/jit/zend_jit_disasm_x86.c +++ b/ext/opcache/jit/zend_jit_disasm_x86.c @@ -397,6 +397,8 @@ static int zend_jit_disasm_init(void) REGISTER_HELPER(zend_jit_zval_copy_unref_helper); REGISTER_HELPER(zend_jit_new_ref_helper); REGISTER_HELPER(zend_jit_fetch_global_helper); + REGISTER_HELPER(zend_jit_verify_arg_object); + REGISTER_HELPER(zend_jit_verify_arg_slow); #undef REGISTER_HELPER zend_elf_load_symbols(); diff --git a/ext/opcache/jit/zend_jit_helpers.c b/ext/opcache/jit/zend_jit_helpers.c index 88c50a7da34f4..f27976ae909bb 100644 --- a/ext/opcache/jit/zend_jit_helpers.c +++ b/ext/opcache/jit/zend_jit_helpers.c @@ -401,6 +401,66 @@ static zval* ZEND_FASTCALL zend_jit_fetch_global_helper(zend_execute_data *execu return value; } +static void ZEND_FASTCALL zend_jit_verify_arg_object(zval *arg, zend_op_array *op_array, uint32_t arg_num, zend_arg_info *arg_info, void **cache_slot) +{ + zend_class_entry *ce; + if (EXPECTED(*cache_slot)) { + ce = (zend_class_entry *)*cache_slot; + } else { + ce = zend_fetch_class(arg_info->class_name, (ZEND_FETCH_CLASS_AUTO | ZEND_FETCH_CLASS_NO_AUTOLOAD)); + if (UNEXPECTED(!ce)) { + zend_verify_arg_error(op_array, arg_info, arg_num, NULL, arg); + return; + } + *cache_slot = (void *)ce; + } + if (UNEXPECTED(!instanceof_function(Z_OBJCE_P(arg), ce))) { + zend_verify_arg_error(op_array, arg_info, arg_num, ce, arg); + } +} + +static void ZEND_FASTCALL zend_jit_verify_arg_slow(zval *arg, zend_op_array *op_array, uint32_t arg_num, zend_arg_info *arg_info, void **cache_slot, zval *default_value) +{ + zend_class_entry *ce = NULL; + + if (Z_TYPE_P(arg) == IS_NULL && + (arg_info->allow_null || (default_value && is_null_constant(op_array->scope, default_value)))) { + /* Null passed to nullable type */ + return; + } + + if (UNEXPECTED(arg_info->class_name)) { + /* This is always an error - we fetch the class name for the error message here */ + if (EXPECTED(*cache_slot)) { + ce = (zend_class_entry *) *cache_slot; + } else { + ce = zend_fetch_class(arg_info->class_name, (ZEND_FETCH_CLASS_AUTO | ZEND_FETCH_CLASS_NO_AUTOLOAD)); + if (ce) { + *cache_slot = (void *)ce; + } + } + goto err; + } else if (arg_info->type_hint == IS_CALLABLE) { + if (zend_is_callable(arg, IS_CALLABLE_CHECK_SILENT, NULL) == 0) { + goto err; + } + } else if (arg_info->type_hint == IS_ITERABLE) { + if (zend_is_iterable(arg) == 0) { + goto err; + } + } else if (arg_info->type_hint == _IS_BOOL && + EXPECTED(Z_TYPE_P(arg) == IS_FALSE || Z_TYPE_P(arg) == IS_TRUE)) { + return; + } else { + if (zend_verify_scalar_type_hint(arg_info->type_hint, arg, ZEND_RET_USES_STRICT_TYPES()) == 0) { + goto err; + } + } + return; +err: + zend_verify_arg_error(op_array, arg_info, arg_num, ce, arg); +} + /* * Local variables: * tab-width: 4 diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 2161a687ea3a3..62a9d93a5adea 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -3695,9 +3695,9 @@ static int zend_jit_bind_global(dasm_State **Dst, zend_op *opline, zend_op_array | call &memcmp |.else | sub r4, 4 - | push r1 - | push Z_STRVAL_P(varname) | push Z_STRLEN_P(varname) + | push Z_STRVAL_P(varname) + | push r1 | call &memcmp | add r4, 16 |.endif @@ -3791,6 +3791,113 @@ static int zend_jit_bind_global(dasm_State **Dst, zend_op *opline, zend_op_array return 1; } +static int zend_jit_recv_init(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_bool is_last) { + zend_arg_info *arg_info; + zend_bool has_slow = 0; + uint32_t arg_num = opline->op1.num; + zval *zv = RT_CONSTANT(op_array, opline->op2); + + | cmp dword EX->This.u2.num_args, arg_num + | jae >4 + | ZVAL_COPY_CONST FP + opline->result.var, -1, zv, r0 + if (Z_REFCOUNTED_P(zv)) { + | ADDREF_CONST zv, r0 + } + if (Z_CONSTANT_P(zv)) { + has_slow = 1; + | LOAD_ADDR IP, opline + |.if X64 + | lea CARG1, [FP + opline->result.var] + | mov r0, EX->func + | mov CARG2, [r0 + offsetof(zend_op_array, scope)] + | EXT_CALL zval_update_constant_ex, r0 + |.else + | sub r4, 8 + | mov r0, EX->func + | push dword [r0 + offsetof(zend_op_array, scope)] + | lea r0, [FP + opline->result.var] + | push r0 + | EXT_CALL zval_update_constant_ex, r0 + | add r4, 16 + |.endif + | test al, al + | jnz >7 + } + |4: + if (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) { + do { + if (arg_num <= op_array->num_args) { + arg_info = &op_array->arg_info[arg_num-1]; + } else if (op_array->fn_flags & ZEND_ACC_VARIADIC) { + arg_info = &op_array->arg_info[op_array->num_args]; + } else { + break; + } + if (!arg_info->type_hint) { + break; + } + has_slow = 2; + | mov r0, aword [FP + opline->result.var] + | ZVAL_DEREF r0, MAY_BE_REF + | cmp byte [r0 + 8], arg_info->type_hint + | jne >8 + if (arg_info->class_name) { + | mov FCARG1a, r0 + | mov r0, EX->run_time_cache + | mov r0, aword [r0 + Z_CACHE_SLOT_P(zv)] + | LOAD_ADDR FCARG2a, (ptrdiff_t)op_array + |.if X64 + | mov CARG3, arg_num + | LOAD_ADDR CARG4, (ptrdiff_t)arg_info + | mov CARG5, r0 + |.else + | push arg_num + | push (ptrdiff_t)arg_info + | push r0 + |.endif + | EXT_CALL zend_jit_verify_arg_object, r0 + } + } while (0); + } + |9: + if (is_last) { + | LOAD_ADDR IP, (opline + 1) + } + if (Z_CONSTANT_P(zv)) { + zend_jit_check_exception(Dst); + } + + if (has_slow) { + |.cold_code + |7: + | ZVAL_PTR_DTOR FP + opline->result.var, -1, 0, (op_array->filename ? ZSTR_VAL(op_array->filename) : NULL), opline->lineno + | SET_Z_TYPE_INFO FP + opline->result.var, IS_UNDEF + | jmp <4 + if (has_slow > 1) { + |8: + | mov FCARG1a, r0 + | mov r0, EX->run_time_cache + | mov r0, aword [r0 + Z_CACHE_SLOT_P(zv)] + | LOAD_ADDR FCARG2a, (ptrdiff_t)op_array + |.if X64 + | mov CARG3, arg_num + | LOAD_ADDR CARG4, (ptrdiff_t)arg_info + | mov CARG5, r0 + | mov CARG6, (ptrdiff_t)zv + |.else + | push arg_num + | push (ptrdiff_t)arg_info + | push r0 + | push zv + |.endif + | EXT_CALL zend_jit_verify_arg_slow, r0 + | jmp <9 + } + |.code + } + + return 1; +} #endif /* From c63fcd29ee8fad33ec688d2c11567ac9232e1938 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Wed, 21 Sep 2016 14:53:08 +0800 Subject: [PATCH 223/569] Fixed argument passing --- ext/opcache/jit/zend_jit_x86.dasc | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 62a9d93a5adea..d62035304039b 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -1525,20 +1525,19 @@ static int zend_jit_shift(dasm_State **Dst, zend_op *opline, zend_op_array *op_a if (has_slow) { |.cold_code |9: - | lea FCARG1a, aword [FP + opline->result.var] + | lea FCARG1a, aword [FP + opline->result.var] + | lea FCARG2a, aword [FP + opline->op1.var] |.if X64 - | mov FCARG2a, r0 - | mov CARG3, Z_LVAL_P(op2) + | LOAD_ADDR CARG3, op2 |.else - | push r0 - | push dword Z_LVAL_P(op2) + | push dword op2 |.endif if (opline->opcode == ZEND_SR) { - | EXT_CALL shift_right_function, r0 + | EXT_CALL shift_right_function, r0 } else { - | EXT_CALL shift_left_function, r0 + | EXT_CALL shift_left_function, r0 } - | jmp <2 + | jmp <2 |.code } From baefe2227f8d9641b7dc9fe2d83e1333f4443c26 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Wed, 21 Sep 2016 14:59:36 +0800 Subject: [PATCH 224/569] Fixed arguments pushing order --- ext/opcache/jit/zend_jit_x86.dasc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index d62035304039b..dd5b24733a7d8 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -816,9 +816,9 @@ static void* dasm_labels[zend_lb_MAX]; | xor CARG4, CARG4 | xor CARG5, CARG5 |.else -| push opline->lineno | push 0 | push 0 +| push opline->lineno |.endif || } | EXT_CALL _efree, r0 From fc97c255b76a68058e23a2e0d2ba8df2933c6f7d Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Wed, 21 Sep 2016 15:55:24 +0800 Subject: [PATCH 225/569] Fixed missed functions --- ext/opcache/jit/zend_jit_helpers.c | 105 ++++++++++++++++++++++++++++- 1 file changed, 102 insertions(+), 3 deletions(-) diff --git a/ext/opcache/jit/zend_jit_helpers.c b/ext/opcache/jit/zend_jit_helpers.c index f27976ae909bb..cc2d5efc836fa 100644 --- a/ext/opcache/jit/zend_jit_helpers.c +++ b/ext/opcache/jit/zend_jit_helpers.c @@ -401,6 +401,105 @@ static zval* ZEND_FASTCALL zend_jit_fetch_global_helper(zend_execute_data *execu return value; } +static ZEND_COLD void zend_verify_type_error_common( + const zend_function *zf, const zend_arg_info *arg_info, + const zend_class_entry *ce, zval *value, + const char **fname, const char **fsep, const char **fclass, + const char **need_msg, const char **need_kind, const char **need_or_null, + const char **given_msg, const char **given_kind) +{ + zend_bool is_interface = 0; + *fname = ZSTR_VAL(zf->common.function_name); + + if (zf->common.scope) { + *fsep = "::"; + *fclass = ZSTR_VAL(zf->common.scope->name); + } else { + *fsep = ""; + *fclass = ""; + } + + switch (arg_info->type_hint) { + case IS_OBJECT: + if (ce) { + if (ce->ce_flags & ZEND_ACC_INTERFACE) { + *need_msg = "implement interface "; + is_interface = 1; + } else { + *need_msg = "be an instance of "; + } + *need_kind = ZSTR_VAL(ce->name); + } else { + /* We don't know whether it's a class or interface, assume it's a class */ + *need_msg = "be an instance of "; + *need_kind = zf->common.type == ZEND_INTERNAL_FUNCTION + ? ((zend_internal_arg_info *) arg_info)->class_name + : ZSTR_VAL(arg_info->class_name); + } + break; + case IS_CALLABLE: + *need_msg = "be callable"; + *need_kind = ""; + break; + case IS_ITERABLE: + *need_msg = "be iterable"; + *need_kind = ""; + break; + default: + *need_msg = "be of the type "; + *need_kind = zend_get_type_by_const(arg_info->type_hint); + break; + } + + if (arg_info->allow_null) { + *need_or_null = is_interface ? " or be null" : " or null"; + } else { + *need_or_null = ""; + } + + if (value) { + if (arg_info->type_hint == IS_OBJECT && Z_TYPE_P(value) == IS_OBJECT) { + *given_msg = "instance of "; + *given_kind = ZSTR_VAL(Z_OBJCE_P(value)->name); + } else { + *given_msg = zend_zval_type_name(value); + *given_kind = ""; + } + } else { + *given_msg = "none"; + *given_kind = ""; + } +} + +static ZEND_COLD void zend_verify_arg_error( + const zend_function *zf, const zend_arg_info *arg_info, + int arg_num, const zend_class_entry *ce, zval *value) +{ + zend_execute_data *ptr = EG(current_execute_data)->prev_execute_data; + const char *fname, *fsep, *fclass; + const char *need_msg, *need_kind, *need_or_null, *given_msg, *given_kind; + + if (value) { + zend_verify_type_error_common( + zf, arg_info, ce, value, + &fname, &fsep, &fclass, &need_msg, &need_kind, &need_or_null, &given_msg, &given_kind); + + if (zf->common.type == ZEND_USER_FUNCTION) { + if (ptr && ptr->func && ZEND_USER_CODE(ptr->func->common.type)) { + zend_type_error("Argument %d passed to %s%s%s() must %s%s%s, %s%s given, called in %s on line %d", + arg_num, fclass, fsep, fname, need_msg, need_kind, need_or_null, given_msg, given_kind, + ZSTR_VAL(ptr->func->op_array.filename), ptr->opline->lineno); + } else { + zend_type_error("Argument %d passed to %s%s%s() must %s%s%s, %s%s given", arg_num, fclass, fsep, fname, need_msg, need_kind, need_or_null, given_msg, given_kind); + } + } else { + zend_type_error("Argument %d passed to %s%s%s() must %s%s%s, %s%s given", arg_num, fclass, fsep, fname, need_msg, need_kind, need_or_null, given_msg, given_kind); + } + } else { + zend_missing_arg_error(ptr); + } +} + static void ZEND_FASTCALL zend_jit_verify_arg_object(zval *arg, zend_op_array *op_array, uint32_t arg_num, zend_arg_info *arg_info, void **cache_slot) { zend_class_entry *ce; @@ -409,13 +508,13 @@ static void ZEND_FASTCALL zend_jit_verify_arg_object(zval *arg, zend_op_array *o } else { ce = zend_fetch_class(arg_info->class_name, (ZEND_FETCH_CLASS_AUTO | ZEND_FETCH_CLASS_NO_AUTOLOAD)); if (UNEXPECTED(!ce)) { - zend_verify_arg_error(op_array, arg_info, arg_num, NULL, arg); + zend_verify_arg_error((zend_function*)op_array, arg_info, arg_num, NULL, arg); return; } *cache_slot = (void *)ce; } if (UNEXPECTED(!instanceof_function(Z_OBJCE_P(arg), ce))) { - zend_verify_arg_error(op_array, arg_info, arg_num, ce, arg); + zend_verify_arg_error((zend_function*)op_array, arg_info, arg_num, ce, arg); } } @@ -458,7 +557,7 @@ static void ZEND_FASTCALL zend_jit_verify_arg_slow(zval *arg, zend_op_array *op_ } return; err: - zend_verify_arg_error(op_array, arg_info, arg_num, ce, arg); + zend_verify_arg_error((zend_function*)op_array, arg_info, arg_num, ce, arg); } /* From 948a6d73f9637579b811e6be0d094c55eb137687 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Wed, 21 Sep 2016 16:02:17 +0800 Subject: [PATCH 226/569] Added more --- ext/opcache/jit/zend_jit_helpers.c | 78 ++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/ext/opcache/jit/zend_jit_helpers.c b/ext/opcache/jit/zend_jit_helpers.c index cc2d5efc836fa..fd04e079d2f6a 100644 --- a/ext/opcache/jit/zend_jit_helpers.c +++ b/ext/opcache/jit/zend_jit_helpers.c @@ -401,6 +401,84 @@ static zval* ZEND_FASTCALL zend_jit_fetch_global_helper(zend_execute_data *execu return value; } +static int is_null_constant(zend_class_entry *scope, zval *default_value) +{ + if (Z_CONSTANT_P(default_value)) { + zval constant; + + ZVAL_COPY(&constant, default_value); + if (UNEXPECTED(zval_update_constant_ex(&constant, scope) != SUCCESS)) { + return 0; + } + if (Z_TYPE(constant) == IS_NULL) { + return 1; + } + zval_ptr_dtor(&constant); + } + return 0; +} + +static zend_bool zend_verify_weak_scalar_type_hint(zend_uchar type_hint, zval *arg) +{ + switch (type_hint) { + case _IS_BOOL: { + zend_bool dest; + + if (!zend_parse_arg_bool_weak(arg, &dest)) { + return 0; + } + zval_ptr_dtor(arg); + ZVAL_BOOL(arg, dest); + return 1; + } + case IS_LONG: { + zend_long dest; + + if (!zend_parse_arg_long_weak(arg, &dest)) { + return 0; + } + zval_ptr_dtor(arg); + ZVAL_LONG(arg, dest); + return 1; + } + case IS_DOUBLE: { + double dest; + + if (!zend_parse_arg_double_weak(arg, &dest)) { + return 0; + } + zval_ptr_dtor(arg); + ZVAL_DOUBLE(arg, dest); + return 1; + } + case IS_STRING: { + zend_string *dest; + + /* on success "arg" is converted to IS_STRING */ + if (!zend_parse_arg_str_weak(arg, &dest)) { + return 0; + } + return 1; + } + default: + return 0; + } +} + +static zend_bool zend_verify_scalar_type_hint(zend_uchar type_hint, zval *arg, zend_bool strict) +{ + if (UNEXPECTED(strict)) { + /* SSTH Exception: IS_LONG may be accepted as IS_DOUBLE (converted) */ + if (type_hint != IS_DOUBLE || Z_TYPE_P(arg) != IS_LONG) { + return 0; + } + } else if (UNEXPECTED(Z_TYPE_P(arg) == IS_NULL)) { + /* NULL may be accepted only by nullable hints (this is already checked) */ + return 0; + } + return zend_verify_weak_scalar_type_hint(type_hint, arg); +} + static ZEND_COLD void zend_verify_type_error_common( const zend_function *zf, const zend_arg_info *arg_info, const zend_class_entry *ce, zval *value, From 5722df1af04bcab05a86d9399d236841f6753c30 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Wed, 21 Sep 2016 16:15:43 +0800 Subject: [PATCH 227/569] Revert "Added more" This reverts commit 948a6d73f9637579b811e6be0d094c55eb137687. --- ext/opcache/jit/zend_jit_helpers.c | 78 ------------------------------ 1 file changed, 78 deletions(-) diff --git a/ext/opcache/jit/zend_jit_helpers.c b/ext/opcache/jit/zend_jit_helpers.c index fd04e079d2f6a..cc2d5efc836fa 100644 --- a/ext/opcache/jit/zend_jit_helpers.c +++ b/ext/opcache/jit/zend_jit_helpers.c @@ -401,84 +401,6 @@ static zval* ZEND_FASTCALL zend_jit_fetch_global_helper(zend_execute_data *execu return value; } -static int is_null_constant(zend_class_entry *scope, zval *default_value) -{ - if (Z_CONSTANT_P(default_value)) { - zval constant; - - ZVAL_COPY(&constant, default_value); - if (UNEXPECTED(zval_update_constant_ex(&constant, scope) != SUCCESS)) { - return 0; - } - if (Z_TYPE(constant) == IS_NULL) { - return 1; - } - zval_ptr_dtor(&constant); - } - return 0; -} - -static zend_bool zend_verify_weak_scalar_type_hint(zend_uchar type_hint, zval *arg) -{ - switch (type_hint) { - case _IS_BOOL: { - zend_bool dest; - - if (!zend_parse_arg_bool_weak(arg, &dest)) { - return 0; - } - zval_ptr_dtor(arg); - ZVAL_BOOL(arg, dest); - return 1; - } - case IS_LONG: { - zend_long dest; - - if (!zend_parse_arg_long_weak(arg, &dest)) { - return 0; - } - zval_ptr_dtor(arg); - ZVAL_LONG(arg, dest); - return 1; - } - case IS_DOUBLE: { - double dest; - - if (!zend_parse_arg_double_weak(arg, &dest)) { - return 0; - } - zval_ptr_dtor(arg); - ZVAL_DOUBLE(arg, dest); - return 1; - } - case IS_STRING: { - zend_string *dest; - - /* on success "arg" is converted to IS_STRING */ - if (!zend_parse_arg_str_weak(arg, &dest)) { - return 0; - } - return 1; - } - default: - return 0; - } -} - -static zend_bool zend_verify_scalar_type_hint(zend_uchar type_hint, zval *arg, zend_bool strict) -{ - if (UNEXPECTED(strict)) { - /* SSTH Exception: IS_LONG may be accepted as IS_DOUBLE (converted) */ - if (type_hint != IS_DOUBLE || Z_TYPE_P(arg) != IS_LONG) { - return 0; - } - } else if (UNEXPECTED(Z_TYPE_P(arg) == IS_NULL)) { - /* NULL may be accepted only by nullable hints (this is already checked) */ - return 0; - } - return zend_verify_weak_scalar_type_hint(type_hint, arg); -} - static ZEND_COLD void zend_verify_type_error_common( const zend_function *zf, const zend_arg_info *arg_info, const zend_class_entry *ce, zval *value, From 09f96b720e7d544dbaa8f71748e8dd19d72fc8b9 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Wed, 21 Sep 2016 16:15:53 +0800 Subject: [PATCH 228/569] Revert "Fixed missed functions" This reverts commit fc97c255b76a68058e23a2e0d2ba8df2933c6f7d. --- ext/opcache/jit/zend_jit_helpers.c | 105 +---------------------------- 1 file changed, 3 insertions(+), 102 deletions(-) diff --git a/ext/opcache/jit/zend_jit_helpers.c b/ext/opcache/jit/zend_jit_helpers.c index cc2d5efc836fa..f27976ae909bb 100644 --- a/ext/opcache/jit/zend_jit_helpers.c +++ b/ext/opcache/jit/zend_jit_helpers.c @@ -401,105 +401,6 @@ static zval* ZEND_FASTCALL zend_jit_fetch_global_helper(zend_execute_data *execu return value; } -static ZEND_COLD void zend_verify_type_error_common( - const zend_function *zf, const zend_arg_info *arg_info, - const zend_class_entry *ce, zval *value, - const char **fname, const char **fsep, const char **fclass, - const char **need_msg, const char **need_kind, const char **need_or_null, - const char **given_msg, const char **given_kind) -{ - zend_bool is_interface = 0; - *fname = ZSTR_VAL(zf->common.function_name); - - if (zf->common.scope) { - *fsep = "::"; - *fclass = ZSTR_VAL(zf->common.scope->name); - } else { - *fsep = ""; - *fclass = ""; - } - - switch (arg_info->type_hint) { - case IS_OBJECT: - if (ce) { - if (ce->ce_flags & ZEND_ACC_INTERFACE) { - *need_msg = "implement interface "; - is_interface = 1; - } else { - *need_msg = "be an instance of "; - } - *need_kind = ZSTR_VAL(ce->name); - } else { - /* We don't know whether it's a class or interface, assume it's a class */ - *need_msg = "be an instance of "; - *need_kind = zf->common.type == ZEND_INTERNAL_FUNCTION - ? ((zend_internal_arg_info *) arg_info)->class_name - : ZSTR_VAL(arg_info->class_name); - } - break; - case IS_CALLABLE: - *need_msg = "be callable"; - *need_kind = ""; - break; - case IS_ITERABLE: - *need_msg = "be iterable"; - *need_kind = ""; - break; - default: - *need_msg = "be of the type "; - *need_kind = zend_get_type_by_const(arg_info->type_hint); - break; - } - - if (arg_info->allow_null) { - *need_or_null = is_interface ? " or be null" : " or null"; - } else { - *need_or_null = ""; - } - - if (value) { - if (arg_info->type_hint == IS_OBJECT && Z_TYPE_P(value) == IS_OBJECT) { - *given_msg = "instance of "; - *given_kind = ZSTR_VAL(Z_OBJCE_P(value)->name); - } else { - *given_msg = zend_zval_type_name(value); - *given_kind = ""; - } - } else { - *given_msg = "none"; - *given_kind = ""; - } -} - -static ZEND_COLD void zend_verify_arg_error( - const zend_function *zf, const zend_arg_info *arg_info, - int arg_num, const zend_class_entry *ce, zval *value) -{ - zend_execute_data *ptr = EG(current_execute_data)->prev_execute_data; - const char *fname, *fsep, *fclass; - const char *need_msg, *need_kind, *need_or_null, *given_msg, *given_kind; - - if (value) { - zend_verify_type_error_common( - zf, arg_info, ce, value, - &fname, &fsep, &fclass, &need_msg, &need_kind, &need_or_null, &given_msg, &given_kind); - - if (zf->common.type == ZEND_USER_FUNCTION) { - if (ptr && ptr->func && ZEND_USER_CODE(ptr->func->common.type)) { - zend_type_error("Argument %d passed to %s%s%s() must %s%s%s, %s%s given, called in %s on line %d", - arg_num, fclass, fsep, fname, need_msg, need_kind, need_or_null, given_msg, given_kind, - ZSTR_VAL(ptr->func->op_array.filename), ptr->opline->lineno); - } else { - zend_type_error("Argument %d passed to %s%s%s() must %s%s%s, %s%s given", arg_num, fclass, fsep, fname, need_msg, need_kind, need_or_null, given_msg, given_kind); - } - } else { - zend_type_error("Argument %d passed to %s%s%s() must %s%s%s, %s%s given", arg_num, fclass, fsep, fname, need_msg, need_kind, need_or_null, given_msg, given_kind); - } - } else { - zend_missing_arg_error(ptr); - } -} - static void ZEND_FASTCALL zend_jit_verify_arg_object(zval *arg, zend_op_array *op_array, uint32_t arg_num, zend_arg_info *arg_info, void **cache_slot) { zend_class_entry *ce; @@ -508,13 +409,13 @@ static void ZEND_FASTCALL zend_jit_verify_arg_object(zval *arg, zend_op_array *o } else { ce = zend_fetch_class(arg_info->class_name, (ZEND_FETCH_CLASS_AUTO | ZEND_FETCH_CLASS_NO_AUTOLOAD)); if (UNEXPECTED(!ce)) { - zend_verify_arg_error((zend_function*)op_array, arg_info, arg_num, NULL, arg); + zend_verify_arg_error(op_array, arg_info, arg_num, NULL, arg); return; } *cache_slot = (void *)ce; } if (UNEXPECTED(!instanceof_function(Z_OBJCE_P(arg), ce))) { - zend_verify_arg_error((zend_function*)op_array, arg_info, arg_num, ce, arg); + zend_verify_arg_error(op_array, arg_info, arg_num, ce, arg); } } @@ -557,7 +458,7 @@ static void ZEND_FASTCALL zend_jit_verify_arg_slow(zval *arg, zend_op_array *op_ } return; err: - zend_verify_arg_error((zend_function*)op_array, arg_info, arg_num, ce, arg); + zend_verify_arg_error(op_array, arg_info, arg_num, ce, arg); } /* From ef7d3a9f29afd9722fb9c796b32ee318fd8ef625 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Wed, 21 Sep 2016 16:16:05 +0800 Subject: [PATCH 229/569] Revert "Implemented JIT for RECV_INIT" This reverts commit d1fd16d6743462a209b944f2011d18075d5f0510. --- ext/opcache/jit/zend_jit.c | 8 +- ext/opcache/jit/zend_jit_disasm_x86.c | 2 - ext/opcache/jit/zend_jit_helpers.c | 60 -------------- ext/opcache/jit/zend_jit_x86.dasc | 111 +------------------------- 4 files changed, 8 insertions(+), 173 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 5ebc603f1d9aa..1c5ffaf2788d3 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -725,7 +725,11 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) } else if (ssa->cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_RECV_ENTRY)) { opline = op_array->opcodes + ssa->cfg.blocks[b].start; if (ssa->cfg.split_at_recv && opline->opcode == ZEND_RECV_INIT) { - if (opline == op_array->opcodes || (opline-1)->opcode != ZEND_RECV_INIT) { + if (opline > op_array->opcodes && + (opline-1)->opcode == ZEND_RECV_INIT) { + /* repeatable opcode */ + continue; + } else { for (i = 0; (opline+i)->opcode == ZEND_RECV_INIT; i++) { } zend_jit_jmp(&dasm_state, b + i); @@ -886,7 +890,7 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) #endif case ZEND_RECV_INIT: if (ssa->cfg.split_at_recv) { - if (!zend_jit_recv_init(&dasm_state, opline, op_array, (opline + 1)->opcode != ZEND_RECV_INIT)) { + if (!zend_jit_handler(&dasm_state, opline, zend_may_throw(opline, op_array, ssa))) { goto jit_failure; } break; diff --git a/ext/opcache/jit/zend_jit_disasm_x86.c b/ext/opcache/jit/zend_jit_disasm_x86.c index 5093d1417f4ac..39f28373a73d0 100644 --- a/ext/opcache/jit/zend_jit_disasm_x86.c +++ b/ext/opcache/jit/zend_jit_disasm_x86.c @@ -397,8 +397,6 @@ static int zend_jit_disasm_init(void) REGISTER_HELPER(zend_jit_zval_copy_unref_helper); REGISTER_HELPER(zend_jit_new_ref_helper); REGISTER_HELPER(zend_jit_fetch_global_helper); - REGISTER_HELPER(zend_jit_verify_arg_object); - REGISTER_HELPER(zend_jit_verify_arg_slow); #undef REGISTER_HELPER zend_elf_load_symbols(); diff --git a/ext/opcache/jit/zend_jit_helpers.c b/ext/opcache/jit/zend_jit_helpers.c index f27976ae909bb..88c50a7da34f4 100644 --- a/ext/opcache/jit/zend_jit_helpers.c +++ b/ext/opcache/jit/zend_jit_helpers.c @@ -401,66 +401,6 @@ static zval* ZEND_FASTCALL zend_jit_fetch_global_helper(zend_execute_data *execu return value; } -static void ZEND_FASTCALL zend_jit_verify_arg_object(zval *arg, zend_op_array *op_array, uint32_t arg_num, zend_arg_info *arg_info, void **cache_slot) -{ - zend_class_entry *ce; - if (EXPECTED(*cache_slot)) { - ce = (zend_class_entry *)*cache_slot; - } else { - ce = zend_fetch_class(arg_info->class_name, (ZEND_FETCH_CLASS_AUTO | ZEND_FETCH_CLASS_NO_AUTOLOAD)); - if (UNEXPECTED(!ce)) { - zend_verify_arg_error(op_array, arg_info, arg_num, NULL, arg); - return; - } - *cache_slot = (void *)ce; - } - if (UNEXPECTED(!instanceof_function(Z_OBJCE_P(arg), ce))) { - zend_verify_arg_error(op_array, arg_info, arg_num, ce, arg); - } -} - -static void ZEND_FASTCALL zend_jit_verify_arg_slow(zval *arg, zend_op_array *op_array, uint32_t arg_num, zend_arg_info *arg_info, void **cache_slot, zval *default_value) -{ - zend_class_entry *ce = NULL; - - if (Z_TYPE_P(arg) == IS_NULL && - (arg_info->allow_null || (default_value && is_null_constant(op_array->scope, default_value)))) { - /* Null passed to nullable type */ - return; - } - - if (UNEXPECTED(arg_info->class_name)) { - /* This is always an error - we fetch the class name for the error message here */ - if (EXPECTED(*cache_slot)) { - ce = (zend_class_entry *) *cache_slot; - } else { - ce = zend_fetch_class(arg_info->class_name, (ZEND_FETCH_CLASS_AUTO | ZEND_FETCH_CLASS_NO_AUTOLOAD)); - if (ce) { - *cache_slot = (void *)ce; - } - } - goto err; - } else if (arg_info->type_hint == IS_CALLABLE) { - if (zend_is_callable(arg, IS_CALLABLE_CHECK_SILENT, NULL) == 0) { - goto err; - } - } else if (arg_info->type_hint == IS_ITERABLE) { - if (zend_is_iterable(arg) == 0) { - goto err; - } - } else if (arg_info->type_hint == _IS_BOOL && - EXPECTED(Z_TYPE_P(arg) == IS_FALSE || Z_TYPE_P(arg) == IS_TRUE)) { - return; - } else { - if (zend_verify_scalar_type_hint(arg_info->type_hint, arg, ZEND_RET_USES_STRICT_TYPES()) == 0) { - goto err; - } - } - return; -err: - zend_verify_arg_error(op_array, arg_info, arg_num, ce, arg); -} - /* * Local variables: * tab-width: 4 diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index dd5b24733a7d8..3831eabe6845c 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -3694,9 +3694,9 @@ static int zend_jit_bind_global(dasm_State **Dst, zend_op *opline, zend_op_array | call &memcmp |.else | sub r4, 4 - | push Z_STRLEN_P(varname) - | push Z_STRVAL_P(varname) | push r1 + | push Z_STRVAL_P(varname) + | push Z_STRLEN_P(varname) | call &memcmp | add r4, 16 |.endif @@ -3790,113 +3790,6 @@ static int zend_jit_bind_global(dasm_State **Dst, zend_op *opline, zend_op_array return 1; } -static int zend_jit_recv_init(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_bool is_last) { - zend_arg_info *arg_info; - zend_bool has_slow = 0; - uint32_t arg_num = opline->op1.num; - zval *zv = RT_CONSTANT(op_array, opline->op2); - - | cmp dword EX->This.u2.num_args, arg_num - | jae >4 - | ZVAL_COPY_CONST FP + opline->result.var, -1, zv, r0 - if (Z_REFCOUNTED_P(zv)) { - | ADDREF_CONST zv, r0 - } - if (Z_CONSTANT_P(zv)) { - has_slow = 1; - | LOAD_ADDR IP, opline - |.if X64 - | lea CARG1, [FP + opline->result.var] - | mov r0, EX->func - | mov CARG2, [r0 + offsetof(zend_op_array, scope)] - | EXT_CALL zval_update_constant_ex, r0 - |.else - | sub r4, 8 - | mov r0, EX->func - | push dword [r0 + offsetof(zend_op_array, scope)] - | lea r0, [FP + opline->result.var] - | push r0 - | EXT_CALL zval_update_constant_ex, r0 - | add r4, 16 - |.endif - | test al, al - | jnz >7 - } - |4: - if (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) { - do { - if (arg_num <= op_array->num_args) { - arg_info = &op_array->arg_info[arg_num-1]; - } else if (op_array->fn_flags & ZEND_ACC_VARIADIC) { - arg_info = &op_array->arg_info[op_array->num_args]; - } else { - break; - } - if (!arg_info->type_hint) { - break; - } - has_slow = 2; - | mov r0, aword [FP + opline->result.var] - | ZVAL_DEREF r0, MAY_BE_REF - | cmp byte [r0 + 8], arg_info->type_hint - | jne >8 - if (arg_info->class_name) { - | mov FCARG1a, r0 - | mov r0, EX->run_time_cache - | mov r0, aword [r0 + Z_CACHE_SLOT_P(zv)] - | LOAD_ADDR FCARG2a, (ptrdiff_t)op_array - |.if X64 - | mov CARG3, arg_num - | LOAD_ADDR CARG4, (ptrdiff_t)arg_info - | mov CARG5, r0 - |.else - | push arg_num - | push (ptrdiff_t)arg_info - | push r0 - |.endif - | EXT_CALL zend_jit_verify_arg_object, r0 - } - } while (0); - } - |9: - if (is_last) { - | LOAD_ADDR IP, (opline + 1) - } - if (Z_CONSTANT_P(zv)) { - zend_jit_check_exception(Dst); - } - - if (has_slow) { - |.cold_code - |7: - | ZVAL_PTR_DTOR FP + opline->result.var, -1, 0, (op_array->filename ? ZSTR_VAL(op_array->filename) : NULL), opline->lineno - | SET_Z_TYPE_INFO FP + opline->result.var, IS_UNDEF - | jmp <4 - if (has_slow > 1) { - |8: - | mov FCARG1a, r0 - | mov r0, EX->run_time_cache - | mov r0, aword [r0 + Z_CACHE_SLOT_P(zv)] - | LOAD_ADDR FCARG2a, (ptrdiff_t)op_array - |.if X64 - | mov CARG3, arg_num - | LOAD_ADDR CARG4, (ptrdiff_t)arg_info - | mov CARG5, r0 - | mov CARG6, (ptrdiff_t)zv - |.else - | push arg_num - | push (ptrdiff_t)arg_info - | push r0 - | push zv - |.endif - | EXT_CALL zend_jit_verify_arg_slow, r0 - | jmp <9 - } - |.code - } - - return 1; -} #endif /* From d438ea40503cc6f7c35cfad296b2560f0e4bc02c Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Wed, 21 Sep 2016 17:21:31 +0800 Subject: [PATCH 230/569] Implemented JIT for RECV_INIT --- ext/opcache/jit/zend_jit.c | 15 +- ext/opcache/jit/zend_jit_disasm_x86.c | 2 + ext/opcache/jit/zend_jit_helpers.c | 237 ++++++++++++++++++++++++++ ext/opcache/jit/zend_jit_x86.dasc | 114 ++++++++++++- 4 files changed, 355 insertions(+), 13 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 1c5ffaf2788d3..682f116d957d9 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -725,11 +725,7 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) } else if (ssa->cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_RECV_ENTRY)) { opline = op_array->opcodes + ssa->cfg.blocks[b].start; if (ssa->cfg.split_at_recv && opline->opcode == ZEND_RECV_INIT) { - if (opline > op_array->opcodes && - (opline-1)->opcode == ZEND_RECV_INIT) { - /* repeatable opcode */ - continue; - } else { + if (opline == op_array->opcodes || (opline-1)->opcode != ZEND_RECV_INIT) { for (i = 0; (opline+i)->opcode == ZEND_RECV_INIT; i++) { } zend_jit_jmp(&dasm_state, b + i); @@ -889,13 +885,10 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) break; #endif case ZEND_RECV_INIT: - if (ssa->cfg.split_at_recv) { - if (!zend_jit_handler(&dasm_state, opline, zend_may_throw(opline, op_array, ssa))) { - goto jit_failure; - } - break; + if (!zend_jit_recv_init(&dasm_state, opline, op_array, (opline + 1)->opcode != ZEND_RECV_INIT, ssa)) { + goto jit_failure; } - /* break missing intentionally */ + break; #if ZEND_JIT_LEVEL < ZEND_JIT_LEVEL_OPT_FUNC case ZEND_BIND_GLOBAL: #endif diff --git a/ext/opcache/jit/zend_jit_disasm_x86.c b/ext/opcache/jit/zend_jit_disasm_x86.c index 39f28373a73d0..5093d1417f4ac 100644 --- a/ext/opcache/jit/zend_jit_disasm_x86.c +++ b/ext/opcache/jit/zend_jit_disasm_x86.c @@ -397,6 +397,8 @@ static int zend_jit_disasm_init(void) REGISTER_HELPER(zend_jit_zval_copy_unref_helper); REGISTER_HELPER(zend_jit_new_ref_helper); REGISTER_HELPER(zend_jit_fetch_global_helper); + REGISTER_HELPER(zend_jit_verify_arg_object); + REGISTER_HELPER(zend_jit_verify_arg_slow); #undef REGISTER_HELPER zend_elf_load_symbols(); diff --git a/ext/opcache/jit/zend_jit_helpers.c b/ext/opcache/jit/zend_jit_helpers.c index 88c50a7da34f4..fd04e079d2f6a 100644 --- a/ext/opcache/jit/zend_jit_helpers.c +++ b/ext/opcache/jit/zend_jit_helpers.c @@ -401,6 +401,243 @@ static zval* ZEND_FASTCALL zend_jit_fetch_global_helper(zend_execute_data *execu return value; } +static int is_null_constant(zend_class_entry *scope, zval *default_value) +{ + if (Z_CONSTANT_P(default_value)) { + zval constant; + + ZVAL_COPY(&constant, default_value); + if (UNEXPECTED(zval_update_constant_ex(&constant, scope) != SUCCESS)) { + return 0; + } + if (Z_TYPE(constant) == IS_NULL) { + return 1; + } + zval_ptr_dtor(&constant); + } + return 0; +} + +static zend_bool zend_verify_weak_scalar_type_hint(zend_uchar type_hint, zval *arg) +{ + switch (type_hint) { + case _IS_BOOL: { + zend_bool dest; + + if (!zend_parse_arg_bool_weak(arg, &dest)) { + return 0; + } + zval_ptr_dtor(arg); + ZVAL_BOOL(arg, dest); + return 1; + } + case IS_LONG: { + zend_long dest; + + if (!zend_parse_arg_long_weak(arg, &dest)) { + return 0; + } + zval_ptr_dtor(arg); + ZVAL_LONG(arg, dest); + return 1; + } + case IS_DOUBLE: { + double dest; + + if (!zend_parse_arg_double_weak(arg, &dest)) { + return 0; + } + zval_ptr_dtor(arg); + ZVAL_DOUBLE(arg, dest); + return 1; + } + case IS_STRING: { + zend_string *dest; + + /* on success "arg" is converted to IS_STRING */ + if (!zend_parse_arg_str_weak(arg, &dest)) { + return 0; + } + return 1; + } + default: + return 0; + } +} + +static zend_bool zend_verify_scalar_type_hint(zend_uchar type_hint, zval *arg, zend_bool strict) +{ + if (UNEXPECTED(strict)) { + /* SSTH Exception: IS_LONG may be accepted as IS_DOUBLE (converted) */ + if (type_hint != IS_DOUBLE || Z_TYPE_P(arg) != IS_LONG) { + return 0; + } + } else if (UNEXPECTED(Z_TYPE_P(arg) == IS_NULL)) { + /* NULL may be accepted only by nullable hints (this is already checked) */ + return 0; + } + return zend_verify_weak_scalar_type_hint(type_hint, arg); +} + +static ZEND_COLD void zend_verify_type_error_common( + const zend_function *zf, const zend_arg_info *arg_info, + const zend_class_entry *ce, zval *value, + const char **fname, const char **fsep, const char **fclass, + const char **need_msg, const char **need_kind, const char **need_or_null, + const char **given_msg, const char **given_kind) +{ + zend_bool is_interface = 0; + *fname = ZSTR_VAL(zf->common.function_name); + + if (zf->common.scope) { + *fsep = "::"; + *fclass = ZSTR_VAL(zf->common.scope->name); + } else { + *fsep = ""; + *fclass = ""; + } + + switch (arg_info->type_hint) { + case IS_OBJECT: + if (ce) { + if (ce->ce_flags & ZEND_ACC_INTERFACE) { + *need_msg = "implement interface "; + is_interface = 1; + } else { + *need_msg = "be an instance of "; + } + *need_kind = ZSTR_VAL(ce->name); + } else { + /* We don't know whether it's a class or interface, assume it's a class */ + *need_msg = "be an instance of "; + *need_kind = zf->common.type == ZEND_INTERNAL_FUNCTION + ? ((zend_internal_arg_info *) arg_info)->class_name + : ZSTR_VAL(arg_info->class_name); + } + break; + case IS_CALLABLE: + *need_msg = "be callable"; + *need_kind = ""; + break; + case IS_ITERABLE: + *need_msg = "be iterable"; + *need_kind = ""; + break; + default: + *need_msg = "be of the type "; + *need_kind = zend_get_type_by_const(arg_info->type_hint); + break; + } + + if (arg_info->allow_null) { + *need_or_null = is_interface ? " or be null" : " or null"; + } else { + *need_or_null = ""; + } + + if (value) { + if (arg_info->type_hint == IS_OBJECT && Z_TYPE_P(value) == IS_OBJECT) { + *given_msg = "instance of "; + *given_kind = ZSTR_VAL(Z_OBJCE_P(value)->name); + } else { + *given_msg = zend_zval_type_name(value); + *given_kind = ""; + } + } else { + *given_msg = "none"; + *given_kind = ""; + } +} + +static ZEND_COLD void zend_verify_arg_error( + const zend_function *zf, const zend_arg_info *arg_info, + int arg_num, const zend_class_entry *ce, zval *value) +{ + zend_execute_data *ptr = EG(current_execute_data)->prev_execute_data; + const char *fname, *fsep, *fclass; + const char *need_msg, *need_kind, *need_or_null, *given_msg, *given_kind; + + if (value) { + zend_verify_type_error_common( + zf, arg_info, ce, value, + &fname, &fsep, &fclass, &need_msg, &need_kind, &need_or_null, &given_msg, &given_kind); + + if (zf->common.type == ZEND_USER_FUNCTION) { + if (ptr && ptr->func && ZEND_USER_CODE(ptr->func->common.type)) { + zend_type_error("Argument %d passed to %s%s%s() must %s%s%s, %s%s given, called in %s on line %d", + arg_num, fclass, fsep, fname, need_msg, need_kind, need_or_null, given_msg, given_kind, + ZSTR_VAL(ptr->func->op_array.filename), ptr->opline->lineno); + } else { + zend_type_error("Argument %d passed to %s%s%s() must %s%s%s, %s%s given", arg_num, fclass, fsep, fname, need_msg, need_kind, need_or_null, given_msg, given_kind); + } + } else { + zend_type_error("Argument %d passed to %s%s%s() must %s%s%s, %s%s given", arg_num, fclass, fsep, fname, need_msg, need_kind, need_or_null, given_msg, given_kind); + } + } else { + zend_missing_arg_error(ptr); + } +} + +static void ZEND_FASTCALL zend_jit_verify_arg_object(zval *arg, zend_op_array *op_array, uint32_t arg_num, zend_arg_info *arg_info, void **cache_slot) +{ + zend_class_entry *ce; + if (EXPECTED(*cache_slot)) { + ce = (zend_class_entry *)*cache_slot; + } else { + ce = zend_fetch_class(arg_info->class_name, (ZEND_FETCH_CLASS_AUTO | ZEND_FETCH_CLASS_NO_AUTOLOAD)); + if (UNEXPECTED(!ce)) { + zend_verify_arg_error((zend_function*)op_array, arg_info, arg_num, NULL, arg); + return; + } + *cache_slot = (void *)ce; + } + if (UNEXPECTED(!instanceof_function(Z_OBJCE_P(arg), ce))) { + zend_verify_arg_error((zend_function*)op_array, arg_info, arg_num, ce, arg); + } +} + +static void ZEND_FASTCALL zend_jit_verify_arg_slow(zval *arg, zend_op_array *op_array, uint32_t arg_num, zend_arg_info *arg_info, void **cache_slot, zval *default_value) +{ + zend_class_entry *ce = NULL; + + if (Z_TYPE_P(arg) == IS_NULL && + (arg_info->allow_null || (default_value && is_null_constant(op_array->scope, default_value)))) { + /* Null passed to nullable type */ + return; + } + + if (UNEXPECTED(arg_info->class_name)) { + /* This is always an error - we fetch the class name for the error message here */ + if (EXPECTED(*cache_slot)) { + ce = (zend_class_entry *) *cache_slot; + } else { + ce = zend_fetch_class(arg_info->class_name, (ZEND_FETCH_CLASS_AUTO | ZEND_FETCH_CLASS_NO_AUTOLOAD)); + if (ce) { + *cache_slot = (void *)ce; + } + } + goto err; + } else if (arg_info->type_hint == IS_CALLABLE) { + if (zend_is_callable(arg, IS_CALLABLE_CHECK_SILENT, NULL) == 0) { + goto err; + } + } else if (arg_info->type_hint == IS_ITERABLE) { + if (zend_is_iterable(arg) == 0) { + goto err; + } + } else if (arg_info->type_hint == _IS_BOOL && + EXPECTED(Z_TYPE_P(arg) == IS_FALSE || Z_TYPE_P(arg) == IS_TRUE)) { + return; + } else { + if (zend_verify_scalar_type_hint(arg_info->type_hint, arg, ZEND_RET_USES_STRICT_TYPES()) == 0) { + goto err; + } + } + return; +err: + zend_verify_arg_error((zend_function*)op_array, arg_info, arg_num, ce, arg); +} + /* * Local variables: * tab-width: 4 diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 3831eabe6845c..63075890a1218 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -3694,9 +3694,9 @@ static int zend_jit_bind_global(dasm_State **Dst, zend_op *opline, zend_op_array | call &memcmp |.else | sub r4, 4 - | push r1 - | push Z_STRVAL_P(varname) | push Z_STRLEN_P(varname) + | push Z_STRVAL_P(varname) + | push r1 | call &memcmp | add r4, 16 |.endif @@ -3790,6 +3790,116 @@ static int zend_jit_bind_global(dasm_State **Dst, zend_op *opline, zend_op_array return 1; } +static int zend_jit_recv_init(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_bool is_last, zend_ssa *ssa) { + zend_arg_info *arg_info; + zend_bool has_slow = 0; + uint32_t arg_num = opline->op1.num; + zval *zv = RT_CONSTANT(op_array, opline->op2); + + | cmp dword EX->This.u2.num_args, arg_num + | jae >4 + | ZVAL_COPY_CONST FP + opline->result.var, -1, zv, r0 + if (Z_REFCOUNTED_P(zv)) { + | ADDREF_CONST zv, r0 + } + if (Z_CONSTANT_P(zv)) { + has_slow = 1; + |.if X64 + | lea CARG1, [FP + opline->result.var] + | mov r0, EX->func + | mov CARG2, [r0 + offsetof(zend_op_array, scope)] + | EXT_CALL zval_update_constant_ex, r0 + |.else + | sub r4, 8 + | mov r0, EX->func + | push dword [r0 + offsetof(zend_op_array, scope)] + | lea r0, [FP + opline->result.var] + | push r0 + | EXT_CALL zval_update_constant_ex, r0 + | add r4, 16 + |.endif + | test al, al + | jnz >7 + } + |4: + if (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) { + do { + if (arg_num <= op_array->num_args) { + arg_info = &op_array->arg_info[arg_num-1]; + } else if (op_array->fn_flags & ZEND_ACC_VARIADIC) { + arg_info = &op_array->arg_info[op_array->num_args]; + } else { + break; + } + if (!arg_info->type_hint) { + break; + } + has_slow += 2; + | lea r0, aword [FP + opline->result.var] + | ZVAL_DEREF r0, MAY_BE_REF + | cmp byte [r0 + 8], arg_info->type_hint + | jne >8 + if (arg_info->class_name) { + | mov FCARG1a, r0 + | mov r0, EX->run_time_cache + | lea r0, [r0 + Z_CACHE_SLOT_P(zv)] + | LOAD_ADDR FCARG2a, (ptrdiff_t)op_array + |.if X64 + | mov CARG3, arg_num + | LOAD_ADDR CARG4, (ptrdiff_t)arg_info + | mov CARG5, r0 + |.else + | push r0 + | push (ptrdiff_t)arg_info + | push arg_num + |.endif + | EXT_CALL zend_jit_verify_arg_object, r0 + } + } while (0); + } + |9: + if (zend_may_throw(opline, op_array, ssa)) { + if (!zend_jit_check_exception(Dst)) { + return 0; + } + } + if (is_last) { + | LOAD_ADDR IP, (opline + 1) + } + + if (has_slow) { + |.cold_code + if (has_slow & 1) { + |7: + | ZVAL_PTR_DTOR FP + opline->result.var, -1, 0, (op_array->filename ? ZSTR_VAL(op_array->filename) : NULL), opline->lineno + | SET_Z_TYPE_INFO FP + opline->result.var, IS_UNDEF + | jmp <4 + } + if (has_slow & 2) { + |8: + | mov FCARG1a, r0 + | mov r0, EX->run_time_cache + | lea r0, [r0 + Z_CACHE_SLOT_P(zv)] + | LOAD_ADDR FCARG2a, (ptrdiff_t)op_array + |.if X64 + | mov CARG3, arg_num + | LOAD_ADDR CARG4, (ptrdiff_t)arg_info + | mov CARG5, r0 + | LOAD_ADDR CARG6, zv + |.else + | push zv + | push r0 + | push (ptrdiff_t)arg_info + | push arg_num + |.endif + | EXT_CALL zend_jit_verify_arg_slow, r0 + | jmp <9 + } + |.code + } + + return 1; +} #endif /* From 2e6e3757506013c5a742c40b67954f8a14f51a71 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 21 Sep 2016 14:02:26 +0300 Subject: [PATCH 231/569] Move call to zend_is_true() back to cold part (we may need to add "fast code" for some missing cases, e.g IS_NULL|IS_FALSE|IS_STRING) --- ext/opcache/jit/zend_jit_x86.dasc | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 63075890a1218..a0ffcd7d55b10 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -2421,6 +2421,10 @@ static int zend_jit_jmpznz(dasm_State **Dst, zend_op *opline, int b, zend_op_arr } else { if (false_label != (uint32_t)-1) { | jl =>false_label + if (!(op1_info & MAY_BE_LONG) && + (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)))) { + | jmp >2 + } } else if (op1_info & (MAY_BE_ANY - (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) { if (op1_info & MAY_BE_LONG) { | jl >9 @@ -2449,6 +2453,7 @@ static int zend_jit_jmpznz(dasm_State **Dst, zend_op *opline, int b, zend_op_arr if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG))) { if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { + |.cold_code |2: } | SAVE_VALID_OPLINE @@ -2486,6 +2491,10 @@ static int zend_jit_jmpznz(dasm_State **Dst, zend_op *opline, int b, zend_op_arr | jmp >9 } } + + if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { + |.code + } } |9: From 1bd13c5a67f1f662e9ca5bf182eb3c65ead8503b Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 21 Sep 2016 16:11:53 +0300 Subject: [PATCH 232/569] Keep code for (ZEND_JIT_LEVEL < ZEND_JIT_LEVEL_OPT_FUNC) and remove useless "jmp" --- ext/opcache/jit/zend_jit.c | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 682f116d957d9..a8f314a3ff823 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -725,10 +725,17 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) } else if (ssa->cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_RECV_ENTRY)) { opline = op_array->opcodes + ssa->cfg.blocks[b].start; if (ssa->cfg.split_at_recv && opline->opcode == ZEND_RECV_INIT) { - if (opline == op_array->opcodes || (opline-1)->opcode != ZEND_RECV_INIT) { - for (i = 0; (opline+i)->opcode == ZEND_RECV_INIT; i++) { + if (opline > op_array->opcodes && + (opline-1)->opcode == ZEND_RECV_INIT) { +#if ZEND_JIT_LEVEL < ZEND_JIT_LEVEL_OPT_FUNC + /* repeatable opcode */ + zend_jit_label(&dasm_state, b); + continue; +#endif + } else { + if (opline != op_array->opcodes) { + zend_jit_jmp(&dasm_state, 1); } - zend_jit_jmp(&dasm_state, b + i); zend_jit_label(&dasm_state, ssa->cfg.blocks_count + b); for (i = 1; (opline+i)->opcode == ZEND_RECV_INIT; i++) { zend_jit_label(&dasm_state, ssa->cfg.blocks_count + b + i); @@ -883,15 +890,21 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) goto jit_failure; } break; -#endif case ZEND_RECV_INIT: if (!zend_jit_recv_init(&dasm_state, opline, op_array, (opline + 1)->opcode != ZEND_RECV_INIT, ssa)) { goto jit_failure; } break; -#if ZEND_JIT_LEVEL < ZEND_JIT_LEVEL_OPT_FUNC +#else + case ZEND_RECV_INIT: + if (ssa->cfg.split_at_recv) { + if (!zend_jit_handler(&dasm_state, opline, zend_may_throw(opline, op_array, ssa))) { + goto jit_failure; + } + break; + } + /* break missing intentionally */ case ZEND_BIND_GLOBAL: -#endif if (opline->opcode != op_array->opcodes[i+1].opcode) { /* repeatable opcodes */ if (!zend_jit_handler(&dasm_state, opline, zend_may_throw(opline, op_array, ssa))) { @@ -899,6 +912,7 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) } } break; +#endif case ZEND_NOP: if (!zend_jit_skip_handler(&dasm_state)) { goto jit_failure; From e258efe8056662659816bb4194b3ff19f0abbef8 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 21 Sep 2016 16:49:14 +0300 Subject: [PATCH 233/569] Fixed JIT for JMPZNZ after SMART_BRANCH opcodes (IS_IDENTICAL and family) --- ext/opcache/jit/zend_jit.c | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index a8f314a3ff823..01e52de0ca2aa 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -862,15 +862,16 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) (opline-1)->opcode == ZEND_IS_SMALLER_OR_EQUAL || (opline-1)->opcode == ZEND_CASE) { /* skip */ - } else if ((opline-1)->opcode == ZEND_IS_IDENTICAL || - (opline-1)->opcode == ZEND_IS_NOT_IDENTICAL || - (opline-1)->opcode == ZEND_ISSET_ISEMPTY_VAR || - (opline-1)->opcode == ZEND_ISSET_ISEMPTY_STATIC_PROP || - (opline-1)->opcode == ZEND_ISSET_ISEMPTY_DIM_OBJ || - (opline-1)->opcode == ZEND_ISSET_ISEMPTY_PROP_OBJ || - (opline-1)->opcode == ZEND_INSTANCEOF || - (opline-1)->opcode == ZEND_TYPE_CHECK || - (opline-1)->opcode == ZEND_DEFINED) { + } else if ((opline->opcode != ZEND_JMPZNZ) && + ((opline-1)->opcode == ZEND_IS_IDENTICAL || + (opline-1)->opcode == ZEND_IS_NOT_IDENTICAL || + (opline-1)->opcode == ZEND_ISSET_ISEMPTY_VAR || + (opline-1)->opcode == ZEND_ISSET_ISEMPTY_STATIC_PROP || + (opline-1)->opcode == ZEND_ISSET_ISEMPTY_DIM_OBJ || + (opline-1)->opcode == ZEND_ISSET_ISEMPTY_PROP_OBJ || + (opline-1)->opcode == ZEND_INSTANCEOF || + (opline-1)->opcode == ZEND_TYPE_CHECK || + (opline-1)->opcode == ZEND_DEFINED)) { /* smart branch */ if (!zend_jit_cond_jmp(&dasm_state, opline + 1, ssa->cfg.blocks[b].successors[0])) { goto jit_failure; From 1bea2513ede05a2d73742b603ae588ce33ddc96b Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 21 Sep 2016 18:39:35 +0300 Subject: [PATCH 234/569] context-threaded JIT (incomplete) --- ext/opcache/jit/zend_jit.c | 19 ++++++++----------- ext/opcache/jit/zend_jit_x86.dasc | 16 ++++++++++++++-- 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 01e52de0ca2aa..2ea067c4b2ff9 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -32,7 +32,7 @@ #include "Optimizer/zend_call_graph.h" #include "Optimizer/zend_dump.h" -//#define CONTEXT_THREDED_JIT +//#define CONTEXT_THREADED_JIT #define PREFER_MAP_32BIT //#define ZEND_RUNTIME_JIT @@ -143,8 +143,8 @@ static void *dasm_link_and_encode(dasm_State **dasm_state, int b; for (b = 0; b < cfg->blocks_count; b++) { -#ifdef CONTEXT_THREDED_JIT - if (cfg->blocks[b].flags & ZEND_BB_START) { +#ifdef CONTEXT_THREADED_JIT + if (cfg->blocks[b].flags & (ZEND_BB_START|ZEND_BB_RECV_ENTRY)) { #else if (cfg->blocks[b].flags & (ZEND_BB_START|ZEND_BB_ENTRY|ZEND_BB_RECV_ENTRY)) { #endif @@ -710,19 +710,16 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) if ((ssa->cfg.blocks[b].flags & ZEND_BB_REACHABLE) == 0) { continue; } -#ifdef CONTEXT_THREDED_JIT - if (ssa->cfg.blocks[b].flags & ZEND_BB_START) { - zend_jit_label(&dasm_state, ssa->cfg.blocks_count + b); - zend_jit_prologue(&dasm_state); - } -#else +#ifndef CONTEXT_THREADED_JIT if (ssa->cfg.blocks[b].flags & ZEND_BB_ENTRY) { if (ssa->cfg.blocks[b].flags & ZEND_BB_TARGET) { zend_jit_jmp(&dasm_state, b); } zend_jit_label(&dasm_state, ssa->cfg.blocks_count + b); zend_jit_prologue(&dasm_state); - } else if (ssa->cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_RECV_ENTRY)) { + } else +#endif + if (ssa->cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_RECV_ENTRY)) { opline = op_array->opcodes + ssa->cfg.blocks[b].start; if (ssa->cfg.split_at_recv && opline->opcode == ZEND_RECV_INIT) { if (opline > op_array->opcodes && @@ -750,7 +747,7 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) zend_jit_prologue(&dasm_state); } } -#endif + zend_jit_label(&dasm_state, b); if (ssa->cfg.blocks[b].flags & ZEND_BB_TARGET) { if (!zend_jit_set_opline(&dasm_state, op_array->opcodes + ssa->cfg.blocks[b].start)) { diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index a0ffcd7d55b10..496ee17986bbf 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -1129,7 +1129,7 @@ static int zend_jit_smart_branch(dasm_State **Dst, zend_op *opline, unsigned int return 1; } -#ifdef CONTEXT_THREDED_JIT +#ifdef CONTEXT_THREADED_JIT static int zend_jit_context_threaded_call(dasm_State **Dst, zend_op *opline) { if (!zend_jit_handler(Dst, opline, 1)) return 0; @@ -1151,7 +1151,7 @@ static int zend_jit_context_threaded_call(dasm_State **Dst, zend_op *opline) static int zend_jit_call(dasm_State **Dst, zend_op *opline) { -#ifdef CONTEXT_THREDED_JIT +#ifdef CONTEXT_THREADED_JIT return zend_jit_context_threaded_call(Dst, opline); #else return zend_jit_tail_handler(Dst, opline); @@ -2880,10 +2880,18 @@ static int zend_jit_do_fcall(dasm_State **Dst, zend_op *opline, zend_op_array *o | mov EX:r1->opline, IP || if (func && op_array == &func->op_array) { || /* recursive call */ +#ifdef CONTEXT_THREADED_JIT + | call =>(call_info->num_args+ssa->cfg.blocks_count) +#else | jmp =>call_info->num_args +#endif || } else { +#ifdef CONTEXT_THREADED_JIT + | call aword [IP] +#else | add r4, SPAD // stack alignment | jmp aword [IP] +#endif || } return 1; @@ -3172,7 +3180,11 @@ static int zend_jit_leave_func(dasm_State **Dst, zend_op *opline, zend_op_array | add IP, sizeof(zend_op) | mov EX->opline, IP | add r4, SPAD // stack alignment +#ifdef CONTEXT_THREADED_JIT + | ret +#else | jmp aword [IP] +#endif return 1; } From 5969c61ca00b3d3d747f178f39d184c6b397054c Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 22 Sep 2016 17:23:00 +0300 Subject: [PATCH 235/569] Resolve "memcmp" --- ext/opcache/jit/zend_jit_disasm_x86.c | 1 + 1 file changed, 1 insertion(+) diff --git a/ext/opcache/jit/zend_jit_disasm_x86.c b/ext/opcache/jit/zend_jit_disasm_x86.c index 5093d1417f4ac..0ee2a0ab2e1bd 100644 --- a/ext/opcache/jit/zend_jit_disasm_x86.c +++ b/ext/opcache/jit/zend_jit_disasm_x86.c @@ -382,6 +382,7 @@ static int zend_jit_disasm_init(void) #define REGISTER_HELPER(n) \ zend_jit_disasm_add_symbol(#n, \ (uint64_t)(uintptr_t)n, sizeof(void*)); + REGISTER_HELPER(memcmp); REGISTER_HELPER(zend_jit_find_func_helper); REGISTER_HELPER(zend_jit_extend_stack_helper); REGISTER_HELPER(zend_jit_leave_nested_func_helper); From a2033e8f444ab38f7513c68bfc55fc84416b9ed8 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 22 Sep 2016 18:18:38 +0300 Subject: [PATCH 236/569] Eliminated useless "jmp" --- ext/opcache/jit/zend_jit_x86.dasc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 496ee17986bbf..c9731734d900a 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -3439,7 +3439,7 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, zend_op *opline, zend_op_ar | SET_Z_TYPE_INFO FP + opline->result.var, IS_NULL | jmp >9 // END |.code - if (op2_info & (MAY_BE_ANY - MAY_BE_STRING)) { + if (op2_info & (MAY_BE_ANY - MAY_BE_LONG)) { | jmp >8 // FOUND } if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_LONG)) { From 4bd52cf132defcf7e5f4eb5f5adb1f8b5701516a Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 22 Sep 2016 18:18:57 +0300 Subject: [PATCH 237/569] Eliminated useless load --- ext/opcache/jit/zend_jit_x86.dasc | 84 +++++++++++++++++++++++++++---- 1 file changed, 73 insertions(+), 11 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index c9731734d900a..5c7d0bc3449c4 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -1770,9 +1770,39 @@ fallback: static int zend_jit_cmp_long_long(dasm_State **Dst, zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa) { unsigned int target_label; - - | LONG_LOAD r0, opline->op1_type, opline->op1 - | LONG_OP cmp, r0, opline->op2_type, opline->op2 + int swap = 0; + + if (opline->op1_type == IS_CONST && opline->op2_type != IS_CONST) { + zend_long val = Z_LVAL_P(RT_CONSTANT(op_array, opline->op1)); + + | .if X64 + || if (!IS_32BIT(val)) { + | mov64 r0, val + | cmp aword [FP + opline->op2.var], r0 + || } else { + | cmp aword [FP + opline->op2.var], val + || } + | .else + | cmp aword [FP + opline->op2.var], val + | .endif + swap = 1; + } else if (opline->op2_type == IS_CONST && opline->op1_type != IS_CONST) { + zend_long val = Z_LVAL_P(RT_CONSTANT(op_array, opline->op2)); + + | .if X64 + || if (!IS_32BIT(val)) { + | mov64 r0, val + | cmp aword [FP + opline->op1.var], r0 + || } else { + | cmp aword [FP + opline->op1.var], val + || } + | .else + | cmp aword [FP + opline->op1.var], val + | .endif + } else { + | LONG_LOAD r0, opline->op1_type, opline->op1 + | LONG_OP cmp, r0, opline->op2_type, opline->op2 + } if ((opline+1)->opcode == ZEND_JMPZ && (opline+1)->op1_type == IS_TMP_VAR && (opline+1)->op1.var == opline->result.var) { @@ -1786,10 +1816,18 @@ static int zend_jit_cmp_long_long(dasm_State **Dst, zend_op *opline, int b, zend | je => target_label break; case ZEND_IS_SMALLER: - | jge => target_label + if (swap) { + | jle => target_label + } else { + | jge => target_label + } break; case ZEND_IS_SMALLER_OR_EQUAL: - | jg => target_label + if (swap) { + | jl => target_label + } else { + | jg => target_label + } break; } } else if ((opline+1)->opcode == ZEND_JMPNZ && @@ -1805,10 +1843,18 @@ static int zend_jit_cmp_long_long(dasm_State **Dst, zend_op *opline, int b, zend | jne => target_label break; case ZEND_IS_SMALLER: - | jl => target_label + if (swap) { + | jg => target_label + } else { + | jl => target_label + } break; case ZEND_IS_SMALLER_OR_EQUAL: - | jle => target_label + if (swap) { + | jge => target_label + } else { + | jle => target_label + } break; } } else if ((opline+1)->opcode == ZEND_JMPZNZ && @@ -1824,10 +1870,18 @@ static int zend_jit_cmp_long_long(dasm_State **Dst, zend_op *opline, int b, zend | je => target_label break; case ZEND_IS_SMALLER: - | jge => target_label + if (swap) { + | jle => target_label + } else { + | jge => target_label + } break; case ZEND_IS_SMALLER_OR_EQUAL: - | jg => target_label + if (swap) { + | jl => target_label + } else { + | jg => target_label + } break; } target_label = ssa->cfg.blocks[b].successors[1]; @@ -1846,12 +1900,20 @@ static int zend_jit_cmp_long_long(dasm_State **Dst, zend_op *opline, int b, zend | add eax, 2 break; case ZEND_IS_SMALLER: - | setl al + if (swap) { + | setg al + } else { + | setl al + } | movzx eax, al | add eax, 2 break; case ZEND_IS_SMALLER_OR_EQUAL: - | setle al + if (swap) { + | setge al + } else { + | setle al + } | movzx eax, al | add eax, 2 break; From 5f025e330bb2e2fe3363ba860a882ed1f0258e6b Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 23 Sep 2016 11:00:58 +0300 Subject: [PATCH 238/569] Eliminated redundand overflow checks --- ext/opcache/jit/zend_jit.c | 104 ++++++++++++++++++++++++++++++ ext/opcache/jit/zend_jit_x86.dasc | 21 +++--- 2 files changed, 115 insertions(+), 10 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 2ea067c4b2ff9..76868fd396b0d 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -56,6 +56,7 @@ static void *dasm_ptr = NULL; static void *dasm_end = NULL; static int zend_may_throw(zend_op *opline, zend_op_array *op_array, zend_ssa *ssa); +static int zend_may_overflow(zend_op *opline, zend_op_array *op_array, zend_ssa *ssa); #include "dynasm/dasm_x86.h" #include "jit/zend_jit_helpers.c" @@ -615,6 +616,109 @@ static int zend_may_throw(zend_op *opline, zend_op_array *op_array, zend_ssa *ss } } +static int zend_may_overflow(zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) +{ + uint32_t num; + int res; + + if (!ssa->ops || !ssa->var_info) { + return 1; + } + switch (opline->opcode) { + case ZEND_PRE_INC: + case ZEND_POST_INC: + num = opline - op_array->opcodes; + res = ssa->ops[num].op1_def; + return (res < 0 || + !ssa->var_info[res].has_range || + ssa->var_info[res].range.overflow); + case ZEND_PRE_DEC: + case ZEND_POST_DEC: + num = opline - op_array->opcodes; + res = ssa->ops[num].op1_def; + return (res < 0 || + !ssa->var_info[res].has_range || + ssa->var_info[res].range.underflow); + case ZEND_ADD: + num = opline - op_array->opcodes; + res = ssa->ops[num].result_def; + if (res < 0 || + !ssa->var_info[res].has_range) { + return 1; + } + if (ssa->var_info[res].range.underflow) { + if (OP1_HAS_RANGE() && OP2_HAS_RANGE()) { + zend_long op1_min, op2_min, res_min; + + op1_min = OP1_MIN_RANGE(); + op2_min = OP2_MIN_RANGE(); + res_min = op1_min + op2_min; + return OP1_RANGE_UNDERFLOW() || + OP2_RANGE_UNDERFLOW() || + (op1_min < 0 && op2_min < 0 && res_min >= 0); + } + return 1; + } + if (ssa->var_info[res].range.overflow) { + if (OP1_HAS_RANGE() && OP2_HAS_RANGE()) { + zend_long op1_max, op2_max, res_max; + + op1_max = OP1_MAX_RANGE(); + op2_max = OP2_MAX_RANGE(); + res_max = op1_max + op2_max; + return OP1_RANGE_OVERFLOW() || + OP2_RANGE_OVERFLOW() || + (op1_max > 0 && op2_max > 0 && res_max <= 0); + } + return 1; + } + return 0; + case ZEND_SUB: + num = opline - op_array->opcodes; + res = ssa->ops[num].result_def; + if (res < 0 || + !ssa->var_info[res].has_range) { + return 1; + } + if (ssa->var_info[res].range.underflow) { + if (OP1_HAS_RANGE() && OP2_HAS_RANGE()) { + zend_long op1_max, op2_min, res_max; + + op1_max = OP1_MAX_RANGE(); + op2_min = OP2_MIN_RANGE(); + res_max = op1_max - op2_min; + return OP1_RANGE_OVERFLOW() || + OP2_RANGE_UNDERFLOW() || + (op1_max > 0 && op2_min < 0 && res_max <= 0); + } + return 1; + } + if (ssa->var_info[res].range.overflow) { + if (OP1_HAS_RANGE() && OP2_HAS_RANGE()) { + zend_long op1_min, op2_max, res_min; + + op1_min = OP1_MIN_RANGE(); + op2_max = OP2_MAX_RANGE(); + res_min = op1_min - op2_max; + return OP1_RANGE_UNDERFLOW() || + OP2_RANGE_OVERFLOW() || + (op1_min < 0 && op2_max > 0 && res_min >= 0); + } + return 1; + } + return 0; + case ZEND_MUL: + num = opline - op_array->opcodes; + res = ssa->ops[num].op1_def; + return (res < 0 || + !ssa->var_info[res].has_range || + ssa->var_info[res].range.underflow || + ssa->var_info[res].range.overflow); + default: + return 1; + } +} + static int zend_jit_op_array_analyze1(zend_op_array *op_array, zend_script *script, zend_ssa *ssa, uint32_t *flags) { if (zend_build_cfg(&CG(arena), op_array, ZEND_CFG_STACKLESS | ZEND_CFG_RECV_ENTRY | ZEND_RT_CONSTANTS | ZEND_CFG_NO_ENTRY_PREDECESSORS | ZEND_SSA_RC_INFERENCE, &ssa->cfg, flags) != SUCCESS) { diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 5c7d0bc3449c4..2f0648cf82a6a 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -1219,7 +1219,8 @@ static int zend_jit_inc_dec(dasm_State **Dst, zend_op *opline, zend_op_array *op | dec aword [FP + opline->op1.var] } op1_def_info = OP1_DEF_INFO(); - if (op1_def_info & MAY_BE_DOUBLE) { + + if ((op1_def_info & MAY_BE_DOUBLE) && zend_may_overflow(opline, op_array, ssa)) { | jo >1 || if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && || opline->result_type != IS_UNUSED) { @@ -1361,15 +1362,7 @@ static int zend_jit_math_long_long(dasm_State **Dst, zend_op *opline, zend_op_ar | LONG_MATH opline->opcode, r0, opline->op2_type, opline->op2 } } - if ((res_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) { - if (res_forward) { - | SET_Z_LVAL r1 + res_forward, r0 - | SET_Z_TYPE_INFO r1 + res_forward, IS_LONG - } else { - | SET_Z_LVAL FP + opline->result.var, r0 - | SET_Z_TYPE_INFO FP + opline->result.var, IS_LONG - } - } else { + if ((res_info & MAY_BE_DOUBLE) && zend_may_overflow(opline, op_array, ssa)) { | jo >1 |.cold_code |1: @@ -1399,6 +1392,14 @@ static int zend_jit_math_long_long(dasm_State **Dst, zend_op *opline, zend_op_ar | SET_Z_TYPE_INFO FP + opline->result.var, IS_LONG } |2: + } else { + if (res_forward) { + | SET_Z_LVAL r1 + res_forward, r0 + | SET_Z_TYPE_INFO r1 + res_forward, IS_LONG + } else { + | SET_Z_LVAL FP + opline->result.var, r0 + | SET_Z_TYPE_INFO FP + opline->result.var, IS_LONG + } } return 1; From 0d627d44a14cdd9f63647e38e2feb3bc02d50737 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 23 Sep 2016 11:53:03 +0300 Subject: [PATCH 239/569] Added TODO comment about mostly uselesd ZEND_CALL_CLOSURE checks --- ext/opcache/jit/zend_jit_x86.dasc | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 2f0648cf82a6a..0db7eeb4f1c18 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -3220,16 +3220,22 @@ static int zend_jit_leave_func(dasm_State **Dst, zend_op *opline, zend_op_array |2: } } - | test FCARG1d, ZEND_CALL_CLOSURE - | jnz >3 - |.cold_code - |3: - | // OBJ_RELEASE((zend_object*)execute_data->func->op_array.prototype); - | mov r0, EX->func - | mov r0, aword [r0 + offsetof(zend_op_array, prototype)] - | OBJ_RELEASE r0, >4 - | jmp >4 - |.code + // TODO: We have to check for ZEND_CALL_CLOSURE flag for every regular + // function, just to support rarely used fake closures created by + // ReflectionFunction::getClosure() ??? + // see ext/reflection/tests/ReflectionFunction_getClosure_basic.php + if (1 || (op_array->fn_flags & ZEND_ACC_CLOSURE)) { + | test FCARG1d, ZEND_CALL_CLOSURE + | jnz >3 + |.cold_code + |3: + | // OBJ_RELEASE((zend_object*)execute_data->func->op_array.prototype); + | mov r0, EX->func + | mov r0, aword [r0 + offsetof(zend_op_array, prototype)] + | OBJ_RELEASE r0, >4 + | jmp >4 + |.code + } |4: | // EG(vm_stack_top) = (zval*)execute_data; | mov aword [&EG(vm_stack_top)], FP From 9d327ac58a40459b60185e946baeba212a9f44a7 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 26 Sep 2016 17:46:50 +0300 Subject: [PATCH 240/569] Avoid extra check for ZEND_CALL_CLOSURE on each regular function return. These checks were necessary to support "fake" closures, that are handled now by "slow" JIT$$leave_function helper. --- ext/opcache/jit/zend_jit_x86.dasc | 147 ++++++++++++++++-------------- 1 file changed, 78 insertions(+), 69 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 0db7eeb4f1c18..737c7231385be 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -3166,81 +3166,90 @@ static int zend_jit_leave_func(dasm_State **Dst, zend_op *opline, zend_op_array return 0; } - | movzx FCARG1d, byte [FP + offsetof(zend_execute_data, This.u1.type_info) + 2] - | test FCARG1d, (ZEND_CALL_TOP|ZEND_CALL_HAS_SYMBOL_TABLE|ZEND_CALL_FREE_EXTRA_ARGS|ZEND_CALL_ALLOCATED) + | movzx FCARG1d, word [FP + offsetof(zend_execute_data, This.u1.type_info) + 2] + | test FCARG1d, (ZEND_CALL_TOP|ZEND_CALL_HAS_SYMBOL_TABLE|ZEND_CALL_FREE_EXTRA_ARGS|ZEND_CALL_ALLOCATED|ZEND_CALL_FAKE_CLOSURE) | jnz ->leave_function_handler - | // EG(current_execute_data) = EX(prev_execute_data); - | mov r0, EX->prev_execute_data - | mov aword [&EG(current_execute_data)], r0 - | // if (call_info & ZEND_CALL_RELEASE_THIS) - if (op_array->scope) { - | test FCARG1d, ZEND_CALL_RELEASE_THIS - if (op_array->fn_flags & ZEND_ACC_STATIC) { - | jnz >1 - |.cold_code - |1: - } else { - | je >2 - } - | // zend_object *object = Z_OBJ(execute_data->This); - | mov r0, EX->This.value.obj - if (op_array->scope && op_array->scope->constructor == (zend_function*)op_array) { - | // if (UNEXPECTED(EG(exception) != NULL) - | cmp aword [&EG(exception)], 0 - | jne >6 - |.cold_code - |6: - | // if (call_info & ZEND_CALL_CTOR) - | test FCARG1d, ZEND_CALL_CTOR - | jz >5 - | // GC_REFCOUNT(object)--; - | GC_DELREF r0 - | // zend_object_store_ctor_failed(object); - |.if X64 - | mov CARG1, r0 - | EXT_CALL zend_object_store_ctor_failed, r0 - |.else - | sub r4, 12 - | push r0 - | EXT_CALL zend_object_store_ctor_failed, r0 - | add r4, 16 - |.endif - | // reload registers + + if (op_array->scope || (op_array->fn_flags & ZEND_ACC_CLOSURE)) { + | // EG(current_execute_data) = EX(prev_execute_data); + | mov r0, EX->prev_execute_data + | mov aword [&EG(current_execute_data)], r0 + if (op_array->scope) { + | // EG(current_execute_data) = EX(prev_execute_data); + | mov r0, EX->prev_execute_data + | mov aword [&EG(current_execute_data)], r0 + | // if (call_info & ZEND_CALL_RELEASE_THIS) + | test FCARG1d, ZEND_CALL_RELEASE_THIS + if (op_array->fn_flags & ZEND_ACC_STATIC) { + | jnz >1 + |.cold_code + |1: + } else { + | je >2 + } + | // zend_object *object = Z_OBJ(execute_data->This); | mov r0, EX->This.value.obj - | jmp >5 - |.code - |5: + if (op_array->scope && op_array->scope->constructor == (zend_function*)op_array) { + | // if (UNEXPECTED(EG(exception) != NULL) + | cmp aword [&EG(exception)], 0 + | jne >6 + |.cold_code + |6: + | // if (call_info & ZEND_CALL_CTOR) + | test FCARG1d, ZEND_CALL_CTOR + | jz >5 + | // GC_REFCOUNT(object)--; + | GC_DELREF r0 + | // zend_object_store_ctor_failed(object); + |.if X64 + | mov CARG1, r0 + | EXT_CALL zend_object_store_ctor_failed, r0 + |.else + | sub r4, 12 + | push r0 + | EXT_CALL zend_object_store_ctor_failed, r0 + | add r4, 16 + |.endif + | // reload registers + | mov r0, EX->This.value.obj + | jmp >5 + |.code + |5: + } + | // OBJ_RELEASE(object); + | OBJ_RELEASE r0, >4 + | jmp >4 + if (op_array->fn_flags & ZEND_ACC_STATIC) { + |.code + } else { + |2: + } } - | // OBJ_RELEASE(object); - | OBJ_RELEASE r0, >4 - | jmp >4 - if (op_array->fn_flags & ZEND_ACC_STATIC) { + if (op_array->fn_flags & ZEND_ACC_CLOSURE) { + | test FCARG1d, ZEND_CALL_CLOSURE + | jnz >3 + |.cold_code + |3: + | // OBJ_RELEASE((zend_object*)execute_data->func->op_array.prototype); + | mov r0, EX->func + | mov r0, aword [r0 + offsetof(zend_op_array, prototype)] + | OBJ_RELEASE r0, >4 + | jmp >4 |.code - } else { - |2: } + |4: + | // EG(vm_stack_top) = (zval*)execute_data; + | mov aword [&EG(vm_stack_top)], FP + | // execute_data = EX(prev_execute_data); + | mov FP, EX->prev_execute_data + } else { + | // EG(vm_stack_top) = (zval*)execute_data; + | mov aword [&EG(vm_stack_top)], FP + | // execute_data = EX(prev_execute_data); + | mov FP, EX->prev_execute_data + | // EG(current_execute_data) = execute_data + | mov aword [&EG(current_execute_data)], FP } - // TODO: We have to check for ZEND_CALL_CLOSURE flag for every regular - // function, just to support rarely used fake closures created by - // ReflectionFunction::getClosure() ??? - // see ext/reflection/tests/ReflectionFunction_getClosure_basic.php - if (1 || (op_array->fn_flags & ZEND_ACC_CLOSURE)) { - | test FCARG1d, ZEND_CALL_CLOSURE - | jnz >3 - |.cold_code - |3: - | // OBJ_RELEASE((zend_object*)execute_data->func->op_array.prototype); - | mov r0, EX->func - | mov r0, aword [r0 + offsetof(zend_op_array, prototype)] - | OBJ_RELEASE r0, >4 - | jmp >4 - |.code - } - |4: - | // EG(vm_stack_top) = (zval*)execute_data; - | mov aword [&EG(vm_stack_top)], FP - | // execute_data = EX(prev_execute_data); - | mov FP, EX->prev_execute_data | // if (EG(exception)) | cmp aword [&EG(exception)], 0 | mov IP, EX->opline From 03d88fccf61deb4a4d3e6ba09cbcb606de42439b Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 26 Sep 2016 17:58:18 +0300 Subject: [PATCH 241/569] Removed dead code --- ext/opcache/jit/zend_jit_x86.dasc | 3 --- 1 file changed, 3 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 737c7231385be..8ce500f3d10a3 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -3175,9 +3175,6 @@ static int zend_jit_leave_func(dasm_State **Dst, zend_op *opline, zend_op_array | mov r0, EX->prev_execute_data | mov aword [&EG(current_execute_data)], r0 if (op_array->scope) { - | // EG(current_execute_data) = EX(prev_execute_data); - | mov r0, EX->prev_execute_data - | mov aword [&EG(current_execute_data)], r0 | // if (call_info & ZEND_CALL_RELEASE_THIS) | test FCARG1d, ZEND_CALL_RELEASE_THIS if (op_array->fn_flags & ZEND_ACC_STATIC) { From b269e5dcd0197d878369ece77c460a4dfac3f65f Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 26 Sep 2016 19:55:43 +0300 Subject: [PATCH 242/569] Removed useless store --- ext/opcache/jit/zend_jit_x86.dasc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 8ce500f3d10a3..5f91b74e6cef7 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -3251,9 +3251,8 @@ static int zend_jit_leave_func(dasm_State **Dst, zend_op *opline, zend_op_array | cmp aword [&EG(exception)], 0 | mov IP, EX->opline | jne ->leave_throw_handler - | // EX(opline)++ + | // opline = EX(opline) + 1 | add IP, sizeof(zend_op) - | mov EX->opline, IP | add r4, SPAD // stack alignment #ifdef CONTEXT_THREADED_JIT | ret From d6efc476064dcd16226556d95ad3257f0fc30c29 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 26 Sep 2016 22:35:09 +0300 Subject: [PATCH 243/569] Save one instruction --- ext/opcache/jit/zend_jit_x86.dasc | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 5f91b74e6cef7..f1f3342794c1f 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -2884,10 +2884,18 @@ static int zend_jit_do_fcall(dasm_State **Dst, zend_op *opline, zend_op_array *o | // fbc = call->func; | // mov r2, EX:r1->func ??? | // SAVE_OPLINE(); - if (!zend_jit_set_valid_ip(Dst)) { - return 0; - } - | mov EX->opline, IP + |.if X64 + || if (IS_32BIT(opline)) { + | mov aword EX->opline, opline + || } else { + || if (!zend_jit_set_valid_ip(Dst)) { + || return 0; + || } + | mov EX->opline, IP + || } + |.else + | mov aword EX->opline, opline + |.endif || if (call_level == 1) { | mov aword EX->call, 0 || } else { @@ -2957,6 +2965,9 @@ static int zend_jit_do_fcall(dasm_State **Dst, zend_op *opline, zend_op_array *o #endif || } + valid_opline = NULL; + valid_opline_offset = 0; + return 1; fallback: From 346820af203df7998322244ecd0de8134da709c6 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 27 Sep 2016 02:43:28 +0300 Subject: [PATCH 244/569] EX(opline), opline caching refactoring --- ext/opcache/jit/zend_jit.c | 18 +- ext/opcache/jit/zend_jit_x86.dasc | 262 ++++++++++++++---------------- 2 files changed, 127 insertions(+), 153 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 76868fd396b0d..01bfd10f47247 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -55,8 +55,8 @@ static void *dasm_buf = NULL; static void *dasm_ptr = NULL; static void *dasm_end = NULL; -static int zend_may_throw(zend_op *opline, zend_op_array *op_array, zend_ssa *ssa); -static int zend_may_overflow(zend_op *opline, zend_op_array *op_array, zend_ssa *ssa); +static int zend_may_throw(const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa); +static int zend_may_overflow(const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa); #include "dynasm/dasm_x86.h" #include "jit/zend_jit_helpers.c" @@ -313,7 +313,7 @@ static void jit_free(void *p, size_t size) #endif } -static int zend_may_throw(zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) +static int zend_may_throw(const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) { uint32_t t1 = OP1_INFO(); uint32_t t2 = OP2_INFO(); @@ -616,7 +616,7 @@ static int zend_may_throw(zend_op *opline, zend_op_array *op_array, zend_ssa *ss } } -static int zend_may_overflow(zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) +static int zend_may_overflow(const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) { uint32_t num; int res; @@ -854,12 +854,16 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) zend_jit_label(&dasm_state, b); if (ssa->cfg.blocks[b].flags & ZEND_BB_TARGET) { + if (!zend_jit_reset_opline(&dasm_state, op_array->opcodes + ssa->cfg.blocks[b].start)) { + goto jit_failure; + } + } else if (ssa->cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_RECV_ENTRY|ZEND_BB_ENTRY)) { if (!zend_jit_set_opline(&dasm_state, op_array->opcodes + ssa->cfg.blocks[b].start)) { goto jit_failure; } } if (ssa->cfg.blocks[b].flags & ZEND_BB_LOOP_HEADER) { - if (!zend_jit_check_timeout(&dasm_state)) { + if (!zend_jit_check_timeout(&dasm_state, op_array->opcodes + ssa->cfg.blocks[b].start)) { goto jit_failure; } } @@ -1016,10 +1020,6 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) break; #endif case ZEND_NOP: - if (!zend_jit_skip_handler(&dasm_state)) { - goto jit_failure; - } - break; case ZEND_OP_DATA: break; case ZEND_JMP: diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index f1f3342794c1f..608b4c7a84ce7 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -68,11 +68,13 @@ static void* dasm_labels[zend_lb_MAX]; |.section code, cold_code -#define IS_32BIT(addr) (((uintptr_t)(addr)) <= 0x7fffffff) +#define IS_32BIT(addr) (((uintptr_t)(addr)) <= 0xffffffff) + +#define IS_SIGNED_32BIT(val) ((((intptr_t)(val)) <= 0x7fffffff) && (((intptr_t)(val)) >= -2147483648)) |.macro LOAD_ADDR, reg, addr |.if X64 -||if (((ptrdiff_t)addr) <= 0x7fffffff) { +||if (IS_32BIT(addr)) { | mov reg, ((ptrdiff_t)addr) // 0x48 0xc7 0xc0 || } else { | mov64 reg, ((ptrdiff_t)addr) // 0x48 0xb8 @@ -84,7 +86,7 @@ static void* dasm_labels[zend_lb_MAX]; |.macro PUSH_ADDR, addr, tmp_reg |.if X64 -||if (((ptrdiff_t)addr) <= 0x7fffffff) { +||if (IS_32BIT(addr)) { | push ((ptrdiff_t)addr) // 0x48 0xc7 0xc0 || } else { | mov64 tmp_reg, ((ptrdiff_t)addr) // 0x48 0xb8 @@ -304,7 +306,7 @@ static void* dasm_labels[zend_lb_MAX]; |.macro LONG_OP, long_ins, reg, op_type, op ||if (op_type == IS_CONST) { | .if X64 -|| if (!IS_32BIT(Z_LVAL_P(RT_CONSTANT(op_array, op)))) { +|| if (!IS_SIGNED_32BIT(Z_LVAL_P(RT_CONSTANT(op_array, op)))) { | mov64 r1, Z_LVAL_P(RT_CONSTANT(op_array, op)) | long_ins reg, r1 || } else { @@ -321,7 +323,7 @@ static void* dasm_labels[zend_lb_MAX]; |.macro LONG_LOAD, reg, op_type, op ||if (op_type == IS_CONST) { | .if X64 -|| if (!IS_32BIT(Z_LVAL_P(RT_CONSTANT(op_array, op)))) { +|| if (!IS_SIGNED_32BIT(Z_LVAL_P(RT_CONSTANT(op_array, op)))) { | mov64 reg, Z_LVAL_P(RT_CONSTANT(op_array, op)) || } else { | mov reg, Z_LVAL_P(RT_CONSTANT(op_array, op)) @@ -429,7 +431,7 @@ static void* dasm_labels[zend_lb_MAX]; |.endif || } else { |.if X64 -|| if (!IS_32BIT(Z_LVAL_P(zv))) { +|| if (!IS_SIGNED_32BIT(Z_LVAL_P(zv))) { | mov64 tmp_reg, ((uintptr_t)Z_LVAL_P(zv)) | SET_Z_LVAL dst, tmp_reg || } else { @@ -481,7 +483,7 @@ static void* dasm_labels[zend_lb_MAX]; |.endif || } else { |.if X64 -|| if (!IS_32BIT(Z_LVAL_P(zv))) { +|| if (!IS_SIGNED_32BIT(Z_LVAL_P(zv))) { | mov64 tmp_reg, ((uintptr_t)Z_LVAL_P(zv)) | SET_Z_LVAL dst, tmp_reg | SET_Z_LVAL dst2, tmp_reg @@ -652,7 +654,7 @@ static void* dasm_labels[zend_lb_MAX]; |.macro ADDREF_CONST, zv, tmp_reg |.if X64 -|| if (!IS_32BIT(Z_LVAL_P(zv))) { +|| if (!IS_SIGNED_32BIT(Z_LVAL_P(zv))) { | mov64 tmp_reg, ((uintptr_t)Z_LVAL_P(zv)) | inc dword [tmp_reg] || } else { @@ -846,22 +848,24 @@ static void* dasm_labels[zend_lb_MAX]; | EXT_CALL gc_possible_root, r0 |.endmacro -|.macro SAVE_VALID_OPLINE -|| if (valid_opline || valid_opline_offset) { -|| if (!valid_opline) { -| lea r0, [IP + sizeof(zend_op) * valid_opline_offset] +|.macro SAVE_VALID_OPLINE, opline +|| if (opline == last_valid_opline) { +| mov aword EX->opline, IP +|| } else { +|.if X64 +|| if (IS_32BIT(opline)) { +| mov aword EX->opline, ((ptrdiff_t)opline) || } else { -|| const zend_op *target_opline = valid_opline + valid_opline_offset; -| LOAD_ADDR r0, target_opline +| mov64 r0, ((ptrdiff_t)opline) +| mov aword EX->opline, r0 || } -| mov EX->opline, r0 -|| } else { -| mov EX->opline, IP +|.else +| mov aword EX->opline, opline +|.endif || } |.endmacro -static const zend_op *valid_opline; -static uint32_t valid_opline_offset; +static const zend_op *last_valid_opline; static int jit_return_label; #endif @@ -995,9 +999,8 @@ static int zend_jit_leave_throw_stub(dasm_State **Dst) | add r0, FP | ZVAL_PTR_DTOR r0, MAY_BE_ANY, 0, NULL, 0 |5: - | // EX(opline) = EG(exception_op); + | // opline = EG(exception_op); | LOAD_ADDR IP, &EG(exception_op) - | mov EX->opline, IP | add r4, SPAD // stack alignment | EXT_JMP EG(exception_op)->handler, r0 @@ -1013,8 +1016,7 @@ static const zend_jit_stub zend_jit_stubs[] = { static int zend_jit_align_func(dasm_State **Dst) { - valid_opline = NULL; - valid_opline_offset = 0; + last_valid_opline = NULL; jit_return_label = -1; |.align 16 return 1; @@ -1032,26 +1034,21 @@ static int zend_jit_label(dasm_State **Dst, unsigned int label) return 1; } -static int zend_jit_set_valid_ip(dasm_State **Dst) +static int zend_jit_set_valid_ip(dasm_State **Dst, const zend_op *opline) { - if (valid_opline || valid_opline_offset) { - if (!valid_opline) { - | add IP, sizeof(zend_op) * valid_opline_offset - } else { - const zend_op *target_opline = valid_opline + valid_opline_offset; - - valid_opline = NULL; - | LOAD_ADDR IP, target_opline - } - valid_opline_offset = 0; + if (!last_valid_opline) { + | LOAD_ADDR IP, opline + } else if (last_valid_opline != opline) { + | add IP, (opline - last_valid_opline) * sizeof(zend_op); } + last_valid_opline = opline; return 1; } -static int zend_jit_check_timeout(dasm_State **Dst) +static int zend_jit_check_timeout(dasm_State **Dst, const zend_op *opline) { if (zend_interrupt_function) { - if (!zend_jit_set_valid_ip(Dst)) { + if (!zend_jit_set_valid_ip(Dst, opline)) { return 0; } } @@ -1067,38 +1064,43 @@ static int zend_jit_check_exception(dasm_State **Dst) return 1; } -static int zend_jit_handler(dasm_State **Dst, zend_op *opline, int may_throw) +static int zend_jit_handler(dasm_State **Dst, const zend_op *opline, int may_throw) { - if (!zend_jit_set_valid_ip(Dst)) { + if (!zend_jit_set_valid_ip(Dst, opline)) { return 0; } | EXT_CALL opline->handler, r0 if (may_throw) { zend_jit_check_exception(Dst); } + last_valid_opline++; + if ((opline+1)->opcode == ZEND_OP_DATA) { + last_valid_opline++; + } + return 1; } -static int zend_jit_tail_handler(dasm_State **Dst, zend_op *opline) +static int zend_jit_tail_handler(dasm_State **Dst, const zend_op *opline) { - if (!zend_jit_set_valid_ip(Dst)) { + if (!zend_jit_set_valid_ip(Dst, opline)) { return 0; } | add r4, SPAD // stack alignment | EXT_JMP opline->handler, r0 + last_valid_opline = NULL; return 1; } -static int zend_jit_skip_handler(dasm_State **Dst) +static int zend_jit_set_opline(dasm_State **Dst, const zend_op *target_opline) { - valid_opline_offset++; + last_valid_opline = target_opline; return 1; } -static int zend_jit_set_opline(dasm_State **Dst, zend_op *target_opline) +static int zend_jit_reset_opline(dasm_State **Dst, const zend_op *target_opline) { - valid_opline = target_opline; - valid_opline_offset = 0; + last_valid_opline = NULL; return 1; } @@ -1108,18 +1110,20 @@ static int zend_jit_jmp(dasm_State **Dst, unsigned int target_label) return 1; } -static int zend_jit_cond_jmp(dasm_State **Dst, zend_op *next_opline, unsigned int target_label) +static int zend_jit_cond_jmp(dasm_State **Dst, const zend_op *next_opline, unsigned int target_label) { | cmp IPl, next_opline | jne =>target_label + last_valid_opline = next_opline; + return 1; } -static int zend_jit_smart_branch(dasm_State **Dst, zend_op *opline, unsigned int next_label, unsigned int target_label) +static int zend_jit_smart_branch(dasm_State **Dst, const zend_op *opline, unsigned int next_label, unsigned int target_label) { - zend_op *next_opline = opline + 1; - zend_op *target_opline = OP_JMP_ADDR(opline, opline->op2); + const zend_op *next_opline = opline + 1; + const zend_op *target_opline = OP_JMP_ADDR(opline, opline->op2); | cmp IPl, next_opline | je =>next_label @@ -1130,7 +1134,7 @@ static int zend_jit_smart_branch(dasm_State **Dst, zend_op *opline, unsigned int } #ifdef CONTEXT_THREADED_JIT -static int zend_jit_context_threaded_call(dasm_State **Dst, zend_op *opline) +static int zend_jit_context_threaded_call(dasm_State **Dst, const zend_op *opline) { if (!zend_jit_handler(Dst, opline, 1)) return 0; if (opline->opcode == ZEND_DO_UCALL) { @@ -1149,7 +1153,7 @@ static int zend_jit_context_threaded_call(dasm_State **Dst, zend_op *opline) } #endif -static int zend_jit_call(dasm_State **Dst, zend_op *opline) +static int zend_jit_call(dasm_State **Dst, const zend_op *opline) { #ifdef CONTEXT_THREADED_JIT return zend_jit_context_threaded_call(Dst, opline); @@ -1158,7 +1162,7 @@ static int zend_jit_call(dasm_State **Dst, zend_op *opline) #endif } -static int zend_jit_new(dasm_State **Dst, zend_op *opline, int *opnum, zend_op_array *op_array, zend_ssa *ssa) +static int zend_jit_new(dasm_State **Dst, const zend_op *opline, int *opnum, zend_op_array *op_array, zend_ssa *ssa) { if (!zend_jit_handler(Dst, opline, 1)) { return 0; @@ -1183,13 +1187,12 @@ static int zend_jit_new(dasm_State **Dst, zend_op *opline, int *opnum, zend_op_a #endif (*opnum)++; if (!ce || ce->constructor) { - zend_op *next_opline = opline + 1; + const zend_op *next_opline = opline + 1; + | cmp IPl, next_opline | jne >1 zend_jit_call(Dst, next_opline); |1: - } else { - valid_opline_offset++; } } return 1; @@ -1197,7 +1200,7 @@ static int zend_jit_new(dasm_State **Dst, zend_op *opline, int *opnum, zend_op_a #if ZEND_JIT_LEVEL >= ZEND_JIT_LEVEL_OPT_FUNC -static int zend_jit_inc_dec(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) +static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) { uint32_t op1_info, op1_def_info; @@ -1261,6 +1264,7 @@ static int zend_jit_inc_dec(dasm_State **Dst, zend_op *opline, zend_op_array *op if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { |.cold_code |2: + | SAVE_VALID_OPLINE opline || if (op1_info & MAY_BE_UNDEF) { | IF_NOT_Z_TYPE FP + opline->op1.var, IS_UNDEF, >2 | // zend_error(E_NOTICE, "Undefined variable: %s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); @@ -1322,7 +1326,6 @@ static int zend_jit_inc_dec(dasm_State **Dst, zend_op *opline, zend_op_array *op |.code } |3: - valid_opline_offset++; return 1; fallback: @@ -1330,17 +1333,17 @@ fallback: return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); } -static int zend_jit_math_long_long(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, uint32_t res_forward) +static int zend_jit_math_long_long(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, uint32_t res_forward) { uint32_t res_info = RES_INFO(); zend_bool same_ops = (opline->op1_type == opline->op2_type) && (opline->op1.var == opline->op2.var); if (opline->opcode == ZEND_MUL && ((opline->op2_type == IS_CONST && - IS_32BIT(Z_LVAL_P(RT_CONSTANT(op_array, opline->op2))) && + IS_SIGNED_32BIT(Z_LVAL_P(RT_CONSTANT(op_array, opline->op2))) && is_power_of_two(Z_LVAL_P(RT_CONSTANT(op_array, opline->op2)))) || (opline->op1_type == IS_CONST && - IS_32BIT(Z_LVAL_P(RT_CONSTANT(op_array, opline->op1))) && + IS_SIGNED_32BIT(Z_LVAL_P(RT_CONSTANT(op_array, opline->op1))) && is_power_of_two(Z_LVAL_P(RT_CONSTANT(op_array, opline->op1)))))) { if (opline->op2_type == IS_CONST) { | LONG_LOAD r0, opline->op1_type, opline->op1 @@ -1405,7 +1408,7 @@ static int zend_jit_math_long_long(dasm_State **Dst, zend_op *opline, zend_op_ar return 1; } -static int zend_jit_math_long_double(dasm_State **Dst, zend_op *opline, uint32_t res_forward) +static int zend_jit_math_long_double(dasm_State **Dst, const zend_op *opline, uint32_t res_forward) { |.if X64 or SSE | SSE_LOAD_LONG xmm0, opline->op1_type, opline->op1 @@ -1425,7 +1428,7 @@ static int zend_jit_math_long_double(dasm_State **Dst, zend_op *opline, uint32_t return 1; } -static int zend_jit_math_double_long(dasm_State **Dst, zend_op *opline, uint32_t res_forward) +static int zend_jit_math_double_long(dasm_State **Dst, const zend_op *opline, uint32_t res_forward) { |.if X64 or SSE || if (opline->opcode == ZEND_ADD || opline->opcode == ZEND_MUL) { @@ -1456,7 +1459,7 @@ static int zend_jit_math_double_long(dasm_State **Dst, zend_op *opline, uint32_t return 1; } -static int zend_jit_math_double_double(dasm_State **Dst, zend_op *opline, uint32_t res_forward) +static int zend_jit_math_double_double(dasm_State **Dst, const zend_op *opline, uint32_t res_forward) { zend_bool same_ops = (opline->op1_type == opline->op2_type) && (opline->op1.var == opline->op2.var); @@ -1483,7 +1486,7 @@ static int zend_jit_math_double_double(dasm_State **Dst, zend_op *opline, uint32 } -static int zend_jit_shift(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) +static int zend_jit_shift(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) { zval *op2; uint32_t op1_info; @@ -1542,8 +1545,6 @@ static int zend_jit_shift(dasm_State **Dst, zend_op *opline, zend_op_array *op_a |.code } - valid_opline_offset++; - return 1; fallback: @@ -1551,7 +1552,7 @@ fallback: return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); } -static int zend_jit_math(dasm_State **Dst, zend_op *opline, int *opnum, zend_op_array *op_array, zend_ssa *ssa) +static int zend_jit_math(dasm_State **Dst, const zend_op *opline, int *opnum, zend_op_array *op_array, zend_ssa *ssa) { uint32_t op1_info, op2_info; zend_bool same_ops = (opline->op1_type == opline->op2_type) && (opline->op1.var == opline->op2.var); @@ -1720,16 +1721,11 @@ static int zend_jit_math(dasm_State **Dst, zend_op *opline, int *opnum, zend_op_ } |6: - if (res_forward) { - valid_opline_offset += 2; - } else { - valid_opline_offset++; - } if (has_slow) { |.cold_code |9: - | mov EX->opline, IP + | SAVE_VALID_OPLINE opline if (res_forward) { | lea FCARG1a, [r1 + res_forward] } else { @@ -1768,7 +1764,7 @@ fallback: return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); } -static int zend_jit_cmp_long_long(dasm_State **Dst, zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa) +static int zend_jit_cmp_long_long(dasm_State **Dst, const zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa) { unsigned int target_label; int swap = 0; @@ -1777,7 +1773,7 @@ static int zend_jit_cmp_long_long(dasm_State **Dst, zend_op *opline, int b, zend zend_long val = Z_LVAL_P(RT_CONSTANT(op_array, opline->op1)); | .if X64 - || if (!IS_32BIT(val)) { + || if (!IS_SIGNED_32BIT(val)) { | mov64 r0, val | cmp aword [FP + opline->op2.var], r0 || } else { @@ -1791,7 +1787,7 @@ static int zend_jit_cmp_long_long(dasm_State **Dst, zend_op *opline, int b, zend zend_long val = Z_LVAL_P(RT_CONSTANT(op_array, opline->op2)); | .if X64 - || if (!IS_32BIT(val)) { + || if (!IS_SIGNED_32BIT(val)) { | mov64 r0, val | cmp aword [FP + opline->op1.var], r0 || } else { @@ -1925,7 +1921,7 @@ static int zend_jit_cmp_long_long(dasm_State **Dst, zend_op *opline, int b, zend return 1; } -static int zend_jit_cmp_double_common(dasm_State **Dst, zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa) +static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa) { unsigned int target_label; @@ -2036,7 +2032,7 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, zend_op *opline, int b, return 1; } -static int zend_jit_cmp_long_double(dasm_State **Dst, zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa) +static int zend_jit_cmp_long_double(dasm_State **Dst, const zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa) { |.if X64 or SSE | SSE_LOAD_LONG xmm0, opline->op1_type, opline->op1 @@ -2051,7 +2047,7 @@ static int zend_jit_cmp_long_double(dasm_State **Dst, zend_op *opline, int b, ze return zend_jit_cmp_double_common(Dst, opline, b, op_array, ssa); } -static int zend_jit_cmp_double_long(dasm_State **Dst, zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa) +static int zend_jit_cmp_double_long(dasm_State **Dst, const zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa) { |.if X64 or SSE | SSE_LOAD xmm0, opline->op1_type, opline->op1 @@ -2067,7 +2063,7 @@ static int zend_jit_cmp_double_long(dasm_State **Dst, zend_op *opline, int b, ze return zend_jit_cmp_double_common(Dst, opline, b, op_array, ssa); } -static int zend_jit_cmp_double_double(dasm_State **Dst, zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa) +static int zend_jit_cmp_double_double(dasm_State **Dst, const zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa) { |.if X64 or SSE | SSE_LOAD xmm0, opline->op1_type, opline->op1 @@ -2082,7 +2078,7 @@ static int zend_jit_cmp_double_double(dasm_State **Dst, zend_op *opline, int b, return zend_jit_cmp_double_common(Dst, opline, b, op_array, ssa); } -static int zend_jit_cmp_slow(dasm_State **Dst, zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa) +static int zend_jit_cmp_slow(dasm_State **Dst, const zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa) { unsigned int target_label; @@ -2176,7 +2172,7 @@ static int zend_jit_cmp_slow(dasm_State **Dst, zend_op *opline, int b, zend_op_a return 1; } -static int zend_jit_cmp(dasm_State **Dst, zend_op *opline, int b, int *opnum, zend_op_array *op_array, zend_ssa *ssa) +static int zend_jit_cmp(dasm_State **Dst, const zend_op *opline, int b, int *opnum, zend_op_array *op_array, zend_ssa *ssa) { uint32_t op1_info, op2_info; zend_bool same_ops = (opline->op1_type == opline->op2_type) && (opline->op1.var == opline->op2.var); @@ -2327,7 +2323,7 @@ static int zend_jit_cmp(dasm_State **Dst, zend_op *opline, int b, int *opnum, ze |.cold_code |9: } - | mov EX->opline, IP + | SAVE_VALID_OPLINE opline | lea FCARG1a, [FP + opline->result.var] | LOAD_ZVAL_ADDR FCARG2a, opline->op1_type, opline->op1 if (opline->op1_type == IS_CV && (op1_info & MAY_BE_UNDEF)) { @@ -2387,15 +2383,12 @@ static int zend_jit_cmp(dasm_State **Dst, zend_op *opline, int b, int *opnum, ze (opline+1)->op1_type == IS_TMP_VAR && (opline+1)->op1.var == opline->result.var) { (*opnum)++; - valid_opline_offset += 2; - } else { - valid_opline_offset++; } return 1; } -static int zend_jit_jmpznz(dasm_State **Dst, zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa) +static int zend_jit_jmpznz(dasm_State **Dst, const zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa) { uint32_t op1_info = OP1_INFO(); uint32_t true_label, false_label; @@ -2425,7 +2418,7 @@ static int zend_jit_jmpznz(dasm_State **Dst, zend_op *opline, int b, zend_op_arr |1: } | mov FCARG1d, opline->op1.var - | SAVE_VALID_OPLINE + | SAVE_VALID_OPLINE opline | EXT_CALL zend_jit_undefined_op_helper, r0 if (zend_may_throw(opline, op_array, ssa)) { if (!zend_jit_check_exception(Dst)) { @@ -2468,7 +2461,7 @@ static int zend_jit_jmpznz(dasm_State **Dst, zend_op *opline, int b, zend_op_arr |.cold_code |1: | mov FCARG1d, opline->op1.var - | SAVE_VALID_OPLINE + | SAVE_VALID_OPLINE opline | EXT_CALL zend_jit_undefined_op_helper, r0 if (zend_may_throw(opline, op_array, ssa)) { if (!zend_jit_check_exception(Dst)) { @@ -2519,7 +2512,7 @@ static int zend_jit_jmpznz(dasm_State **Dst, zend_op *opline, int b, zend_op_arr |.cold_code |2: } - | SAVE_VALID_OPLINE + | SAVE_VALID_OPLINE opline | EXT_CALL zend_is_true, r0 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && @@ -2563,12 +2556,10 @@ static int zend_jit_jmpznz(dasm_State **Dst, zend_op *opline, int b, zend_op_arr |9: } - valid_opline_offset++; - return 1; } -static int zend_jit_simple_assign(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, uint32_t var, uint32_t var_info, zend_uchar val_type, znode_op val, uint32_t val_info, uint32_t var2, int in_cold) +static int zend_jit_simple_assign(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, uint32_t var, uint32_t var_info, zend_uchar val_type, znode_op val, uint32_t val_info, uint32_t var2, int in_cold) { if (val_type == IS_CONST) { zval *zv = RT_CONSTANT(op_array, val); @@ -2591,6 +2582,7 @@ static int zend_jit_simple_assign(dasm_State **Dst, zend_op *opline, zend_op_arr |1: } | // zend_error(E_NOTICE, "Undefined variable: %s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); + | SAVE_VALID_OPLINE opline | mov FCARG1d, val.var | EXT_CALL zend_jit_undefined_op_helper, r0 | SET_Z_TYPE_INFO FP + val.var, IS_NULL @@ -2677,15 +2669,14 @@ static int zend_jit_simple_assign(dasm_State **Dst, zend_op *opline, zend_op_arr return 1; } -static int zend_jit_qm_assign(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) +static int zend_jit_qm_assign(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) { uint32_t op1_info = OP1_INFO(); - valid_opline_offset++; return zend_jit_simple_assign(Dst, opline, op_array, ssa, opline->result.var, -1, opline->op1_type, opline->op1, op1_info, -1, 0); } -static int zend_jit_assign_to_variable(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, uint32_t var, uint32_t var_info, zend_uchar val_type, znode_op val, uint32_t val_info, uint32_t var2) +static int zend_jit_assign_to_variable(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, uint32_t var, uint32_t var_info, zend_uchar val_type, znode_op val, uint32_t val_info, uint32_t var2) { if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { int in_cold = 0; @@ -2728,7 +2719,7 @@ static int zend_jit_assign_to_variable(dasm_State **Dst, zend_op *opline, zend_o return 1; } -static int zend_jit_assign(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) +static int zend_jit_assign(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) { uint32_t op1_info, op2_info; @@ -2748,7 +2739,6 @@ static int zend_jit_assign(dasm_State **Dst, zend_op *opline, zend_op_array *op_ return 0; } - valid_opline_offset++; return 1; fallback: @@ -2756,7 +2746,7 @@ fallback: return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); } -static int zend_jit_push_call_frame(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_function *func) +static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_function *func) { if (!func) { assert(0); @@ -2794,7 +2784,7 @@ static int zend_jit_push_call_frame(dasm_State **Dst, zend_op *opline, zend_op_a return 1; } -static int zend_jit_init_fcall(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, int call_level) +static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, int call_level) { zend_func_info *info = ZEND_FUNC_INFO(op_array); zend_function *func = NULL; @@ -2825,7 +2815,7 @@ static int zend_jit_init_fcall(dasm_State **Dst, zend_op *opline, zend_op_array |.cold_code |1: | // SAVE_OPLINE(); - | mov EX->opline, IP + | SAVE_VALID_OPLINE opline | LOAD_ADDR FCARG1a, Z_STR_P(zv); | EXT_CALL zend_jit_find_func_helper, r0 | // This opcode must not throw exception ??? @@ -2850,12 +2840,11 @@ static int zend_jit_init_fcall(dasm_State **Dst, zend_op *opline, zend_op_array } | // EX(call) = call; | mov EX->call, r1 - valid_opline_offset++; return 1; } -static int zend_jit_do_fcall(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, int call_level) +static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, int call_level) { zend_func_info *info = ZEND_FUNC_INFO(op_array); zend_call_info *call_info = NULL; @@ -2884,18 +2873,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, zend_op *opline, zend_op_array *o | // fbc = call->func; | // mov r2, EX:r1->func ??? | // SAVE_OPLINE(); - |.if X64 - || if (IS_32BIT(opline)) { - | mov aword EX->opline, opline - || } else { - || if (!zend_jit_set_valid_ip(Dst)) { - || return 0; - || } - | mov EX->opline, IP - || } - |.else - | mov aword EX->opline, opline - |.endif + | SAVE_VALID_OPLINE opline || if (call_level == 1) { | mov aword EX->call, 0 || } else { @@ -2946,9 +2924,8 @@ static int zend_jit_do_fcall(dasm_State **Dst, zend_op *opline, zend_op_array *o | // EG(current_execute_data) = execute_data; | mov aword [&EG(current_execute_data)], r1 | mov FP, r1 - | // EX(opline) = op_array->opcodes; + | // opline = op_array->opcodes; | LOAD_ADDR IP, (func->op_array.opcodes + call_info->num_args) - | mov EX:r1->opline, IP || if (func && op_array == &func->op_array) { || /* recursive call */ #ifdef CONTEXT_THREADED_JIT @@ -2965,9 +2942,6 @@ static int zend_jit_do_fcall(dasm_State **Dst, zend_op *opline, zend_op_array *o #endif || } - valid_opline = NULL; - valid_opline_offset = 0; - return 1; fallback: @@ -2981,7 +2955,7 @@ fallback: } } -static int zend_jit_send_val(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) +static int zend_jit_send_val(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) { uint32_t op1_info; uint32_t arg_num = opline->op2.num; @@ -3008,7 +2982,7 @@ static int zend_jit_send_val(dasm_State **Dst, zend_op *opline, zend_op_array *o |.cold_code |1: | SET_Z_TYPE_INFO r1 + opline->result.var, IS_UNDEF - | mov EX->opline, IP + | SAVE_VALID_OPLINE opline |.if X64 | mov CARG1, 0 | LOAD_ADDR CARG2, "Cannot pass parameter %d by reference" @@ -3037,7 +3011,6 @@ static int zend_jit_send_val(dasm_State **Dst, zend_op *opline, zend_op_array *o } else { | ZVAL_COPY_VALUE r1 + opline->result.var, FP + opline->op1.var, op1_info, r0, eax, r2 } - valid_opline_offset++; return 1; @@ -3046,7 +3019,7 @@ fallback: return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); } -static int zend_jit_send_var(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) +static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) { uint32_t op1_info; if (!ssa->ops || !ssa->var_info) { @@ -3065,7 +3038,7 @@ static int zend_jit_send_var(dasm_State **Dst, zend_op *opline, zend_op_array *o || if (opline->op1_type == IS_CV) { | TRY_ADDREF op1_info, ah, r2 || } - valid_opline_offset++; + return 1; fallback: @@ -3073,7 +3046,7 @@ fallback: return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); } -static int zend_jit_type_check(dasm_State **Dst, zend_op *opline, int b, int *opnum, zend_op_array *op_array, zend_ssa *ssa) +static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, int b, int *opnum, zend_op_array *op_array, zend_ssa *ssa) { uint32_t op1_info, mask; unsigned int target_label; @@ -3097,7 +3070,6 @@ static int zend_jit_type_check(dasm_State **Dst, zend_op *opline, int b, int *op (opline+1)->op1_type == IS_TMP_VAR && (opline+1)->op1.var == opline->result.var) { (*opnum)++; - valid_opline_offset++; } else if ((opline+1)->opcode == ZEND_JMPNZ && (opline+1)->op1_type == IS_TMP_VAR && (opline+1)->op1.var == opline->result.var) { @@ -3106,7 +3078,6 @@ static int zend_jit_type_check(dasm_State **Dst, zend_op *opline, int b, int *op | jmp =>target_label } else { | SET_Z_TYPE_INFO FP + opline->result.var, IS_TRUE - valid_opline_offset++; } } else if (!(op1_info & mask)) { if ((opline+1)->opcode == ZEND_JMPZ && @@ -3119,10 +3090,8 @@ static int zend_jit_type_check(dasm_State **Dst, zend_op *opline, int b, int *op (opline+1)->op1_type == IS_TMP_VAR && (opline+1)->op1.var == opline->result.var) { (*opnum)++; - valid_opline_offset += 2; } else { | SET_Z_TYPE_INFO FP + opline->result.var, IS_FALSE - valid_opline_offset++; } } else { goto fallback; @@ -3135,7 +3104,7 @@ fallback: return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); } -static int zend_jit_free_compiled_variables(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) +static int zend_jit_free_compiled_variables(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) { uint32_t i, j, info; @@ -3160,7 +3129,7 @@ static int zend_jit_free_compiled_variables(dasm_State **Dst, zend_op *opline, z return 1; } -static int zend_jit_leave_func(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) +static int zend_jit_leave_func(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) { // Avoid multiple leave sequnces if (jit_return_label >= 0) { @@ -3274,7 +3243,7 @@ static int zend_jit_leave_func(dasm_State **Dst, zend_op *opline, zend_op_array return 1; } -static int zend_jit_return(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) +static int zend_jit_return(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) { uint32_t op1_info; @@ -3310,7 +3279,7 @@ static int zend_jit_return(dasm_State **Dst, zend_op *opline, zend_op_array *op_ | jnz >9 || } | //SAVE_OPLINE() - | mov EX->opline, IP + | SAVE_VALID_OPLINE opline | ZVAL_DTOR_FUNC (op_array->filename ? ZSTR_VAL(op_array->filename) : NULL), opline->lineno | mov r1, EX->return_value // reload ??? || if (jit_return_label >= 0) { @@ -3419,7 +3388,7 @@ static int zend_jit_return(dasm_State **Dst, zend_op *opline, zend_op_array *op_ return zend_jit_leave_func(Dst, opline, op_array, ssa); } -static int zend_jit_fetch_dim_read(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) +static int zend_jit_fetch_dim_read(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) { uint32_t op1_info, op2_info, res_info; @@ -3504,6 +3473,7 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, zend_op *opline, zend_op_ar |.cold_code |2: if (opline->opcode == ZEND_FETCH_DIM_R) { + | SAVE_VALID_OPLINE opline | // zend_error(E_NOTICE,"Undefined offset: " ZEND_LONG_FMT, hval); |.if X64 | mov CARG1, E_NOTICE @@ -3556,6 +3526,7 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, zend_op *opline, zend_op_ar |2: if (opline->opcode == ZEND_FETCH_DIM_R) { // zend_error(E_NOTICE, "Undefined index: %s", ZSTR_VAL(offset_key)); + | SAVE_VALID_OPLINE opline |.if X64 | mov CARG1, E_NOTICE | LOAD_ADDR CARG2, "Undefined index: %s" @@ -3585,7 +3556,7 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, zend_op *opline, zend_op_ar |.cold_code |3: } - | SAVE_VALID_OPLINE + | SAVE_VALID_OPLINE opline | LOAD_ZVAL_ADDR FCARG2a, opline->op2_type, opline->op2 |.if X64 | lea CARG3, [FP + opline->result.var] @@ -3621,7 +3592,7 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, zend_op *opline, zend_op_ar | IF_NOT_Z_TYPE FP + opline->op1.var, IS_STRING, >6 } } - | SAVE_VALID_OPLINE + | SAVE_VALID_OPLINE opline if (!(op1_info & MAY_BE_REF)) { | LOAD_ZVAL_ADDR FCARG1a, opline->op1_type, opline->op1 } @@ -3651,7 +3622,7 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, zend_op *opline, zend_op_ar | IF_NOT_Z_TYPE FP + opline->op1.var, IS_OBJECT, >6 } } - | SAVE_VALID_OPLINE + | SAVE_VALID_OPLINE opline if (!(op1_info & MAY_BE_REF)) { | LOAD_ZVAL_ADDR FCARG1a, opline->op1_type, opline->op1 } @@ -3673,6 +3644,10 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, zend_op *opline, zend_op_ar |6: } + if ((opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) || + (op2_info & MAY_BE_UNDEF)) { + | SAVE_VALID_OPLINE opline + } if (opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) { if (op1_info & MAY_BE_REF) { | IF_NOT_Z_TYPE FCARG1a, IS_UNDEF, >1 @@ -3729,8 +3704,6 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, zend_op *opline, zend_op_ar } } - valid_opline_offset++; - return 1; fallback: @@ -3738,7 +3711,7 @@ fallback: return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); } -static int zend_jit_bind_global(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) +static int zend_jit_bind_global(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) { uint32_t op1_info; zval *varname = RT_CONSTANT(op_array, opline->op2); @@ -3891,12 +3864,11 @@ static int zend_jit_bind_global(dasm_State **Dst, zend_op *opline, zend_op_array | jmp <1 |.code - valid_opline_offset++; - return 1; } -static int zend_jit_recv_init(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_bool is_last, zend_ssa *ssa) { +static int zend_jit_recv_init(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_bool is_last, zend_ssa *ssa) +{ zend_arg_info *arg_info; zend_bool has_slow = 0; uint32_t arg_num = opline->op1.num; @@ -3910,6 +3882,7 @@ static int zend_jit_recv_init(dasm_State **Dst, zend_op *opline, zend_op_array * } if (Z_CONSTANT_P(zv)) { has_slow = 1; + | SAVE_VALID_OPLINE opline |.if X64 | lea CARG1, [FP + opline->result.var] | mov r0, EX->func @@ -3971,6 +3944,7 @@ static int zend_jit_recv_init(dasm_State **Dst, zend_op *opline, zend_op_array * } if (is_last) { | LOAD_ADDR IP, (opline + 1) + last_valid_opline = (opline + 1); } if (has_slow) { From 799ab5e32d926f0edec8a42784d0e80a0c65c933 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 27 Sep 2016 15:29:31 +0300 Subject: [PATCH 245/569] Implemanted JIT for simple ASSIGN_ADD/SUB/MUL ($a += ...) --- ext/opcache/jit/zend_jit.c | 92 ++++++ ext/opcache/jit/zend_jit_x86.dasc | 499 ++++++++++++++++++++---------- 2 files changed, 430 insertions(+), 161 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 01bfd10f47247..a386ad51bbad7 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -708,6 +708,90 @@ static int zend_may_overflow(const zend_op *opline, zend_op_array *op_array, zen } return 0; case ZEND_MUL: + num = opline - op_array->opcodes; + res = ssa->ops[num].result_def; + return (res < 0 || + !ssa->var_info[res].has_range || + ssa->var_info[res].range.underflow || + ssa->var_info[res].range.overflow); + case ZEND_ASSIGN_ADD: + if (opline->extended_value != 0) { + return 1; + } + num = opline - op_array->opcodes; + res = ssa->ops[num].op1_def; + if (res < 0 || + !ssa->var_info[res].has_range) { + return 1; + } + if (ssa->var_info[res].range.underflow) { + if (OP1_HAS_RANGE() && OP2_HAS_RANGE()) { + zend_long op1_min, op2_min, res_min; + + op1_min = OP1_MIN_RANGE(); + op2_min = OP2_MIN_RANGE(); + res_min = op1_min + op2_min; + return OP1_RANGE_UNDERFLOW() || + OP2_RANGE_UNDERFLOW() || + (op1_min < 0 && op2_min < 0 && res_min >= 0); + } + return 1; + } + if (ssa->var_info[res].range.overflow) { + if (OP1_HAS_RANGE() && OP2_HAS_RANGE()) { + zend_long op1_max, op2_max, res_max; + + op1_max = OP1_MAX_RANGE(); + op2_max = OP2_MAX_RANGE(); + res_max = op1_max + op2_max; + return OP1_RANGE_OVERFLOW() || + OP2_RANGE_OVERFLOW() || + (op1_max > 0 && op2_max > 0 && res_max <= 0); + } + return 1; + } + return 0; + case ZEND_ASSIGN_SUB: + if (opline->extended_value != 0) { + return 1; + } + num = opline - op_array->opcodes; + res = ssa->ops[num].op1_def; + if (res < 0 || + !ssa->var_info[res].has_range) { + return 1; + } + if (ssa->var_info[res].range.underflow) { + if (OP1_HAS_RANGE() && OP2_HAS_RANGE()) { + zend_long op1_max, op2_min, res_max; + + op1_max = OP1_MAX_RANGE(); + op2_min = OP2_MIN_RANGE(); + res_max = op1_max - op2_min; + return OP1_RANGE_OVERFLOW() || + OP2_RANGE_UNDERFLOW() || + (op1_max > 0 && op2_min < 0 && res_max <= 0); + } + return 1; + } + if (ssa->var_info[res].range.overflow) { + if (OP1_HAS_RANGE() && OP2_HAS_RANGE()) { + zend_long op1_min, op2_max, res_min; + + op1_min = OP1_MIN_RANGE(); + op2_max = OP2_MAX_RANGE(); + res_min = op1_min - op2_max; + return OP1_RANGE_UNDERFLOW() || + OP2_RANGE_OVERFLOW() || + (op1_min < 0 && op2_max > 0 && res_min >= 0); + } + return 1; + } + return 0; + case ZEND_ASSIGN_MUL: + if (opline->extended_value != 0) { + return 1; + } num = opline - op_array->opcodes; res = ssa->ops[num].op1_def; return (res < 0 || @@ -908,6 +992,14 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) goto jit_failure; } break; + case ZEND_ASSIGN_ADD: + case ZEND_ASSIGN_SUB: + case ZEND_ASSIGN_MUL: +// case ZEND_ASSIGN_DIV: // TODO: check for division by zero ??? + if (!zend_jit_assign_math(&dasm_state, opline, op_array, ssa)) { + goto jit_failure; + } + break; case ZEND_ASSIGN: if (!zend_jit_assign(&dasm_state, opline, op_array, ssa)) { goto jit_failure; diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 608b4c7a84ce7..febe22ead7ea0 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -57,6 +57,30 @@ |.define SSE, 1 |.endif +#define r0 0 +#define r1 1 +#define r2 2 +#define r3 3 +#define r4 4 +#define r5 5 +#define r6 6 +#define r7 7 +#if SIZEOF_SIZE_T == 8 +# define r8 8 +# define r9 9 +# define r10 10 +# define r11 11 +# define r12 12 +# define r13 13 +# define r14 14 +# define r15 15 +# define FP r14 +# define FCARG1a r7 +#else +# define FP r6 +# define FCARG1a r1 +#endif + |.type EX, zend_execute_data, FP |.type OP, zend_op, IP |.type ZVAL, zval @@ -205,15 +229,19 @@ static void* dasm_labels[zend_lb_MAX]; |.macro FPU_MATH, opcode, op_type, op ||switch (opcode) { || case ZEND_ADD: +|| case ZEND_ASSIGN_ADD: | FPU_OP fadd, op_type, op || break; || case ZEND_SUB: +|| case ZEND_ASSIGN_SUB: | FPU_OP fsub, op_type, op || break; || case ZEND_MUL: +|| case ZEND_ASSIGN_MUL: | FPU_OP fmul, op_type, op || break; || case ZEND_DIV: +|| case ZEND_ASSIGN_DIV: | FPU_OP fdiv, op_type, op || break; ||} @@ -245,30 +273,38 @@ static void* dasm_labels[zend_lb_MAX]; | cvtsi2sd reg, dword [op.zv] | .endif ||} else { -| .if X64 -| cvtsi2sd reg, qword [FP + op.var] -| .else -| cvtsi2sd reg, dword [FP + op.var] -| .endif +| cvtsi2sd reg, aword [FP + op.var] ||} |.endmacro +|.macro SSE_GET_Z_LVAL, reg, zv +| cvtsi2sd reg, aword [zv] +|.endmacro + |.macro SSE_LOAD, reg, op_type, op | SSE_OP movsd, reg, op_type, op |.endmacro +|.macro SSE_GET_Z_DVAL, reg, zv +| movsd reg, qword [zv] +|.endmacro + |.macro SSE_MATH, opcode, reg, op_type, op ||switch (opcode) { || case ZEND_ADD: +|| case ZEND_ASSIGN_ADD: | SSE_OP addsd, reg, op_type, op || break; || case ZEND_SUB: +|| case ZEND_ASSIGN_SUB: | SSE_OP subsd, reg, op_type, op || break; || case ZEND_MUL: +|| case ZEND_ASSIGN_MUL: | SSE_OP mulsd, reg, op_type, op || break; || case ZEND_DIV: +|| case ZEND_ASSIGN_DIV: | SSE_OP divsd, reg, op_type, op || break; ||} @@ -277,15 +313,19 @@ static void* dasm_labels[zend_lb_MAX]; |.macro SSE_MATH2, opcode, dst_reg, src_reg ||switch (opcode) { || case ZEND_ADD: +|| case ZEND_ASSIGN_ADD: | addsd dst_reg, src_reg || break; || case ZEND_SUB: +|| case ZEND_ASSIGN_SUB: | subsd dst_reg, src_reg || break; || case ZEND_MUL: +|| case ZEND_ASSIGN_MUL: | mulsd dst_reg, src_reg || break; || case ZEND_DIV: +|| case ZEND_ASSIGN_DIV: | divsd dst_reg, src_reg || break; ||} @@ -339,15 +379,19 @@ static void* dasm_labels[zend_lb_MAX]; |.macro LONG_MATH, opcode, reg, op_type, op ||switch (opcode) { || case ZEND_ADD: +|| case ZEND_ASSIGN_ADD: | LONG_OP add, reg, op_type, op || break; || case ZEND_SUB: +|| case ZEND_ASSIGN_SUB: | LONG_OP sub, reg, op_type, op || break; || case ZEND_MUL: +|| case ZEND_ASSIGN_MUL: | LONG_OP imul, reg, op_type, op || break; || case ZEND_DIV: +|| case ZEND_ASSIGN_DIV: | idiv aword [FP + op.var] // (reg == r0) || break; ||} @@ -356,15 +400,19 @@ static void* dasm_labels[zend_lb_MAX]; |.macro LONG_MATH2, opcode, dst_reg, src_reg ||switch (opcode) { || case ZEND_ADD: +|| case ZEND_ASSIGN_ADD: | add dst_reg, src_reg || break; || case ZEND_SUB: +|| case ZEND_ASSIGN_SUB: | sub dst_reg, src_reg || break; || case ZEND_MUL: +|| case ZEND_ASSIGN_MUL: | imul dst_reg, src_reg || break; || case ZEND_DIV: +|| case ZEND_ASSIGN_DIV: | idiv src_reg // (reg1 == r0) || break; ||} @@ -387,19 +435,31 @@ static void* dasm_labels[zend_lb_MAX]; | FPU_LONG_OP fild, op_type, op |.endmacro -|.macro FPU_MATH2, opcode +|.macro FPU_GET_Z_LVAL, zv +| fild aword [zv] +|.endmacro + +|.macro FPU_GET_Z_DVAL, zv +| fld qword [zv] +|.endmacro + +|.macro FPU_MATH2, opcode, reg ||switch (opcode) { || case ZEND_ADD: -| fadd st1 +|| case ZEND_ASSIGN_ADD: +| fadd reg || break; || case ZEND_SUB: -| fsub st1 +|| case ZEND_ASSIGN_SUB: +| fsub reg || break; || case ZEND_MUL: -| fmul st1 +|| case ZEND_ASSIGN_MUL: +| fmul reg || break; || case ZEND_DIV: -| fdiv st1 +|| case ZEND_ASSIGN_DIV: +| fdiv reg || break; ||} |.endmacro @@ -1333,9 +1393,8 @@ fallback: return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); } -static int zend_jit_math_long_long(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, uint32_t res_forward) +static int zend_jit_math_long_long(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, uint32_t op1_reg, uint32_t op1_offset, uint32_t res_reg, uint32_t res_offset, uint32_t res_info) { - uint32_t res_info = RES_INFO(); zend_bool same_ops = (opline->op1_type == opline->op2_type) && (opline->op1.var == opline->op2.var); if (opline->opcode == ZEND_MUL && @@ -1346,23 +1405,35 @@ static int zend_jit_math_long_long(dasm_State **Dst, const zend_op *opline, zend IS_SIGNED_32BIT(Z_LVAL_P(RT_CONSTANT(op_array, opline->op1))) && is_power_of_two(Z_LVAL_P(RT_CONSTANT(op_array, opline->op1)))))) { if (opline->op2_type == IS_CONST) { - | LONG_LOAD r0, opline->op1_type, opline->op1 - | shl r0, floor_log2(Z_LVAL_P(RT_CONSTANT(op_array, opline->op2))) + if (opline->op1_type == IS_CONST) { + | LONG_LOAD r0, opline->op1_type, opline->op1 + } else { + | GET_Z_LVAL r0, Ra(op1_reg)+op1_offset + } + | shl r0, floor_log2(Z_LVAL_P(RT_CONSTANT(op_array, opline->op2))) } else { - | LONG_LOAD r0, opline->op2_type, opline->op2 - | shl r0, floor_log2(Z_LVAL_P(RT_CONSTANT(op_array, opline->op1))) + | LONG_LOAD r0, opline->op2_type, opline->op2 + | shl r0, floor_log2(Z_LVAL_P(RT_CONSTANT(op_array, opline->op1))) } } else if (opline->opcode == ZEND_DIV && (opline->op2_type == IS_CONST && is_power_of_two(Z_LVAL_P(RT_CONSTANT(op_array, opline->op2))))) { - | LONG_LOAD r0, opline->op1_type, opline->op1 + if (opline->op1_type == IS_CONST) { + | LONG_LOAD r0, opline->op1_type, opline->op1 + } else { + | GET_Z_LVAL r0, Ra(op1_reg)+op1_offset + } | shr r0, floor_log2(Z_LVAL_P(RT_CONSTANT(op_array, opline->op2))) } else { - | LONG_LOAD r0, opline->op1_type, opline->op1 + if (opline->op1_type == IS_CONST) { + | LONG_LOAD r0, opline->op1_type, opline->op1 + } else { + | GET_Z_LVAL r0, Ra(op1_reg)+op1_offset + } if (same_ops && opline->opcode != ZEND_DIV) { - | LONG_MATH2 opline->opcode, r0, r0 + | LONG_MATH2 opline->opcode, r0, r0 } else { - | LONG_MATH opline->opcode, r0, opline->op2_type, opline->op2 + | LONG_MATH opline->opcode, r0, opline->op2_type, opline->op2 } } if ((res_info & MAY_BE_DOUBLE) && zend_may_overflow(opline, op_array, ssa)) { @@ -1370,116 +1441,133 @@ static int zend_jit_math_long_long(dasm_State **Dst, const zend_op *opline, zend |.cold_code |1: |.if X64 or SSE - | SSE_LOAD_LONG xmm0, opline->op1_type, opline->op1 + if (opline->op1_type == IS_CONST) { + | SSE_LOAD_LONG xmm0, opline->op1_type, opline->op1 + } else { + | SSE_GET_Z_LVAL xmm0, Ra(op1_reg)+op1_offset + } | SSE_LOAD_LONG xmm1, opline->op2_type, opline->op2 | SSE_MATH2 opline->opcode, xmm0, xmm1 |.else | FPU_LONG_LOAD opline->op2_type, opline->op2 - | FPU_LONG_LOAD opline->op1_type, opline->op1 - | FPU_MATH2 opline->opcode - |.endif - if (res_forward) { - | DOUBLE_STORE r1 + res_forward, xmm0 - | SET_Z_TYPE_INFO r1 + res_forward, IS_DOUBLE + if (opline->op1_type == IS_CONST) { + | FPU_LONG_LOAD opline->op1_type, opline->op1 } else { - | DOUBLE_STORE FP + opline->result.var, xmm0 - | SET_Z_TYPE_INFO FP + opline->result.var, IS_DOUBLE + | FPU_GET_Z_LVAL Ra(op1_reg)+op1_offset } + | FPU_MATH2 opline->opcode, st1 + |.endif + | DOUBLE_STORE Ra(res_reg)+res_offset, xmm0 + | SET_Z_TYPE_INFO Ra(res_reg)+res_offset, IS_DOUBLE | jmp >2 |.code - if (res_forward) { - | SET_Z_LVAL r1 + res_forward, r0 - | SET_Z_TYPE_INFO r1 + res_forward, IS_LONG - } else { - | SET_Z_LVAL FP + opline->result.var, r0 - | SET_Z_TYPE_INFO FP + opline->result.var, IS_LONG + | SET_Z_LVAL Ra(res_reg)+res_offset, r0 + if (op1_reg != res_reg || op1_offset != res_offset) { + | SET_Z_TYPE_INFO Ra(res_reg)+res_offset, IS_LONG } |2: } else { - if (res_forward) { - | SET_Z_LVAL r1 + res_forward, r0 - | SET_Z_TYPE_INFO r1 + res_forward, IS_LONG - } else { - | SET_Z_LVAL FP + opline->result.var, r0 - | SET_Z_TYPE_INFO FP + opline->result.var, IS_LONG + | SET_Z_LVAL Ra(res_reg)+res_offset, r0 + if (op1_reg != res_reg || op1_offset != res_offset) { + | SET_Z_TYPE_INFO Ra(res_reg)+res_offset, IS_LONG } } return 1; } -static int zend_jit_math_long_double(dasm_State **Dst, const zend_op *opline, uint32_t res_forward) +static int zend_jit_math_long_double(dasm_State **Dst, const zend_op *opline, uint32_t op1_reg, uint32_t op1_offset, uint32_t res_reg, uint32_t res_offset) { |.if X64 or SSE - | SSE_LOAD_LONG xmm0, opline->op1_type, opline->op1 + if (opline->op1_type == IS_CONST) { + | SSE_LOAD_LONG xmm0, opline->op1_type, opline->op1 + } else { + | SSE_GET_Z_LVAL xmm0, Ra(op1_reg)+op1_offset + } | SSE_MATH opline->opcode, xmm0, opline->op2_type, opline->op2 |.else - | FPU_LONG_LOAD opline->op1_type, opline->op1 + if (opline->op1_type == IS_CONST) { + | FPU_LONG_LOAD opline->op1_type, opline->op1 + } else { + | FPU_GET_Z_LVAL Ra(op1_reg)+op1_offset + } | FPU_MATH opline->opcode, opline->op2_type, opline->op2 |.endif - if (res_forward) { - | DOUBLE_STORE r1 + res_forward, xmm0 - | SET_Z_TYPE_INFO r1 + res_forward, IS_DOUBLE - } else { - | DOUBLE_STORE FP + opline->result.var, xmm0 - | SET_Z_TYPE_INFO FP + opline->result.var, IS_DOUBLE - } + | DOUBLE_STORE Ra(res_reg)+res_offset, xmm0 + | SET_Z_TYPE_INFO Ra(res_reg)+res_offset, IS_DOUBLE return 1; } -static int zend_jit_math_double_long(dasm_State **Dst, const zend_op *opline, uint32_t res_forward) +static int zend_jit_math_double_long(dasm_State **Dst, const zend_op *opline, uint32_t op1_reg, uint32_t op1_offset, uint32_t res_reg, uint32_t res_offset) { |.if X64 or SSE - || if (opline->opcode == ZEND_ADD || opline->opcode == ZEND_MUL) { + || if (opline->opcode == ZEND_ADD || opline->opcode == ZEND_MUL || opline->opcode == ZEND_ASSIGN_ADD || opline->opcode == ZEND_ASSIGN_MUL) { | SSE_LOAD_LONG xmm0, opline->op2_type, opline->op2 - | SSE_MATH opline->opcode, xmm0, opline->op1_type, opline->op1 + || if (opline->op1_type == IS_CONST) { + | SSE_MATH opline->opcode, xmm0, opline->op1_type, opline->op1 + || } else { + | SSE_MATH2 opline->opcode, xmm0, qword [Ra(op1_reg)+op1_offset] + || } || } else { - | SSE_LOAD xmm0, opline->op1_type, opline->op1 + || if (opline->op1_type == IS_CONST) { + | SSE_LOAD xmm0, opline->op1_type, opline->op1 + || } else { + | SSE_GET_Z_DVAL xmm0, Ra(op1_reg)+op1_offset + || } | SSE_LOAD_LONG xmm1, opline->op2_type, opline->op2 | SSE_MATH2 opline->opcode, xmm0, xmm1 || } |.else | FPU_LONG_LOAD opline->op2_type, opline->op2 - || if (opline->opcode == ZEND_ADD || opline->opcode == ZEND_MUL) { - | FPU_MATH opline->opcode, opline->op1_type, opline->op1 + || if (opline->opcode == ZEND_ADD || opline->opcode == ZEND_MUL || opline->opcode == ZEND_ASSIGN_ADD || opline->opcode == ZEND_ASSIGN_MUL) { + || if (opline->op1_type == IS_CONST) { + | FPU_MATH opline->opcode, opline->op1_type, opline->op1 + || } else { + | FPU_MATH2 opline->opcode, qword [Ra(op1_reg)+op1_offset] + || } || } else { - | FPU_LOAD opline->op1_type, opline->op1 - | FPU_MATH2 opline->opcode + || if (opline->op1_type == IS_CONST) { + | FPU_LOAD opline->op1_type, opline->op1 + || } else { + | FPU_GET_Z_DVAL Ra(op1_reg)+op1_offset + || } + | FPU_MATH2 opline->opcode, st1 || } |.endif - if (res_forward) { - | DOUBLE_STORE r1 + res_forward, xmm0 - | SET_Z_TYPE_INFO r1 + res_forward, IS_DOUBLE - } else { - | DOUBLE_STORE FP + opline->result.var, xmm0 - | SET_Z_TYPE_INFO FP + opline->result.var, IS_DOUBLE + | DOUBLE_STORE Ra(res_reg)+res_offset, xmm0 + if (op1_reg != res_reg || op1_offset != res_offset) { + | SET_Z_TYPE_INFO Ra(res_reg)+res_offset, IS_DOUBLE } - return 1; } -static int zend_jit_math_double_double(dasm_State **Dst, const zend_op *opline, uint32_t res_forward) +static int zend_jit_math_double_double(dasm_State **Dst, const zend_op *opline, uint32_t op1_reg, uint32_t op1_offset, uint32_t res_reg, uint32_t res_offset) { zend_bool same_ops = (opline->op1_type == opline->op2_type) && (opline->op1.var == opline->op2.var); |.if X64 or SSE - | SSE_LOAD xmm0, opline->op1_type, opline->op1 + if (opline->op1_type == IS_CONST) { + | SSE_LOAD xmm0, opline->op1_type, opline->op1 + } else { + | SSE_GET_Z_DVAL xmm0, Ra(op1_reg)+op1_offset + } if (same_ops) { | SSE_MATH2 opline->opcode, xmm0, xmm0 } else { | SSE_MATH opline->opcode, xmm0, opline->op2_type, opline->op2 } |.else - | FPU_LOAD opline->op1_type, opline->op1 + if (opline->op1_type == IS_CONST) { + | FPU_LOAD opline->op1_type, opline->op1 + } else { + | FPU_GET_Z_DVAL Ra(op1_reg)+op1_offset + } | FPU_MATH opline->opcode, opline->op2_type, opline->op2 |.endif - if (res_forward) { - | DOUBLE_STORE r1 + res_forward, xmm0 - | SET_Z_TYPE_INFO r1 + res_forward, IS_DOUBLE - } else { - | DOUBLE_STORE FP + opline->result.var, xmm0 - | SET_Z_TYPE_INFO FP + opline->result.var, IS_DOUBLE + | DOUBLE_STORE Ra(res_reg)+res_offset, xmm0 + if (op1_reg != res_reg || op1_offset != res_offset) { + | SET_Z_TYPE_INFO Ra(res_reg)+res_offset, IS_DOUBLE } return 1; @@ -1552,51 +1640,21 @@ fallback: return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); } -static int zend_jit_math(dasm_State **Dst, const zend_op *opline, int *opnum, zend_op_array *op_array, zend_ssa *ssa) +static int zend_jit_math_helper(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, uint32_t op1_reg, uint32_t op1_offset, uint32_t res_reg, uint32_t res_offset, uint32_t op1_info, uint32_t op2_info, uint32_t res_info) { - uint32_t op1_info, op2_info; zend_bool same_ops = (opline->op1_type == opline->op2_type) && (opline->op1.var == opline->op2.var); - zend_bool has_slow; - uint32_t res_forward = 0; - - if (!ssa->ops || !ssa->var_info) { - goto fallback; - } - - op1_info = OP1_INFO(); - op2_info = OP2_INFO(); - - if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) { - goto fallback; - } - - if (!(op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) || - !(op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { - goto fallback; - } - - has_slow = + zend_bool has_slow = (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) || (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))); - if (opline->result_type == IS_TMP_VAR && - (opline+1)->opcode == ZEND_SEND_VAL && - (opline+1)->op1_type == IS_TMP_VAR && - (opline+1)->op1.var == opline->result.var) { - /* Eliminate the following SEND_VAL */ - (*opnum)++; - res_forward = (opline+1)->result.var; - | mov r1, EX->call - } - if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)) { if (op1_info & (MAY_BE_ANY-MAY_BE_LONG)) { if (op1_info & MAY_BE_DOUBLE) { - | IF_NOT_Z_TYPE FP + opline->op1.var, IS_LONG, >4 + | IF_NOT_Z_TYPE Ra(op1_reg)+op1_offset, IS_LONG, >4 } else { - | IF_NOT_Z_TYPE FP + opline->op1.var, IS_LONG, >9 + | IF_NOT_Z_TYPE Ra(op1_reg)+op1_offset, IS_LONG, >9 } } if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_LONG))) { @@ -1607,7 +1665,7 @@ static int zend_jit_math(dasm_State **Dst, const zend_op *opline, int *opnum, ze if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { | IF_NOT_Z_TYPE FP + opline->op2.var, IS_DOUBLE, >9 } - if (!zend_jit_math_long_double(Dst, opline, res_forward)) { + if (!zend_jit_math_long_double(Dst, opline, op1_reg, op1_offset, res_reg, res_offset)) { return 0; } | jmp >6 @@ -1616,14 +1674,14 @@ static int zend_jit_math(dasm_State **Dst, const zend_op *opline, int *opnum, ze | IF_NOT_Z_TYPE FP + opline->op2.var, IS_LONG, >9 } } - if (!zend_jit_math_long_long(Dst, opline, op_array, ssa, res_forward)) { + if (!zend_jit_math_long_long(Dst, opline, op_array, ssa, op1_reg, op1_offset, res_reg, res_offset, res_info)) { return 0; } if (op1_info & MAY_BE_DOUBLE) { |.cold_code |4: if (op1_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { - | IF_NOT_Z_TYPE FP + opline->op1.var, IS_DOUBLE, >9 + | IF_NOT_Z_TYPE Ra(op1_reg)+op1_offset, IS_DOUBLE, >9 } if (op2_info & MAY_BE_DOUBLE) { if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { @@ -1633,7 +1691,7 @@ static int zend_jit_math(dasm_State **Dst, const zend_op *opline, int *opnum, ze | IF_NOT_Z_TYPE, FP + opline->op2.var, IS_DOUBLE, >9 } } - if (!zend_jit_math_double_double(Dst, opline, res_forward)) { + if (!zend_jit_math_double_double(Dst, opline, op1_reg, op1_offset, res_reg, res_offset)) { return 0; } | jmp >6 @@ -1643,7 +1701,7 @@ static int zend_jit_math(dasm_State **Dst, const zend_op *opline, int *opnum, ze if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { | IF_NOT_Z_TYPE FP + opline->op2.var, IS_LONG, >9 } - if (!zend_jit_math_double_long(Dst, opline, res_forward)) { + if (!zend_jit_math_double_long(Dst, opline, op1_reg, op1_offset, res_reg, res_offset)) { return 0; } | jmp >6 @@ -1654,7 +1712,7 @@ static int zend_jit_math(dasm_State **Dst, const zend_op *opline, int *opnum, ze !(op1_info & MAY_BE_LONG) && (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { if (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) { - | IF_NOT_Z_TYPE FP + opline->op1.var, IS_DOUBLE, >9 + | IF_NOT_Z_TYPE Ra(op1_reg)+op1_offset, IS_DOUBLE, >9 } if (op2_info & MAY_BE_DOUBLE) { if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { @@ -1664,7 +1722,7 @@ static int zend_jit_math(dasm_State **Dst, const zend_op *opline, int *opnum, ze | IF_NOT_Z_TYPE FP + opline->op2.var, IS_DOUBLE, >9 } } - if (!zend_jit_math_double_double(Dst, opline, res_forward)) { + if (!zend_jit_math_double_double(Dst, opline, op1_reg, op1_offset, res_reg, res_offset)) { return 0; } } @@ -1676,7 +1734,7 @@ static int zend_jit_math(dasm_State **Dst, const zend_op *opline, int *opnum, ze if (op2_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) { | IF_NOT_Z_TYPE FP + opline->op2.var, IS_LONG, >9 } - if (!zend_jit_math_double_long(Dst, opline, res_forward)) { + if (!zend_jit_math_double_long(Dst, opline, op1_reg, op1_offset, res_reg, res_offset)) { return 0; } if (op2_info & MAY_BE_DOUBLE) { @@ -1693,12 +1751,12 @@ static int zend_jit_math(dasm_State **Dst, const zend_op *opline, int *opnum, ze if (op1_info & MAY_BE_DOUBLE) { if (!same_ops && (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { if (!same_ops && (op1_info & MAY_BE_LONG)) { - | IF_NOT_Z_TYPE FP + opline->op1.var, IS_DOUBLE, >3 + | IF_NOT_Z_TYPE Ra(op1_reg)+op1_offset, IS_DOUBLE, >3 } else { - | IF_NOT_Z_TYPE FP + opline->op1.var, IS_DOUBLE, >9 + | IF_NOT_Z_TYPE Ra(op1_reg)+op1_offset, IS_DOUBLE, >9 } } - if (!zend_jit_math_double_double(Dst, opline, res_forward)) { + if (!zend_jit_math_double_double(Dst, opline, op1_reg, op1_offset, res_reg, res_offset)) { return 0; } } @@ -1708,9 +1766,9 @@ static int zend_jit_math(dasm_State **Dst, const zend_op *opline, int *opnum, ze } |3: if (op1_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) { - | IF_NOT_Z_TYPE FP + opline->op1.var, IS_LONG, >9 + | IF_NOT_Z_TYPE Ra(op1_reg)+op1_offset, IS_LONG, >9 } - if (!zend_jit_math_long_double(Dst, opline, res_forward)) { + if (!zend_jit_math_long_double(Dst, opline, op1_reg, op1_offset, res_reg, res_offset)) { return 0; } if (op1_info & MAY_BE_DOUBLE) { @@ -1726,24 +1784,26 @@ static int zend_jit_math(dasm_State **Dst, const zend_op *opline, int *opnum, ze |.cold_code |9: | SAVE_VALID_OPLINE opline - if (res_forward) { - | lea FCARG1a, [r1 + res_forward] + if (res_reg != FCARG1a || res_offset != 0) { + | lea FCARG1a, [Ra(res_reg)+res_offset] + } + if (opline->op1_type == IS_CONST) { + | LOAD_ZVAL_ADDR FCARG2a, opline->op1_type, opline->op1 } else { - | lea FCARG1a, [FP + opline->result.var] + | lea FCARG2a, [Ra(op1_reg)+op1_offset] } - | LOAD_ZVAL_ADDR FCARG2a, opline->op1_type, opline->op1 |.if X64 | LOAD_ZVAL_ADDR CARG3, opline->op2_type, opline->op2 |.else | PUSH_ZVAL_ADDR opline->op2_type, opline->op2, r0 |.endif - || if (opline->opcode == ZEND_ADD) { + || if (opline->opcode == ZEND_ADD || opline->opcode == ZEND_ASSIGN_ADD) { | EXT_CALL add_function, r0 - || } else if (opline->opcode == ZEND_SUB) { + || } else if (opline->opcode == ZEND_SUB || opline->opcode == ZEND_ASSIGN_SUB) { | EXT_CALL sub_function, r0 - || } else if (opline->opcode == ZEND_MUL) { + || } else if (opline->opcode == ZEND_MUL || opline->opcode == ZEND_ASSIGN_MUL) { | EXT_CALL mul_function, r0 - || } else if (opline->opcode == ZEND_DIV) { + || } else if (opline->opcode == ZEND_DIV || opline->opcode == ZEND_ASSIGN_DIV) { | EXT_CALL div_function, r0 || } else { || ZEND_ASSERT(0); @@ -1758,6 +1818,123 @@ static int zend_jit_math(dasm_State **Dst, const zend_op *opline, int *opnum, ze } return 1; +} + +static int zend_jit_math(dasm_State **Dst, const zend_op *opline, int *opnum, zend_op_array *op_array, zend_ssa *ssa) +{ + uint32_t op1_info, op2_info; + + if (!ssa->ops || !ssa->var_info) { + goto fallback; + } + + op1_info = OP1_INFO(); + op2_info = OP2_INFO(); + + if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) { + goto fallback; + } + + if (!(op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) || + !(op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { + goto fallback; + } + + if (opline->result_type == IS_TMP_VAR && + (opline+1)->opcode == ZEND_SEND_VAL && + (opline+1)->op1_type == IS_TMP_VAR && + (opline+1)->op1.var == opline->result.var) { + /* Eliminate the following SEND_VAL */ + (*opnum)++; + | mov FCARG1a, EX->call + return zend_jit_math_helper(Dst, opline, op_array, ssa, FP, opline->op1.var, FCARG1a, (opline+1)->result.var, op1_info, op2_info, RES_INFO()); + } else { + return zend_jit_math_helper(Dst, opline, op_array, ssa, FP, opline->op1.var, FP, opline->result.var, op1_info, op2_info, RES_INFO()); + } + +fallback: + /* fallback to subroutine threading */ + return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); +} + +static int zend_jit_assign_dim_math(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) +{ + // TODO: ??? + return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); +} + +static int zend_jit_assign_math(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) +{ + uint32_t op1_info, op2_info; + + if (opline->extended_value == ZEND_ASSIGN_DIM) { + return zend_jit_assign_dim_math(Dst, opline, op_array, ssa); + } else if (opline->extended_value == ZEND_ASSIGN_OBJ) { + goto fallback; + } + + if (opline->op1_type != IS_CV || opline->result_type != IS_UNUSED) { + goto fallback; + } + + if (!ssa->ops || !ssa->var_info) { + goto fallback; + } + + op1_info = OP1_INFO(); + op2_info = OP2_INFO(); + + if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) { + goto fallback; + } + + if (!(op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) || + !(op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { + goto fallback; + } + + if (op1_info & MAY_BE_REF) { + | LOAD_ZVAL_ADDR FCARG1a, opline->op1_type, opline->op1 + | ZVAL_DEREF FCARG1a, op1_info + if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY)) { + | // SEPARATE_ZVAL_NOREF(var); + | IF_Z_FLAGS FCARG1a, IS_TYPE_COPYABLE + IS_TYPE_IMMUTABLE, >1 + |.cold_code + |1: + | GET_Z_PTR r0, FCARG1a + | cmp dword [r0], 1 // if (GC_REFCOUNTED() > 1) + | jbe >3 + | IF_Z_FLAGS FCARG1a, IS_TYPE_IMMUTABLE, >2 + | GC_DELREF r0 + |2: + | mov aword [r4], FCARG1a // save + | ZVAL_COPY_CTOR_FUNC (op_array->filename ? ZSTR_VAL(op_array->filename) : NULL), opline->lineno + | mov FCARG1a, aword [r4] // restore + | jmp >3 + |.code + |3: + } + return zend_jit_math_helper(Dst, opline, op_array, ssa, FCARG1a, 0, FCARG1a, 0, op1_info, op2_info, OP1_DEF_INFO()); + } else { + if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY)) { + | // SEPARATE_ZVAL_NOREF(var); + | IF_Z_FLAGS FP + opline->op1.var, IS_TYPE_COPYABLE + IS_TYPE_IMMUTABLE, >1 + |.cold_code + |1: + | GET_Z_PTR r0, FP + opline->op1.var + | cmp dword [r0], 1 // if (GC_REFCOUNTED() > 1) + | jbe >3 + | IF_Z_FLAGS FP + opline->op1.var, IS_TYPE_IMMUTABLE, >2 + | GC_DELREF r0 + |2: + | lea FCARG1a, [FP + opline->op1.var] + | ZVAL_COPY_CTOR_FUNC (op_array->filename ? ZSTR_VAL(op_array->filename) : NULL), opline->lineno + | jmp >3 + |.code + |3: + } + return zend_jit_math_helper(Dst, opline, op_array, ssa, FP, opline->op1.var, FP, opline->op1.var, op1_info, op2_info, OP1_DEF_INFO()); + } fallback: /* fallback to subroutine threading */ @@ -2755,29 +2932,29 @@ static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, zen uint32_t used_stack = zend_vm_calc_used_stack(opline->extended_value, func); | // if (UNEXPECTED(used_stack > (size_t)(((char*)EG(vm_stack_end)) - (char*)call))) { - | mov r1, aword [&EG(vm_stack_top)] + | mov FCARG1a, aword [&EG(vm_stack_top)] | mov r2, aword [&EG(vm_stack_end)] - | sub r2,r1 + | sub r2, FCARG1a | cmp r2, used_stack | jb >2 | // EG(vm_stack_top) = (zval*)((char*)call + used_stack); | add aword [&EG(vm_stack_top)], used_stack | // zend_vm_init_call_frame(call, call_info, func, num_args, called_scope, object); | // call->func = func; - | mov aword EX:r1->func, r0 + | mov aword EX:FCARG1a->func, r0 | // ZEND_SET_CALL_INFO(call, 0, call_info); - | mov dword EX:r1->This.u1.type_info, (IS_UNDEF | (ZEND_CALL_NESTED_FUNCTION << ZEND_CALL_INFO_SHIFT)) + | mov dword EX:FCARG1a->This.u1.type_info, (IS_UNDEF | (ZEND_CALL_NESTED_FUNCTION << ZEND_CALL_INFO_SHIFT)) |1: | // Z_CE(call->This) = called_scope; - | mov aword EX:r1->This.value.ptr, 0 + | mov aword EX:FCARG1a->This.value.ptr, 0 | // ZEND_CALL_NUM_ARGS(call) = num_args; - | mov dword EX:r1->This.u2.num_args, opline->extended_value + | mov dword EX:FCARG1a->This.u2.num_args, opline->extended_value |.cold_code |2: | mov FCARG1d, used_stack | mov FCARG2a, r0 | EXT_CALL zend_jit_extend_stack_helper, r0 - | mov r1, r0 + | mov FCARG1a, r0 | jmp <1 |.code @@ -2833,13 +3010,13 @@ static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ } | // call->prev_execute_data = EX(call); if (call_level == 1) { - | mov aword EX:r1->prev_execute_data, 0 + | mov aword EX:FCARG1a->prev_execute_data, 0 } else { | mov r0, EX->call - | mov EX:r1->prev_execute_data, r0 + | mov EX:FCARG1a->prev_execute_data, r0 } | // EX(call) = call; - | mov EX->call, r1 + | mov EX->call, FCARG1a return 1; } @@ -2869,37 +3046,37 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar } | // call = EX(call); - | mov r1, EX->call + | mov FCARG1a, EX->call | // fbc = call->func; - | // mov r2, EX:r1->func ??? + | // mov r2, EX:FCARG1a->func ??? | // SAVE_OPLINE(); | SAVE_VALID_OPLINE opline || if (call_level == 1) { | mov aword EX->call, 0 || } else { | //EX(call) = call->prev_execute_data; - | mov r0, EX:r1->prev_execute_data + | mov r0, EX:FCARG1a->prev_execute_data | mov EX->call, r0 || } | //call->prev_execute_data = execute_data; - | mov EX:r1->prev_execute_data, EX + | mov EX:FCARG1a->prev_execute_data, EX | | // EX(call) = NULL; - | mov aword EX:r1->call, 0 + | mov aword EX:FCARG1a->call, 0 || if (RETURN_VALUE_USED(opline)) { | // ZVAL_NULL(EX_VAR(opline->result.var)); | SET_Z_TYPE_INFO FP + opline->result.var, IS_NULL | // EX(return_value) = EX_VAR(opline->result.var); | lea r0, aword [FP + opline->result.var] - | mov aword EX:r1->return_value, r0 + | mov aword EX:FCARG1a->return_value, r0 || } else { | // EX(return_value) = 0; - | mov aword EX:r1->return_value, 0 + | mov aword EX:FCARG1a->return_value, 0 || } | || for (i = call_info->num_args; i < func->op_array.last_var; i++) { || uint32_t n = (uint32_t)(uintptr_t)ZEND_CALL_VAR_NUM(NULL, i); - | SET_Z_TYPE_INFO r1 + n, IS_UNDEF + | SET_Z_TYPE_INFO FCARG1a + n, IS_UNDEF || } | | //EX_LOAD_RUN_TIME_CACHE(op_array); @@ -2908,22 +3085,22 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar || /* recursive call */ || if (func->op_array.cache_size > sizeof(void*)) { | mov r0, EX->run_time_cache - | mov EX:r1->run_time_cache, r0 + | mov EX:FCARG1a->run_time_cache, r0 || } || } else { - | mov r2, EX:r1->func + | mov r2, EX:FCARG1a->func | mov r0, aword [r2 + offsetof(zend_op_array, run_time_cache)] - | mov EX:r1->run_time_cache, r0 + | mov EX:FCARG1a->run_time_cache, r0 || } || } | //EX_LOAD_LITERALS(op_array); |.if X64 | LOAD_ADDR r0, func->op_array.literals - | mov EX:r1->literals, r0 + | mov EX:FCARG1a->literals, r0 |.endif | // EG(current_execute_data) = execute_data; - | mov aword [&EG(current_execute_data)], r1 - | mov FP, r1 + | mov aword [&EG(current_execute_data)], FCARG1a + | mov FP, FCARG1a | // opline = op_array->opcodes; | LOAD_ADDR IP, (func->op_array.opcodes + call_info->num_args) || if (func && op_array == &func->op_array) { @@ -2966,12 +3143,12 @@ static int zend_jit_send_val(dasm_State **Dst, const zend_op *opline, zend_op_ar op1_info = OP1_INFO(); - | mov r1, EX->call + | mov FCARG1a, EX->call if (opline->opcode == ZEND_SEND_VAL_EX) { uint32_t mask = ZEND_SEND_BY_REF << ((arg_num + 3) * 2); - | mov r0, EX:r1->func + | mov r0, EX:FCARG1a->func if (arg_num <= MAX_ARG_FLAG_NUM) { | mov eax, dword [r0 + offsetof(zend_function, quick_arg_flags)] | test eax, mask @@ -2981,7 +3158,7 @@ static int zend_jit_send_val(dasm_State **Dst, const zend_op *opline, zend_op_ar } |.cold_code |1: - | SET_Z_TYPE_INFO r1 + opline->result.var, IS_UNDEF + | SET_Z_TYPE_INFO FCARG1a + opline->result.var, IS_UNDEF | SAVE_VALID_OPLINE opline |.if X64 | mov CARG1, 0 @@ -3004,12 +3181,12 @@ static int zend_jit_send_val(dasm_State **Dst, const zend_op *opline, zend_op_ar if (opline->op1_type == IS_CONST) { zval *zv = RT_CONSTANT(op_array, opline->op1); - | ZVAL_COPY_CONST r1 + opline->result.var, -1, zv, r0 + | ZVAL_COPY_CONST FCARG1a + opline->result.var, -1, zv, r0 || if (Z_REFCOUNTED_P(zv)) { | ADDREF_CONST zv, r0 || } } else { - | ZVAL_COPY_VALUE r1 + opline->result.var, FP + opline->op1.var, op1_info, r0, eax, r2 + | ZVAL_COPY_VALUE FCARG1a + opline->result.var, FP + opline->op1.var, op1_info, r0, eax, r2 } return 1; @@ -3033,8 +3210,8 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, zend_op_ar } op1_info = OP1_INFO(); - | mov r1, EX->call - | ZVAL_COPY_VALUE r1 + opline->result.var, FP + opline->op1.var, op1_info, r0, eax, r2 + | mov FCARG1a, EX->call + | ZVAL_COPY_VALUE FCARG1a + opline->result.var, FP + opline->op1.var, op1_info, r0, eax, r2 || if (opline->op1_type == IS_CV) { | TRY_ADDREF op1_info, ah, r2 || } From 259ef5ed33a4ea38a4cc2e5c7c21709cf9ada6f8 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 27 Sep 2016 19:09:15 +0300 Subject: [PATCH 246/569] JIT for ASSIGN_ADD/SUB/MUL with ASSIGN_DIM (incomplete) --- ext/opcache/jit/zend_jit_x86.dasc | 859 +++++++++++++++++++++--------- 1 file changed, 610 insertions(+), 249 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index febe22ead7ea0..8c03b637e8919 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -1393,47 +1393,67 @@ fallback: return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); } -static int zend_jit_math_long_long(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, uint32_t op1_reg, uint32_t op1_offset, uint32_t res_reg, uint32_t res_offset, uint32_t res_info) +static int zend_jit_math_long_long(dasm_State **Dst, + const zend_op *opline, + zend_op_array *op_array, + zend_ssa *ssa, + zend_uchar op1_type, + znode_op op1, + uint32_t op1_reg, + uint32_t op1_offset, + zend_uchar op2_type, + znode_op op2, + uint32_t op2_reg, + uint32_t op2_offset, + uint32_t res_reg, + uint32_t res_offset, + uint32_t res_info) { - zend_bool same_ops = (opline->op1_type == opline->op2_type) && (opline->op1.var == opline->op2.var); + zend_bool same_ops = (op1_type == op2_type) && (op1.var == op2.var); if (opline->opcode == ZEND_MUL && - ((opline->op2_type == IS_CONST && - IS_SIGNED_32BIT(Z_LVAL_P(RT_CONSTANT(op_array, opline->op2))) && - is_power_of_two(Z_LVAL_P(RT_CONSTANT(op_array, opline->op2)))) || - (opline->op1_type == IS_CONST && - IS_SIGNED_32BIT(Z_LVAL_P(RT_CONSTANT(op_array, opline->op1))) && - is_power_of_two(Z_LVAL_P(RT_CONSTANT(op_array, opline->op1)))))) { - if (opline->op2_type == IS_CONST) { - if (opline->op1_type == IS_CONST) { - | LONG_LOAD r0, opline->op1_type, opline->op1 + ((op2_type == IS_CONST && + IS_SIGNED_32BIT(Z_LVAL_P(RT_CONSTANT(op_array, op2))) && + is_power_of_two(Z_LVAL_P(RT_CONSTANT(op_array, op2)))) || + (op1_type == IS_CONST && + IS_SIGNED_32BIT(Z_LVAL_P(RT_CONSTANT(op_array, op1))) && + is_power_of_two(Z_LVAL_P(RT_CONSTANT(op_array, op1)))))) { + if (op2_type == IS_CONST) { + if (op1_type == IS_CONST) { + | LONG_LOAD r0, op1_type, op1 } else { | GET_Z_LVAL r0, Ra(op1_reg)+op1_offset } - | shl r0, floor_log2(Z_LVAL_P(RT_CONSTANT(op_array, opline->op2))) + | shl r0, floor_log2(Z_LVAL_P(RT_CONSTANT(op_array, op2))) } else { - | LONG_LOAD r0, opline->op2_type, opline->op2 - | shl r0, floor_log2(Z_LVAL_P(RT_CONSTANT(op_array, opline->op1))) + if (op2_type == IS_CONST) { + | LONG_LOAD r0, op2_type, op2 + } else { + | GET_Z_LVAL r0, Ra(op2_reg)+op2_offset + } + | shl r0, floor_log2(Z_LVAL_P(RT_CONSTANT(op_array, op1))) } } else if (opline->opcode == ZEND_DIV && - (opline->op2_type == IS_CONST && - is_power_of_two(Z_LVAL_P(RT_CONSTANT(op_array, opline->op2))))) { - if (opline->op1_type == IS_CONST) { - | LONG_LOAD r0, opline->op1_type, opline->op1 + (op2_type == IS_CONST && + is_power_of_two(Z_LVAL_P(RT_CONSTANT(op_array, op2))))) { + if (op1_type == IS_CONST) { + | LONG_LOAD r0, op1_type, op1 } else { | GET_Z_LVAL r0, Ra(op1_reg)+op1_offset } - | shr r0, floor_log2(Z_LVAL_P(RT_CONSTANT(op_array, opline->op2))) + | shr r0, floor_log2(Z_LVAL_P(RT_CONSTANT(op_array, op2))) } else { - if (opline->op1_type == IS_CONST) { - | LONG_LOAD r0, opline->op1_type, opline->op1 + if (op1_type == IS_CONST) { + | LONG_LOAD r0, op1_type, op1 } else { | GET_Z_LVAL r0, Ra(op1_reg)+op1_offset } if (same_ops && opline->opcode != ZEND_DIV) { | LONG_MATH2 opline->opcode, r0, r0 + } else if (op2_type == IS_CONST) { + | LONG_MATH opline->opcode, r0, op2_type, op2 } else { - | LONG_MATH opline->opcode, r0, opline->op2_type, opline->op2 + | LONG_MATH2 opline->opcode, r0, aword [Ra(op2_reg)+op2_offset] } } if ((res_info & MAY_BE_DOUBLE) && zend_may_overflow(opline, op_array, ssa)) { @@ -1441,17 +1461,25 @@ static int zend_jit_math_long_long(dasm_State **Dst, const zend_op *opline, zend |.cold_code |1: |.if X64 or SSE - if (opline->op1_type == IS_CONST) { - | SSE_LOAD_LONG xmm0, opline->op1_type, opline->op1 + if (op1_type == IS_CONST) { + | SSE_LOAD_LONG xmm0, op1_type, op1 } else { | SSE_GET_Z_LVAL xmm0, Ra(op1_reg)+op1_offset } - | SSE_LOAD_LONG xmm1, opline->op2_type, opline->op2 + if (op2_type == IS_CONST) { + | SSE_LOAD_LONG xmm1, op2_type, op2 + } else { + | SSE_GET_Z_LVAL xmm1, Ra(op2_reg)+op2_offset + } | SSE_MATH2 opline->opcode, xmm0, xmm1 |.else - | FPU_LONG_LOAD opline->op2_type, opline->op2 - if (opline->op1_type == IS_CONST) { - | FPU_LONG_LOAD opline->op1_type, opline->op1 + if (op2_type == IS_CONST) { + | FPU_LONG_LOAD op2_type, op2 + } else { + | FPU_GET_Z_LVAL Ra(op2_reg)+op2_offset + } + if (op1_type == IS_CONST) { + | FPU_LONG_LOAD op1_type, op1 } else { | FPU_GET_Z_LVAL Ra(op1_reg)+op1_offset } @@ -1476,22 +1504,41 @@ static int zend_jit_math_long_long(dasm_State **Dst, const zend_op *opline, zend return 1; } -static int zend_jit_math_long_double(dasm_State **Dst, const zend_op *opline, uint32_t op1_reg, uint32_t op1_offset, uint32_t res_reg, uint32_t res_offset) +static int zend_jit_math_long_double(dasm_State **Dst, + const zend_op *opline, + zend_uchar op1_type, + znode_op op1, + uint32_t op1_reg, + uint32_t op1_offset, + zend_uchar op2_type, + znode_op op2, + uint32_t op2_reg, + uint32_t op2_offset, + uint32_t res_reg, + uint32_t res_offset) { |.if X64 or SSE - if (opline->op1_type == IS_CONST) { - | SSE_LOAD_LONG xmm0, opline->op1_type, opline->op1 + if (op1_type == IS_CONST) { + | SSE_LOAD_LONG xmm0, op1_type, op1 } else { | SSE_GET_Z_LVAL xmm0, Ra(op1_reg)+op1_offset } - | SSE_MATH opline->opcode, xmm0, opline->op2_type, opline->op2 + if (op2_type == IS_CONST) { + | SSE_MATH opline->opcode, xmm0, op2_type, op2 + } else { + | SSE_MATH2 opline->opcode, xmm0, qword [Ra(op2_reg)+op2_offset] + } |.else - if (opline->op1_type == IS_CONST) { - | FPU_LONG_LOAD opline->op1_type, opline->op1 + if (op1_type == IS_CONST) { + | FPU_LONG_LOAD op1_type, op1 } else { | FPU_GET_Z_LVAL Ra(op1_reg)+op1_offset } - | FPU_MATH opline->opcode, opline->op2_type, opline->op2 + if (op2_type == IS_CONST) { + | FPU_MATH opline->opcode, op2_type, op2 + } else { + | FPU_MATH2 opline->opcode, qword [Ra(op2_reg)+op2_offset] + } |.endif | DOUBLE_STORE Ra(res_reg)+res_offset, xmm0 | SET_Z_TYPE_INFO Ra(res_reg)+res_offset, IS_DOUBLE @@ -1499,36 +1546,59 @@ static int zend_jit_math_long_double(dasm_State **Dst, const zend_op *opline, ui return 1; } -static int zend_jit_math_double_long(dasm_State **Dst, const zend_op *opline, uint32_t op1_reg, uint32_t op1_offset, uint32_t res_reg, uint32_t res_offset) +static int zend_jit_math_double_long(dasm_State **Dst, + const zend_op *opline, + zend_uchar op1_type, + znode_op op1, + uint32_t op1_reg, + uint32_t op1_offset, + zend_uchar op2_type, + znode_op op2, + uint32_t op2_reg, + uint32_t op2_offset, + uint32_t res_reg, + uint32_t res_offset) { |.if X64 or SSE || if (opline->opcode == ZEND_ADD || opline->opcode == ZEND_MUL || opline->opcode == ZEND_ASSIGN_ADD || opline->opcode == ZEND_ASSIGN_MUL) { - | SSE_LOAD_LONG xmm0, opline->op2_type, opline->op2 - || if (opline->op1_type == IS_CONST) { - | SSE_MATH opline->opcode, xmm0, opline->op1_type, opline->op1 + || if (op2_type == IS_CONST) { + | SSE_LOAD_LONG xmm0, op2_type, op2 + || } else { + | SSE_GET_Z_LVAL xmm0, Ra(op2_reg)+op2_offset + || } + || if (op1_type == IS_CONST) { + | SSE_MATH opline->opcode, xmm0, op1_type, op1 || } else { | SSE_MATH2 opline->opcode, xmm0, qword [Ra(op1_reg)+op1_offset] || } || } else { - || if (opline->op1_type == IS_CONST) { - | SSE_LOAD xmm0, opline->op1_type, opline->op1 + || if (op1_type == IS_CONST) { + | SSE_LOAD xmm0, op1_type, op1 || } else { | SSE_GET_Z_DVAL xmm0, Ra(op1_reg)+op1_offset || } - | SSE_LOAD_LONG xmm1, opline->op2_type, opline->op2 + || if (op2_type == IS_CONST) { + | SSE_LOAD_LONG xmm1, op2_type, op2 + || } else { + | SSE_GET_Z_LVAL xmm1, Ra(op2_reg)+op2_offset + || } | SSE_MATH2 opline->opcode, xmm0, xmm1 || } |.else - | FPU_LONG_LOAD opline->op2_type, opline->op2 + || if (op2_type == IS_CONST) { + | FPU_LONG_LOAD op2_type, op2 + || } else { + | FPU_GET_Z_LVAL Ra(op2_reg)+op2_offset + || } || if (opline->opcode == ZEND_ADD || opline->opcode == ZEND_MUL || opline->opcode == ZEND_ASSIGN_ADD || opline->opcode == ZEND_ASSIGN_MUL) { - || if (opline->op1_type == IS_CONST) { - | FPU_MATH opline->opcode, opline->op1_type, opline->op1 + || if (op1_type == IS_CONST) { + | FPU_MATH opline->opcode, op1_type, op1 || } else { | FPU_MATH2 opline->opcode, qword [Ra(op1_reg)+op1_offset] || } || } else { - || if (opline->op1_type == IS_CONST) { - | FPU_LOAD opline->op1_type, opline->op1 + || if (op1_type == IS_CONST) { + | FPU_LOAD op1_type, op1 || } else { | FPU_GET_Z_DVAL Ra(op1_reg)+op1_offset || } @@ -1542,28 +1612,45 @@ static int zend_jit_math_double_long(dasm_State **Dst, const zend_op *opline, ui return 1; } -static int zend_jit_math_double_double(dasm_State **Dst, const zend_op *opline, uint32_t op1_reg, uint32_t op1_offset, uint32_t res_reg, uint32_t res_offset) +static int zend_jit_math_double_double(dasm_State **Dst, + const zend_op *opline, + zend_uchar op1_type, + znode_op op1, + uint32_t op1_reg, + uint32_t op1_offset, + zend_uchar op2_type, + znode_op op2, + uint32_t op2_reg, + uint32_t op2_offset, + uint32_t res_reg, + uint32_t res_offset) { - zend_bool same_ops = (opline->op1_type == opline->op2_type) && (opline->op1.var == opline->op2.var); + zend_bool same_ops = (op1_type == op2_type) && (op1.var == op2.var); |.if X64 or SSE - if (opline->op1_type == IS_CONST) { - | SSE_LOAD xmm0, opline->op1_type, opline->op1 + if (op1_type == IS_CONST) { + | SSE_LOAD xmm0, op1_type, op1 } else { | SSE_GET_Z_DVAL xmm0, Ra(op1_reg)+op1_offset } if (same_ops) { | SSE_MATH2 opline->opcode, xmm0, xmm0 + } else if (op2_type == IS_CONST) { + | SSE_MATH opline->opcode, xmm0, op2_type, op2 } else { - | SSE_MATH opline->opcode, xmm0, opline->op2_type, opline->op2 + | SSE_MATH2 opline->opcode, xmm0, qword [Ra(op2_reg)+op2_offset] } |.else - if (opline->op1_type == IS_CONST) { - | FPU_LOAD opline->op1_type, opline->op1 + if (op1_type == IS_CONST) { + | FPU_LOAD op1_type, op1 } else { | FPU_GET_Z_DVAL Ra(op1_reg)+op1_offset } - | FPU_MATH opline->opcode, opline->op2_type, opline->op2 + if (op2_type == IS_CONST) { + | FPU_MATH opline->opcode, op2_type, op2 + } else { + | FPU_MATH2 opline->opcode, qword [Ra(op2_reg)+op2_offset] + } |.endif | DOUBLE_STORE Ra(res_reg)+res_offset, xmm0 if (op1_reg != res_reg || op1_offset != res_offset) { @@ -1640,9 +1727,25 @@ fallback: return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); } -static int zend_jit_math_helper(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, uint32_t op1_reg, uint32_t op1_offset, uint32_t res_reg, uint32_t res_offset, uint32_t op1_info, uint32_t op2_info, uint32_t res_info) +static int zend_jit_math_helper(dasm_State **Dst, + const zend_op *opline, + zend_op_array *op_array, + zend_ssa *ssa, + zend_uchar op1_type, + znode_op op1, + uint32_t op1_reg, + uint32_t op1_offset, + uint32_t op1_info, + zend_uchar op2_type, + znode_op op2, + uint32_t op2_reg, + uint32_t op2_offset, + uint32_t op2_info, + uint32_t res_reg, + uint32_t res_offset, + uint32_t res_info) { - zend_bool same_ops = (opline->op1_type == opline->op2_type) && (opline->op1.var == opline->op2.var); + zend_bool same_ops = (op1_type == op2_type) && (op1.var == op2.var); zend_bool has_slow = (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && @@ -1659,22 +1762,22 @@ static int zend_jit_math_helper(dasm_State **Dst, const zend_op *opline, zend_op } if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_LONG))) { if (op2_info & MAY_BE_DOUBLE) { - | IF_NOT_Z_TYPE FP + opline->op2.var, IS_LONG, >3 + | IF_NOT_Z_TYPE Ra(op2_reg)+op2_offset, IS_LONG, >3 |.cold_code |3: if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { - | IF_NOT_Z_TYPE FP + opline->op2.var, IS_DOUBLE, >9 + | IF_NOT_Z_TYPE Ra(op2_reg)+op2_offset, IS_DOUBLE, >9 } - if (!zend_jit_math_long_double(Dst, opline, op1_reg, op1_offset, res_reg, res_offset)) { + if (!zend_jit_math_long_double(Dst, opline, op1_type, op1, op1_reg, op1_offset, op2_type, op2, op2_reg, op2_offset, res_reg, res_offset)) { return 0; } | jmp >6 |.code } else { - | IF_NOT_Z_TYPE FP + opline->op2.var, IS_LONG, >9 + | IF_NOT_Z_TYPE Ra(op2_reg)+op2_offset, IS_LONG, >9 } } - if (!zend_jit_math_long_long(Dst, opline, op_array, ssa, op1_reg, op1_offset, res_reg, res_offset, res_info)) { + if (!zend_jit_math_long_long(Dst, opline, op_array, ssa, op1_type, op1, op1_reg, op1_offset, op2_type, op2, op2_reg, op2_offset, res_reg, res_offset, res_info)) { return 0; } if (op1_info & MAY_BE_DOUBLE) { @@ -1686,12 +1789,12 @@ static int zend_jit_math_helper(dasm_State **Dst, const zend_op *opline, zend_op if (op2_info & MAY_BE_DOUBLE) { if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { if (!same_ops) { - | IF_NOT_Z_TYPE, FP + opline->op2.var, IS_DOUBLE, >5 + | IF_NOT_Z_TYPE, Ra(op2_reg)+op2_offset, IS_DOUBLE, >5 } else { - | IF_NOT_Z_TYPE, FP + opline->op2.var, IS_DOUBLE, >9 + | IF_NOT_Z_TYPE, Ra(op2_reg)+op2_offset, IS_DOUBLE, >9 } } - if (!zend_jit_math_double_double(Dst, opline, op1_reg, op1_offset, res_reg, res_offset)) { + if (!zend_jit_math_double_double(Dst, opline, op1_type, op1, op1_reg, op1_offset, op2_type, op2, op2_reg, op2_offset, res_reg, res_offset)) { return 0; } | jmp >6 @@ -1699,9 +1802,9 @@ static int zend_jit_math_helper(dasm_State **Dst, const zend_op *opline, zend_op if (!same_ops) { |5: if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { - | IF_NOT_Z_TYPE FP + opline->op2.var, IS_LONG, >9 + | IF_NOT_Z_TYPE Ra(op2_reg)+op2_offset, IS_LONG, >9 } - if (!zend_jit_math_double_long(Dst, opline, op1_reg, op1_offset, res_reg, res_offset)) { + if (!zend_jit_math_double_long(Dst, opline, op1_type, op1, op1_reg, op1_offset, op2_type, op2, op2_reg, op2_offset, res_reg, res_offset)) { return 0; } | jmp >6 @@ -1717,12 +1820,12 @@ static int zend_jit_math_helper(dasm_State **Dst, const zend_op *opline, zend_op if (op2_info & MAY_BE_DOUBLE) { if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { if (!same_ops && (op2_info & MAY_BE_LONG)) { - | IF_NOT_Z_TYPE FP + opline->op2.var, IS_DOUBLE, >3 + | IF_NOT_Z_TYPE Ra(op2_reg)+op2_offset, IS_DOUBLE, >3 } else { - | IF_NOT_Z_TYPE FP + opline->op2.var, IS_DOUBLE, >9 + | IF_NOT_Z_TYPE Ra(op2_reg)+op2_offset, IS_DOUBLE, >9 } } - if (!zend_jit_math_double_double(Dst, opline, op1_reg, op1_offset, res_reg, res_offset)) { + if (!zend_jit_math_double_double(Dst, opline, op1_type, op1, op1_reg, op1_offset, op2_type, op2, op2_reg, op2_offset, res_reg, res_offset)) { return 0; } } @@ -1732,9 +1835,9 @@ static int zend_jit_math_helper(dasm_State **Dst, const zend_op *opline, zend_op } |3: if (op2_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) { - | IF_NOT_Z_TYPE FP + opline->op2.var, IS_LONG, >9 + | IF_NOT_Z_TYPE Ra(op2_reg)+op2_offset, IS_LONG, >9 } - if (!zend_jit_math_double_long(Dst, opline, op1_reg, op1_offset, res_reg, res_offset)) { + if (!zend_jit_math_double_long(Dst, opline, op1_type, op1, op1_reg, op1_offset, op2_type, op2, op2_reg, op2_offset, res_reg, res_offset)) { return 0; } if (op2_info & MAY_BE_DOUBLE) { @@ -1746,7 +1849,7 @@ static int zend_jit_math_helper(dasm_State **Dst, const zend_op *opline, zend_op !(op2_info & MAY_BE_LONG) && (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { if (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) { - | IF_NOT_Z_TYPE FP + opline->op2.var, IS_DOUBLE, >9 + | IF_NOT_Z_TYPE Ra(op2_reg)+op2_offset, IS_DOUBLE, >9 } if (op1_info & MAY_BE_DOUBLE) { if (!same_ops && (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { @@ -1756,7 +1859,7 @@ static int zend_jit_math_helper(dasm_State **Dst, const zend_op *opline, zend_op | IF_NOT_Z_TYPE Ra(op1_reg)+op1_offset, IS_DOUBLE, >9 } } - if (!zend_jit_math_double_double(Dst, opline, op1_reg, op1_offset, res_reg, res_offset)) { + if (!zend_jit_math_double_double(Dst, opline, op1_type, op1, op1_reg, op1_offset, op2_type, op2, op2_reg, op2_offset, res_reg, res_offset)) { return 0; } } @@ -1768,7 +1871,7 @@ static int zend_jit_math_helper(dasm_State **Dst, const zend_op *opline, zend_op if (op1_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) { | IF_NOT_Z_TYPE Ra(op1_reg)+op1_offset, IS_LONG, >9 } - if (!zend_jit_math_long_double(Dst, opline, op1_reg, op1_offset, res_reg, res_offset)) { + if (!zend_jit_math_long_double(Dst, opline, op1_type, op1, op1_reg, op1_offset, op2_type, op2, op2_reg, op2_offset, res_reg, res_offset)) { return 0; } if (op1_info & MAY_BE_DOUBLE) { @@ -1787,16 +1890,25 @@ static int zend_jit_math_helper(dasm_State **Dst, const zend_op *opline, zend_op if (res_reg != FCARG1a || res_offset != 0) { | lea FCARG1a, [Ra(res_reg)+res_offset] } - if (opline->op1_type == IS_CONST) { - | LOAD_ZVAL_ADDR FCARG2a, opline->op1_type, opline->op1 + if (op1_type == IS_CONST) { + | LOAD_ZVAL_ADDR FCARG2a, op1_type, op1 } else { | lea FCARG2a, [Ra(op1_reg)+op1_offset] } - |.if X64 - | LOAD_ZVAL_ADDR CARG3, opline->op2_type, opline->op2 - |.else - | PUSH_ZVAL_ADDR opline->op2_type, opline->op2, r0 - |.endif + if (op2_type == IS_CONST) { + |.if X64 + | LOAD_ZVAL_ADDR CARG3, op2_type, op2 + |.else + | PUSH_ZVAL_ADDR op2_type, op2, r0 + |.endif + } else { + |.if X64 + | lea CARG3, [Ra(op2_reg)+op2_offset] + |.else + | lea r0, [Ra(op2_reg)+op2_offset] + | push r0 + |.endif + } || if (opline->opcode == ZEND_ADD || opline->opcode == ZEND_ASSIGN_ADD) { | EXT_CALL add_function, r0 || } else if (opline->opcode == ZEND_SUB || opline->opcode == ZEND_ASSIGN_SUB) { @@ -1808,8 +1920,8 @@ static int zend_jit_math_helper(dasm_State **Dst, const zend_op *opline, zend_op || } else { || ZEND_ASSERT(0); || } - | FREE_OP opline->op1_type, opline->op1, op1_info, op_array, opline->lineno - | FREE_OP opline->op2_type, opline->op2, op2_info, op_array, opline->lineno + | FREE_OP op1_type, op1, op1_info, op_array, opline->lineno + | FREE_OP op2_type, op2, op2_info, op_array, opline->lineno || if (zend_may_throw(opline, op_array, ssa)) { || zend_jit_check_exception(Dst); || } @@ -1847,9 +1959,9 @@ static int zend_jit_math(dasm_State **Dst, const zend_op *opline, int *opnum, ze /* Eliminate the following SEND_VAL */ (*opnum)++; | mov FCARG1a, EX->call - return zend_jit_math_helper(Dst, opline, op_array, ssa, FP, opline->op1.var, FCARG1a, (opline+1)->result.var, op1_info, op2_info, RES_INFO()); + return zend_jit_math_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, FP, opline->op1.var, op1_info, opline->op2_type, opline->op2, FP, opline->op2.var, op2_info, FCARG1a, (opline+1)->result.var, RES_INFO()); } else { - return zend_jit_math_helper(Dst, opline, op_array, ssa, FP, opline->op1.var, FP, opline->result.var, op1_info, op2_info, RES_INFO()); + return zend_jit_math_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, FP, opline->op1.var, op1_info, opline->op2_type, opline->op2, FP, opline->op2.var, op2_info, FP, opline->result.var, RES_INFO()); } fallback: @@ -1857,9 +1969,413 @@ fallback: return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); } +static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, uint32_t type, uint32_t op2_info, uint32_t found, uint32_t not_found) +{ + if (op2_info & MAY_BE_LONG) { + if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_LONG)) { + | // if (EXPECTED(Z_TYPE_P(dim) == IS_LONG)) + | IF_NOT_Z_TYPE FP + opline->op2.var, IS_LONG, >3 + } + | // hval = Z_LVAL_P(dim); + | LONG_LOAD FCARG2a, opline->op2_type, opline->op2 + + if (opline->op2_type == IS_CONST) { + zend_long val = Z_LVAL_P(RT_CONSTANT(op_array, opline->op2)); + if (val >= 0 && val < HT_MAX_SIZE) { + | // ZEND_HASH_INDEX_FIND(ht, hval, retval, num_undef); + | test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED + | jz >1 // HASH_FIND + | // if (EXPECTED((zend_ulong)(_h) < (zend_ulong)(_ht)->nNumUsed)) + |.if X64 + | movsxd r0, dword [FCARG1a + offsetof(zend_array, nNumUsed)] + | cmp r0, val + |.else + | cmp dword [FCARG1a + offsetof(zend_array, nNumUsed)], val + |.endif + | jle >2 // NOT_FOUND + | // _ret = &_ht->arData[_h].val; + | mov r0, aword [FCARG1a + offsetof(zend_array, arData)] + | add r0, val * sizeof(Bucket) + | IF_NOT_Z_TYPE r0, IS_UNDEF, >8 + | jmp >2 // NOT_FOUND + } + } else { + | // ZEND_HASH_INDEX_FIND(ht, hval, retval, num_undef); + | test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED + | jz >1 // HASH_FIND + | // if (EXPECTED((zend_ulong)(_h) < (zend_ulong)(_ht)->nNumUsed)) + |.if X64 + | movsxd r0, dword [FCARG1a + offsetof(zend_array, nNumUsed)] + | cmp r0, FCARG2a + |.else + | cmp dword [FCARG1a + offsetof(zend_array, nNumUsed)], FCARG2a + |.endif + | jle >2 // NOT_FOUND + | // _ret = &_ht->arData[_h].val; + |.if X64 + | shl FCARG2a, 5 + |.else + | imul FCARG2a, sizeof(Bucket) + |.endif + | mov r0, aword [FCARG1a + offsetof(zend_array, arData)] + | add r0, FCARG2a + | IF_NOT_Z_TYPE r0, IS_UNDEF, >8 + | jmp >2 // NOT_FOUND + } + |1: + | EXT_CALL zend_hash_index_find, r0 + | test r0, r0 + | jz >2 // NOT_FOUND + |.cold_code + |2: + switch (type) { + case BP_VAR_R: + | SAVE_VALID_OPLINE opline + | // zend_error(E_NOTICE,"Undefined offset: " ZEND_LONG_FMT, hval); + |.if X64 + | mov CARG1, E_NOTICE + | LOAD_ADDR CARG2, "Undefined offset: " ZEND_LONG_FMT + | LONG_LOAD CARG3, opline->op2_type, opline->op2 + | EXT_CALL zend_error, r0 + |.else + | sub r4, 4 + | LONG_LOAD r0, opline->op2_type, opline->op2 + | push r0 + | push "Undefined offset: " ZEND_LONG_FMT + | push E_NOTICE + | EXT_CALL zend_error, r0 + | add r4, 16 + |.endif + /* break missing intentionally */ + case BP_VAR_IS: + case BP_VAR_UNSET: + | // retval = &EG(uninitialized_zval); + | SET_Z_TYPE_INFO FP + opline->result.var, IS_NULL + | jmp >9 + break; + case BP_VAR_RW: + | SAVE_VALID_OPLINE opline + | // zend_error(E_NOTICE,"Undefined offset: " ZEND_LONG_FMT, hval); + |.if X64 + | mov CARG1, E_NOTICE + | LOAD_ADDR CARG2, "Undefined offset: " ZEND_LONG_FMT + | LONG_LOAD CARG3, opline->op2_type, opline->op2 + | EXT_CALL zend_error, r0 + |.else + | sub r4, 4 + | LONG_LOAD r0, opline->op2_type, opline->op2 + | push r0 + | push "Undefined offset: " ZEND_LONG_FMT + | push E_NOTICE + | EXT_CALL zend_error, r0 + | add r4, 16 + |.endif + |//retval = zend_hash_index_update(ht, hval, &EG(uninitialized_zval)); + |//???? + break; + case BP_VAR_W: + |//retval = zend_hash_index_add_new(ht, hval, &EG(uninitialized_zval)); + |//???? + break; + default: + ZEND_ASSERT(0); + } + |.code + if (op2_info & (MAY_BE_ANY - MAY_BE_LONG)) { + | jmp >8 + } + if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_LONG)) { + |3: + } + } + + if (op2_info & MAY_BE_STRING) { + if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) { + | // if (EXPECTED(Z_TYPE_P(dim) == IS_STRING)) + | IF_NOT_Z_TYPE FP + opline->op2.var, IS_STRING, >3 + } + | // offset_key = Z_STR_P(dim); + | LONG_LOAD FCARG2a, opline->op2_type, opline->op2 + | // retval = zend_hash_find(ht, offset_key); + if (opline->op2_type != IS_CONST) { + | EXT_CALL zend_jit_symtable_find, r0 + } else { + | EXT_CALL zend_hash_find, r0 + } + | test r0, r0 + | jz >2 // NOT_FOUND + | // if (UNEXPECTED(Z_TYPE_P(retval) == IS_INDIRECT)) + | IF_Z_TYPE r0, IS_INDIRECT, >1 // SLOW + |.cold_code + |1: + | // retval = Z_INDIRECT_P(retval); + | GET_Z_PTR r0, r0 + | IF_NOT_Z_TYPE r0, IS_UNDEF, >8 + |2: + switch (type) { + case BP_VAR_R: + // zend_error(E_NOTICE, "Undefined index: %s", ZSTR_VAL(offset_key)); + | SAVE_VALID_OPLINE opline + |.if X64 + | mov CARG1, E_NOTICE + | LOAD_ADDR CARG2, "Undefined index: %s" + | LONG_LOAD CARG3, opline->op2_type, opline->op2 + | add CARG3, offsetof(zend_string, val) + | EXT_CALL zend_error, r0 + |.else + | sub r4, 4 + | LONG_LOAD r0, opline->op2_type, opline->op2 + | add r0, offsetof(zend_string, val) + | push r0 + | push "Undefined index: %s" + | push E_NOTICE + | EXT_CALL zend_error, r0 + | add r4, 16 + |.endif + /* break missing intentionally */ + case BP_VAR_IS: + case BP_VAR_UNSET: + | // retval = &EG(uninitialized_zval); + | SET_Z_TYPE_INFO FP + opline->result.var, IS_NULL + | jmp >9 + break; + case BP_VAR_RW: + // zend_error(E_NOTICE, "Undefined index: %s", ZSTR_VAL(offset_key)); + | SAVE_VALID_OPLINE opline + |.if X64 + | mov CARG1, E_NOTICE + | LOAD_ADDR CARG2, "Undefined index: %s" + | LONG_LOAD CARG3, opline->op2_type, opline->op2 + | add CARG3, offsetof(zend_string, val) + | EXT_CALL zend_error, r0 + |.else + | sub r4, 4 + | LONG_LOAD r0, opline->op2_type, opline->op2 + | add r0, offsetof(zend_string, val) + | push r0 + | push "Undefined index: %s" + | push E_NOTICE + | EXT_CALL zend_error, r0 + | add r4, 16 + |.endif + |//retval = zend_hash_update(ht, offset_key, &EG(uninitialized_zval)); + |//???? or ZVAL_NULL(retval); + break; + case BP_VAR_W: + |//retval = zend_hash_add_new(ht, offset_key, &EG(uninitialized_zval)); + |//???? or ZVAL_NULL(retval); + break; + default: + ZEND_ASSERT(0); + } + |.code + if (op2_info & (MAY_BE_ANY - (MAY_BE_LONG|MAY_BE_STRING))) { + | jmp >8 + } + } + + if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) { + if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) { + |.cold_code + |3: + } + | SAVE_VALID_OPLINE opline + | LOAD_ZVAL_ADDR FCARG2a, opline->op2_type, opline->op2 + |.if X64 + | lea CARG3, [FP + opline->result.var] + |.else + | lea r0, [FP + opline->result.var] + | push r0 + |.endif + switch (type) { + case BP_VAR_R: + | EXT_CALL zend_jit_fetch_dim_r_helper, r0 + break; + case BP_VAR_IS: + case BP_VAR_UNSET: + | EXT_CALL zend_jit_fetch_dim_is_helper, r0 + break; + case BP_VAR_RW: + |//????EXT_CALL zend_jit_fetch_dim_rw_helper, r0 + break; + case BP_VAR_W: + |//????EXT_CALL zend_jit_fetch_dim_w_helper, r0 + break; + default: + ZEND_ASSERT(0); + } + | jmp >9 + if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) { + |.code + } + } + + return 1; +} + static int zend_jit_assign_dim_math(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) { - // TODO: ??? + uint32_t op1_info, op2_info; + + if (opline->op1_type != IS_CV || opline->result_type != IS_UNUSED) { + goto fallback; + } + + if (!ssa->ops || !ssa->var_info) { + goto fallback; + } + + op1_info = OP1_INFO(); + op2_info = OP2_INFO(); + + if (op1_info & MAY_BE_REF) { + | LOAD_ZVAL_ADDR FCARG1a, opline->op1_type, opline->op1 + | ZVAL_DEREF FCARG1a, op1_info + } + + if (op1_info & MAY_BE_ARRAY) { + if (op1_info & MAY_BE_REF) { + | IF_NOT_Z_TYPE FCARG1a, IS_ARRAY, >7 + | GET_Z_PTR FCARG1a, FCARG1a + } else { + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { + | IF_NOT_Z_TYPE FP + opline->op1.var, IS_ARRAY, >7 + } + | LONG_LOAD FCARG1a, opline->op1_type, opline->op1 + } + if (!zend_jit_fetch_dimension_address_inner(Dst, opline, op_array, BP_VAR_RW, op2_info, 8, 8)) { + return 0; + } + + |8: + + if (op1_info & (MAY_BE_ARRAY_OF_REF)) { + | ZVAL_DEREF r0, MAY_BE_REF + } + if (op1_info & (MAY_BE_ARRAY_OF_STRING|MAY_BE_ARRAY_OF_ARRAY)) { + // SEPARATE_ZVAL_NOREF(var_ptr); + | IF_Z_FLAGS FCARG1a, IS_TYPE_COPYABLE + IS_TYPE_IMMUTABLE, >1 + |.cold_code + |1: + | GET_Z_PTR r0, FCARG1a + | cmp dword [r0], 1 // if (GC_REFCOUNTED() > 1) + | jbe >3 + | IF_Z_FLAGS FCARG1a, IS_TYPE_IMMUTABLE, >2 + | GC_DELREF r0 + |2: + | mov aword [r4], FCARG1a // save + | ZVAL_COPY_CTOR_FUNC (op_array->filename ? ZSTR_VAL(op_array->filename) : NULL), opline->lineno + | mov FCARG1a, aword [r4] // restore + | jmp >3 + |.code + |3: + } + if (!zend_jit_math_helper(Dst, opline, op_array, ssa, IS_CV, opline->op1, FCARG1a, 0, op1_info, (opline+1)->op2_type, (opline+1)->op2, FP, (opline+1)->op2.var, OP2_DATA_INFO(), FCARG1a, 0, OP1_DEF_INFO())) { + return 0; + } + } + + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY)) { + if (op1_info & MAY_BE_ARRAY) { + |.cold_code + |7: + } + | int3 +#if 0 + if (op1_info & MAY_BE_STRING) { + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_STRING))) { + if (op1_info & MAY_BE_REF) { + | IF_NOT_Z_TYPE FCARG1a, IS_STRING, >6 + } else { + | IF_NOT_Z_TYPE FP + opline->op1.var, IS_STRING, >6 + } + } + | SAVE_VALID_OPLINE opline + if (!(op1_info & MAY_BE_REF)) { + | LOAD_ZVAL_ADDR FCARG1a, opline->op1_type, opline->op1 + } + | LOAD_ZVAL_ADDR FCARG2a, opline->op2_type, opline->op2 + |.if X64 + | lea CARG3, [FP + opline->result.var] + |.else + | lea r0, [FP + opline->result.var] + | push r0 + |.endif + if (opline->opcode == ZEND_FETCH_DIM_R) { + | EXT_CALL zend_jit_fetch_dim_str_r_helper, r0 + } else if (opline->opcode == ZEND_FETCH_DIM_IS) { + | EXT_CALL zend_jit_fetch_dim_str_is_helper, r0 + } else { + ZEND_ASSERT(0); + } + | jmp >9 // END + |6: + } + + if (op1_info & MAY_BE_OBJECT) { + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_STRING|MAY_BE_OBJECT))) { + if (op1_info & MAY_BE_REF) { + | IF_NOT_Z_TYPE FCARG1a, IS_OBJECT, >6 + } else { + | IF_NOT_Z_TYPE FP + opline->op1.var, IS_OBJECT, >6 + } + } + | SAVE_VALID_OPLINE opline + if (!(op1_info & MAY_BE_REF)) { + | LOAD_ZVAL_ADDR FCARG1a, opline->op1_type, opline->op1 + } + | LOAD_ZVAL_ADDR FCARG2a, opline->op2_type, opline->op2 + |.if X64 + | lea CARG3, [FP + opline->result.var] + |.else + | lea r0, [FP + opline->result.var] + | push r0 + |.endif + if (opline->opcode == ZEND_FETCH_DIM_R) { + | EXT_CALL zend_jit_fetch_dim_obj_r_helper, r0 + } else if (opline->opcode == ZEND_FETCH_DIM_IS) { + | EXT_CALL zend_jit_fetch_dim_obj_is_helper, r0 + } else { + ZEND_ASSERT(0); + } + | jmp >9 // END + |6: + } + + if ((opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) || + (op2_info & MAY_BE_UNDEF)) { + | SAVE_VALID_OPLINE opline + } + if (opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) { + if (op1_info & MAY_BE_REF) { + | IF_NOT_Z_TYPE FCARG1a, IS_UNDEF, >1 + } else { + | IF_NOT_Z_TYPE FP + opline->op1.var, IS_UNDEF, >1 + } + | // zend_error(E_NOTICE, "Undefined variable: %s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); + | mov FCARG1d, opline->op1.var + | EXT_CALL zend_jit_undefined_op_helper, r0 + |1: + } + if (op2_info & MAY_BE_UNDEF) { + | IF_NOT_Z_TYPE FP + opline->op2.var, IS_UNDEF, >1 + | mov FCARG1d, opline->op2.var + | EXT_CALL zend_jit_undefined_op_helper, r0 + |1: + } + | SET_Z_TYPE_INFO FP + opline->result.var, IS_NULL + | jmp >9 // END +#endif + if (op1_info & MAY_BE_ARRAY) { + |.code + } + } + + |9: + + return 1; + +fallback: return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); } @@ -1868,6 +2384,9 @@ static int zend_jit_assign_math(dasm_State **Dst, const zend_op *opline, zend_op uint32_t op1_info, op2_info; if (opline->extended_value == ZEND_ASSIGN_DIM) { +#if 1 + goto fallback; +#endif return zend_jit_assign_dim_math(Dst, opline, op_array, ssa); } else if (opline->extended_value == ZEND_ASSIGN_OBJ) { goto fallback; @@ -1914,7 +2433,7 @@ static int zend_jit_assign_math(dasm_State **Dst, const zend_op *opline, zend_op |.code |3: } - return zend_jit_math_helper(Dst, opline, op_array, ssa, FCARG1a, 0, FCARG1a, 0, op1_info, op2_info, OP1_DEF_INFO()); + return zend_jit_math_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, FCARG1a, 0, op1_info, opline->op2_type, opline->op2, FP, opline->op2.var, op2_info, FCARG1a, 0, OP1_DEF_INFO()); } else { if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY)) { | // SEPARATE_ZVAL_NOREF(var); @@ -1933,7 +2452,7 @@ static int zend_jit_assign_math(dasm_State **Dst, const zend_op *opline, zend_op |.code |3: } - return zend_jit_math_helper(Dst, opline, op_array, ssa, FP, opline->op1.var, FP, opline->op1.var, op1_info, op2_info, OP1_DEF_INFO()); + return zend_jit_math_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, FP, opline->op1.var, op1_info, opline->op2_type, opline->op2, FP, opline->op2.var, op2_info, FP, opline->op1.var, OP1_DEF_INFO()); } fallback: @@ -3592,166 +4111,8 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, const zend_op *opline, zend } | LONG_LOAD FCARG1a, opline->op1_type, opline->op1 } - if (op2_info & MAY_BE_LONG) { - if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_LONG)) { - | // if (EXPECTED(Z_TYPE_P(dim) == IS_LONG)) - | IF_NOT_Z_TYPE FP + opline->op2.var, IS_LONG, >3 - } - | // hval = Z_LVAL_P(dim); - | LONG_LOAD FCARG2a, opline->op2_type, opline->op2 - - if (opline->op2_type == IS_CONST) { - zend_long val = Z_LVAL_P(RT_CONSTANT(op_array, opline->op2)); - if (val >= 0 && val < HT_MAX_SIZE) { - | // ZEND_HASH_INDEX_FIND(ht, hval, retval, num_undef); - | test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED - | jz >1 // HASH_FIND - | // if (EXPECTED((zend_ulong)(_h) < (zend_ulong)(_ht)->nNumUsed)) - |.if X64 - | movsxd r0, dword [FCARG1a + offsetof(zend_array, nNumUsed)] - | cmp r0, val - |.else - | cmp dword [FCARG1a + offsetof(zend_array, nNumUsed)], val - |.endif - | jle >2 // NOT_FOUND - | // _ret = &_ht->arData[_h].val; - | mov r0, aword [FCARG1a + offsetof(zend_array, arData)] - | add r0, val * sizeof(Bucket) - | IF_NOT_Z_TYPE r0, IS_UNDEF, >8 //FOUND - | jmp >2 // NOT_FOUND - } - } else { - | // ZEND_HASH_INDEX_FIND(ht, hval, retval, num_undef); - | test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED - | jz >1 // HASH_FIND - | // if (EXPECTED((zend_ulong)(_h) < (zend_ulong)(_ht)->nNumUsed)) - |.if X64 - | movsxd r0, dword [FCARG1a + offsetof(zend_array, nNumUsed)] - | cmp r0, FCARG2a - |.else - | cmp dword [FCARG1a + offsetof(zend_array, nNumUsed)], FCARG2a - |.endif - | jle >2 // NOT_FOUND - | // _ret = &_ht->arData[_h].val; - |.if X64 - | shl FCARG2a, 5 - |.else - | imul FCARG2a, sizeof(Bucket) - |.endif - | mov r0, aword [FCARG1a + offsetof(zend_array, arData)] - | add r0, FCARG2a - | IF_NOT_Z_TYPE r0, IS_UNDEF, >8 // FOUND - | jmp >2 // NOT_FOUND - } - |1: - | EXT_CALL zend_hash_index_find, r0 - | test r0, r0 - | jz >2 // NOT_FOUND - |.cold_code - |2: - if (opline->opcode == ZEND_FETCH_DIM_R) { - | SAVE_VALID_OPLINE opline - | // zend_error(E_NOTICE,"Undefined offset: " ZEND_LONG_FMT, hval); - |.if X64 - | mov CARG1, E_NOTICE - | LOAD_ADDR CARG2, "Undefined offset: " ZEND_LONG_FMT - | LONG_LOAD CARG3, opline->op2_type, opline->op2 - | EXT_CALL zend_error, r0 - |.else - | sub r4, 4 - | LONG_LOAD r0, opline->op2_type, opline->op2 - | push r0 - | push "Undefined offset: " ZEND_LONG_FMT - | push E_NOTICE - | EXT_CALL zend_error, r0 - | add r4, 16 - |.endif - } - | // retval = &EG(uninitialized_zval); - | SET_Z_TYPE_INFO FP + opline->result.var, IS_NULL - | jmp >9 // END - |.code - if (op2_info & (MAY_BE_ANY - MAY_BE_LONG)) { - | jmp >8 // FOUND - } - if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_LONG)) { - |3: - } - } - if (op2_info & MAY_BE_STRING) { - if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) { - | // if (EXPECTED(Z_TYPE_P(dim) == IS_STRING)) - | IF_NOT_Z_TYPE FP + opline->op2.var, IS_STRING, >3 - } - | // offset_key = Z_STR_P(dim); - | LONG_LOAD FCARG2a, opline->op2_type, opline->op2 - | // retval = zend_hash_find(ht, offset_key); - if (opline->op2_type != IS_CONST) { - | EXT_CALL zend_jit_symtable_find, r0 - } else { - | EXT_CALL zend_hash_find, r0 - } - | test r0, r0 - | jz >2 // NOT_FOUND - | // if (UNEXPECTED(Z_TYPE_P(retval) == IS_INDIRECT)) - | IF_Z_TYPE r0, IS_INDIRECT, >1 // SLOW - |.cold_code - |1: - | // retval = Z_INDIRECT_P(retval); - | GET_Z_PTR r0, r0 - | IF_NOT_Z_TYPE r0, IS_UNDEF, >8 // COPY - |2: - if (opline->opcode == ZEND_FETCH_DIM_R) { - // zend_error(E_NOTICE, "Undefined index: %s", ZSTR_VAL(offset_key)); - | SAVE_VALID_OPLINE opline - |.if X64 - | mov CARG1, E_NOTICE - | LOAD_ADDR CARG2, "Undefined index: %s" - | LONG_LOAD CARG3, opline->op2_type, opline->op2 - | add CARG3, offsetof(zend_string, val) - | EXT_CALL zend_error, r0 - |.else - | sub r4, 4 - | LONG_LOAD r0, opline->op2_type, opline->op2 - | add r0, offsetof(zend_string, val) - | push r0 - | push "Undefined index: %s" - | push E_NOTICE - | EXT_CALL zend_error, r0 - | add r4, 16 - |.endif - } - | SET_Z_TYPE_INFO FP + opline->result.var, IS_NULL - | jmp >9 // END - |.code - if (op2_info & (MAY_BE_ANY - (MAY_BE_LONG|MAY_BE_STRING))) { - | jmp >8 // FOUND - } - } - if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) { - if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) { - |.cold_code - |3: - } - | SAVE_VALID_OPLINE opline - | LOAD_ZVAL_ADDR FCARG2a, opline->op2_type, opline->op2 - |.if X64 - | lea CARG3, [FP + opline->result.var] - |.else - | lea r0, [FP + opline->result.var] - | push r0 - |.endif - if (opline->opcode == ZEND_FETCH_DIM_R) { - | EXT_CALL zend_jit_fetch_dim_r_helper, r0 - } else if (opline->opcode == ZEND_FETCH_DIM_IS) { - | EXT_CALL zend_jit_fetch_dim_is_helper, r0 - } else { - ZEND_ASSERT(0); - } - | jmp >9 // END - if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) { - |.code - } + if (!zend_jit_fetch_dimension_address_inner(Dst, opline, op_array, (opline->opcode == ZEND_FETCH_DIM_R) ? BP_VAR_R : BP_VAR_IS, op2_info, 8, 9)) { + return 0; } } From d8d6450131f2780646220b5e90c894d67ffb991e Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 27 Sep 2016 19:28:01 +0300 Subject: [PATCH 247/569] Fixed overflow detection conditions --- ext/opcache/jit/zend_jit.c | 64 ++++++++++++++------------------------ 1 file changed, 24 insertions(+), 40 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index a386ad51bbad7..815c0e2a78b2f 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -653,9 +653,7 @@ static int zend_may_overflow(const zend_op *opline, zend_op_array *op_array, zen op1_min = OP1_MIN_RANGE(); op2_min = OP2_MIN_RANGE(); res_min = op1_min + op2_min; - return OP1_RANGE_UNDERFLOW() || - OP2_RANGE_UNDERFLOW() || - (op1_min < 0 && op2_min < 0 && res_min >= 0); + return (op1_min < 0 && op2_min < 0 && res_min >= 0); } return 1; } @@ -666,9 +664,7 @@ static int zend_may_overflow(const zend_op *opline, zend_op_array *op_array, zen op1_max = OP1_MAX_RANGE(); op2_max = OP2_MAX_RANGE(); res_max = op1_max + op2_max; - return OP1_RANGE_OVERFLOW() || - OP2_RANGE_OVERFLOW() || - (op1_max > 0 && op2_max > 0 && res_max <= 0); + return (op1_max > 0 && op2_max > 0 && res_max <= 0); } return 1; } @@ -682,27 +678,23 @@ static int zend_may_overflow(const zend_op *opline, zend_op_array *op_array, zen } if (ssa->var_info[res].range.underflow) { if (OP1_HAS_RANGE() && OP2_HAS_RANGE()) { - zend_long op1_max, op2_min, res_max; + zend_long op1_min, op2_max, res_min; - op1_max = OP1_MAX_RANGE(); - op2_min = OP2_MIN_RANGE(); - res_max = op1_max - op2_min; - return OP1_RANGE_OVERFLOW() || - OP2_RANGE_UNDERFLOW() || - (op1_max > 0 && op2_min < 0 && res_max <= 0); + op1_min = OP1_MIN_RANGE(); + op2_max = OP2_MAX_RANGE(); + res_min = op1_min - op2_max; + return (op1_min < 0 && op2_max > 0 && res_min >= 0); } return 1; } if (ssa->var_info[res].range.overflow) { if (OP1_HAS_RANGE() && OP2_HAS_RANGE()) { - zend_long op1_min, op2_max, res_min; + zend_long op1_max, op2_min, res_max; - op1_min = OP1_MIN_RANGE(); - op2_max = OP2_MAX_RANGE(); - res_min = op1_min - op2_max; - return OP1_RANGE_UNDERFLOW() || - OP2_RANGE_OVERFLOW() || - (op1_min < 0 && op2_max > 0 && res_min >= 0); + op1_max = OP1_MAX_RANGE(); + op2_min = OP2_MIN_RANGE(); + res_max = op1_max - op2_min; + return (op1_max > 0 && op2_min < 0 && res_max <= 0); } return 1; } @@ -731,9 +723,7 @@ static int zend_may_overflow(const zend_op *opline, zend_op_array *op_array, zen op1_min = OP1_MIN_RANGE(); op2_min = OP2_MIN_RANGE(); res_min = op1_min + op2_min; - return OP1_RANGE_UNDERFLOW() || - OP2_RANGE_UNDERFLOW() || - (op1_min < 0 && op2_min < 0 && res_min >= 0); + return (op1_min < 0 && op2_min < 0 && res_min >= 0); } return 1; } @@ -744,9 +734,7 @@ static int zend_may_overflow(const zend_op *opline, zend_op_array *op_array, zen op1_max = OP1_MAX_RANGE(); op2_max = OP2_MAX_RANGE(); res_max = op1_max + op2_max; - return OP1_RANGE_OVERFLOW() || - OP2_RANGE_OVERFLOW() || - (op1_max > 0 && op2_max > 0 && res_max <= 0); + return (op1_max > 0 && op2_max > 0 && res_max <= 0); } return 1; } @@ -763,27 +751,23 @@ static int zend_may_overflow(const zend_op *opline, zend_op_array *op_array, zen } if (ssa->var_info[res].range.underflow) { if (OP1_HAS_RANGE() && OP2_HAS_RANGE()) { - zend_long op1_max, op2_min, res_max; + zend_long op1_min, op2_max, res_min; - op1_max = OP1_MAX_RANGE(); - op2_min = OP2_MIN_RANGE(); - res_max = op1_max - op2_min; - return OP1_RANGE_OVERFLOW() || - OP2_RANGE_UNDERFLOW() || - (op1_max > 0 && op2_min < 0 && res_max <= 0); + op1_min = OP1_MIN_RANGE(); + op2_max = OP2_MAX_RANGE(); + res_min = op1_min - op2_max; + return (op1_min < 0 && op2_max > 0 && res_min >= 0); } return 1; } if (ssa->var_info[res].range.overflow) { if (OP1_HAS_RANGE() && OP2_HAS_RANGE()) { - zend_long op1_min, op2_max, res_min; + zend_long op1_max, op2_min, res_max; - op1_min = OP1_MIN_RANGE(); - op2_max = OP2_MAX_RANGE(); - res_min = op1_min - op2_max; - return OP1_RANGE_UNDERFLOW() || - OP2_RANGE_OVERFLOW() || - (op1_min < 0 && op2_max > 0 && res_min >= 0); + op1_max = OP1_MAX_RANGE(); + op2_min = OP2_MIN_RANGE(); + res_max = op1_max - op2_min; + return (op1_max > 0 && op2_min < 0 && res_max <= 0); } return 1; } From 5375bd31f51303fabcfb92a1d39ab686ff1ef313 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 27 Sep 2016 22:12:41 +0300 Subject: [PATCH 248/569] More overflow detection conditions fixes. --- ext/opcache/jit/zend_jit.c | 128 +++++++++++++++++++++---------------- 1 file changed, 72 insertions(+), 56 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 815c0e2a78b2f..5a233cfe9886f 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -647,26 +647,30 @@ static int zend_may_overflow(const zend_op *opline, zend_op_array *op_array, zen return 1; } if (ssa->var_info[res].range.underflow) { - if (OP1_HAS_RANGE() && OP2_HAS_RANGE()) { - zend_long op1_min, op2_min, res_min; + zend_long op1_min, op2_min, res_min; - op1_min = OP1_MIN_RANGE(); - op2_min = OP2_MIN_RANGE(); - res_min = op1_min + op2_min; - return (op1_min < 0 && op2_min < 0 && res_min >= 0); + if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) { + return 1; + } + op1_min = OP1_MIN_RANGE(); + op2_min = OP2_MIN_RANGE(); + res_min = op1_min + op2_min; + if (op1_min < 0 && op2_min < 0 && res_min >= 0) { + return 1; } - return 1; } if (ssa->var_info[res].range.overflow) { - if (OP1_HAS_RANGE() && OP2_HAS_RANGE()) { - zend_long op1_max, op2_max, res_max; + zend_long op1_max, op2_max, res_max; - op1_max = OP1_MAX_RANGE(); - op2_max = OP2_MAX_RANGE(); - res_max = op1_max + op2_max; - return (op1_max > 0 && op2_max > 0 && res_max <= 0); + if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) { + return 1; + } + op1_max = OP1_MAX_RANGE(); + op2_max = OP2_MAX_RANGE(); + res_max = op1_max + op2_max; + if (op1_max > 0 && op2_max > 0 && res_max <= 0) { + return 1; } - return 1; } return 0; case ZEND_SUB: @@ -677,26 +681,30 @@ static int zend_may_overflow(const zend_op *opline, zend_op_array *op_array, zen return 1; } if (ssa->var_info[res].range.underflow) { - if (OP1_HAS_RANGE() && OP2_HAS_RANGE()) { - zend_long op1_min, op2_max, res_min; + zend_long op1_min, op2_max, res_min; - op1_min = OP1_MIN_RANGE(); - op2_max = OP2_MAX_RANGE(); - res_min = op1_min - op2_max; - return (op1_min < 0 && op2_max > 0 && res_min >= 0); + if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) { + return 1; + } + op1_min = OP1_MIN_RANGE(); + op2_max = OP2_MAX_RANGE(); + res_min = op1_min - op2_max; + if (op1_min < 0 && op2_max > 0 && res_min >= 0) { + return 1; } - return 1; } if (ssa->var_info[res].range.overflow) { - if (OP1_HAS_RANGE() && OP2_HAS_RANGE()) { - zend_long op1_max, op2_min, res_max; + zend_long op1_max, op2_min, res_max; - op1_max = OP1_MAX_RANGE(); - op2_min = OP2_MIN_RANGE(); - res_max = op1_max - op2_min; - return (op1_max > 0 && op2_min < 0 && res_max <= 0); + if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) { + return 1; + } + op1_max = OP1_MAX_RANGE(); + op2_min = OP2_MIN_RANGE(); + res_max = op1_max - op2_min; + if (op1_max > 0 && op2_min < 0 && res_max <= 0) { + return 1; } - return 1; } return 0; case ZEND_MUL: @@ -717,26 +725,30 @@ static int zend_may_overflow(const zend_op *opline, zend_op_array *op_array, zen return 1; } if (ssa->var_info[res].range.underflow) { - if (OP1_HAS_RANGE() && OP2_HAS_RANGE()) { - zend_long op1_min, op2_min, res_min; + zend_long op1_min, op2_min, res_min; - op1_min = OP1_MIN_RANGE(); - op2_min = OP2_MIN_RANGE(); - res_min = op1_min + op2_min; - return (op1_min < 0 && op2_min < 0 && res_min >= 0); + if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) { + return 1; + } + op1_min = OP1_MIN_RANGE(); + op2_min = OP2_MIN_RANGE(); + res_min = op1_min + op2_min; + if (op1_min < 0 && op2_min < 0 && res_min >= 0) { + return 1; } - return 1; } if (ssa->var_info[res].range.overflow) { - if (OP1_HAS_RANGE() && OP2_HAS_RANGE()) { - zend_long op1_max, op2_max, res_max; + zend_long op1_max, op2_max, res_max; - op1_max = OP1_MAX_RANGE(); - op2_max = OP2_MAX_RANGE(); - res_max = op1_max + op2_max; - return (op1_max > 0 && op2_max > 0 && res_max <= 0); + if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) { + return 1; + } + op1_max = OP1_MAX_RANGE(); + op2_max = OP2_MAX_RANGE(); + res_max = op1_max + op2_max; + if (op1_max > 0 && op2_max > 0 && res_max <= 0) { + return 1; } - return 1; } return 0; case ZEND_ASSIGN_SUB: @@ -750,26 +762,30 @@ static int zend_may_overflow(const zend_op *opline, zend_op_array *op_array, zen return 1; } if (ssa->var_info[res].range.underflow) { - if (OP1_HAS_RANGE() && OP2_HAS_RANGE()) { - zend_long op1_min, op2_max, res_min; + zend_long op1_min, op2_max, res_min; - op1_min = OP1_MIN_RANGE(); - op2_max = OP2_MAX_RANGE(); - res_min = op1_min - op2_max; - return (op1_min < 0 && op2_max > 0 && res_min >= 0); + if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) { + return 1; + } + op1_min = OP1_MIN_RANGE(); + op2_max = OP2_MAX_RANGE(); + res_min = op1_min - op2_max; + if (op1_min < 0 && op2_max > 0 && res_min >= 0) { + return 1; } - return 1; } if (ssa->var_info[res].range.overflow) { - if (OP1_HAS_RANGE() && OP2_HAS_RANGE()) { - zend_long op1_max, op2_min, res_max; + zend_long op1_max, op2_min, res_max; - op1_max = OP1_MAX_RANGE(); - op2_min = OP2_MIN_RANGE(); - res_max = op1_max - op2_min; - return (op1_max > 0 && op2_min < 0 && res_max <= 0); + if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) { + return 1; + } + op1_max = OP1_MAX_RANGE(); + op2_min = OP2_MIN_RANGE(); + res_max = op1_max - op2_min; + if (op1_max > 0 && op2_min < 0 && res_max <= 0) { + return 1; } - return 1; } return 0; case ZEND_ASSIGN_MUL: From d97490396e281db7613aa0c497995130313955af Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 28 Sep 2016 13:25:10 +0300 Subject: [PATCH 249/569] Implemented JIT for ASSIGN_ADD/SUB/MUL with ASSIGN_DIM ($a[...] += ...) --- ext/opcache/jit/zend_jit_helpers.c | 603 +++++++++++++++++++++++++++++ ext/opcache/jit/zend_jit_x86.dasc | 577 +++++++++++++++++---------- 2 files changed, 972 insertions(+), 208 deletions(-) diff --git a/ext/opcache/jit/zend_jit_helpers.c b/ext/opcache/jit/zend_jit_helpers.c index fd04e079d2f6a..dff3ee9f197aa 100644 --- a/ext/opcache/jit/zend_jit_helpers.c +++ b/ext/opcache/jit/zend_jit_helpers.c @@ -68,6 +68,103 @@ static zval* ZEND_FASTCALL zend_jit_symtable_find(HashTable *ht, zend_string *st return zend_hash_find(ht, str); } +static zval* ZEND_FASTCALL zend_jit_hash_lookup_rw(HashTable *ht, zend_string *str) +{ + zval *retval = zend_hash_find(ht, str); + + if (retval) { + if (UNEXPECTED(Z_TYPE_P(retval) == IS_INDIRECT)) { + retval = Z_INDIRECT_P(retval); + if (UNEXPECTED(Z_TYPE_P(retval) == IS_UNDEF)) { + zend_error(E_NOTICE,"Undefined index: %s", ZSTR_VAL(str)); + ZVAL_NULL(retval); + } + } + } else { + zend_error(E_NOTICE,"Undefined index: %s", ZSTR_VAL(str)); + retval = zend_hash_update(ht, str, &EG(uninitialized_zval)); + } + return retval; +} + +static zval* ZEND_FASTCALL zend_jit_hash_lookup_w(HashTable *ht, zend_string *str) +{ + zval *retval = zend_hash_find(ht, str); + + if (retval) { + if (UNEXPECTED(Z_TYPE_P(retval) == IS_INDIRECT)) { + retval = Z_INDIRECT_P(retval); + if (UNEXPECTED(Z_TYPE_P(retval) == IS_UNDEF)) { + ZVAL_NULL(retval); + } + } + } else { + retval = zend_hash_add_new(ht, str, &EG(uninitialized_zval)); + } + return retval; +} + +static zval* ZEND_FASTCALL zend_jit_symtable_lookup_rw(HashTable *ht, zend_string *str) +{ + zend_ulong idx; + register const char *tmp = str->val; + + do { + if (*tmp > '9') { + break; + } else if (*tmp < '0') { + if (*tmp != '-') { + break; + } + tmp++; + if (*tmp > '9' || *tmp < '0') { + break; + } + } + if (_zend_handle_numeric_str_ex(str->val, str->len, &idx)) { + zval *retval = zend_hash_index_find(ht, idx); + + if (!retval) { + zend_error(E_NOTICE,"Undefined index: %s", ZSTR_VAL(str)); + retval = zend_hash_index_update(ht, idx, &EG(uninitialized_zval)); + } + return retval; + } + } while (0); + + return zend_jit_hash_lookup_rw(ht, str); +} + +static zval* ZEND_FASTCALL zend_jit_symtable_lookup_w(HashTable *ht, zend_string *str) +{ + zend_ulong idx; + register const char *tmp = str->val; + + do { + if (*tmp > '9') { + break; + } else if (*tmp < '0') { + if (*tmp != '-') { + break; + } + tmp++; + if (*tmp > '9' || *tmp < '0') { + break; + } + } + if (_zend_handle_numeric_str_ex(str->val, str->len, &idx)) { + zval *retval = zend_hash_index_find(ht, idx); + + if (!retval) { + retval = zend_hash_index_add_new(ht, idx, &EG(uninitialized_zval)); + } + return retval; + } + } while (0); + + return zend_jit_hash_lookup_w(ht, str); +} + static void ZEND_FASTCALL zend_jit_undefined_op_helper(uint32_t var) { const zend_execute_data *execute_data = EG(current_execute_data); @@ -205,6 +302,127 @@ static void ZEND_FASTCALL zend_jit_fetch_dim_is_helper(zend_array *ht, zval *dim ZVAL_NULL(result); } +static zval* ZEND_FASTCALL zend_jit_fetch_dim_rw_helper(zend_array *ht, zval *dim) +{ + zend_long hval; + zend_string *offset_key; + zval *retval; + + if (Z_TYPE_P(dim) == IS_REFERENCE) { + dim = Z_REFVAL_P(dim); + } + + switch (Z_TYPE_P(dim)) { + case IS_UNDEF: + zend_jit_undefined_op_helper(EG(current_execute_data)->opline->op2.var); + /* break missing intentionally */ + case IS_NULL: + offset_key = ZSTR_EMPTY_ALLOC(); + goto str_index; + case IS_DOUBLE: + hval = zend_dval_to_lval(Z_DVAL_P(dim)); + goto num_index; + case IS_RESOURCE: + zend_error(E_NOTICE, "Resource ID#%d used as offset, casting to integer (%d)", Z_RES_HANDLE_P(dim), Z_RES_HANDLE_P(dim)); + hval = Z_RES_HANDLE_P(dim); + goto num_index; + case IS_FALSE: + hval = 0; + goto num_index; + case IS_TRUE: + hval = 1; + goto num_index; + default: + zend_error(E_WARNING, "Illegal offset type"); + return NULL; + } + +str_index: + retval = zend_hash_find(ht, offset_key); + if (retval) { + /* support for $GLOBALS[...] */ + if (UNEXPECTED(Z_TYPE_P(retval) == IS_INDIRECT)) { + retval = Z_INDIRECT_P(retval); + if (UNEXPECTED(Z_TYPE_P(retval) == IS_UNDEF)) { + zend_error(E_NOTICE, "Undefined index: %s", ZSTR_VAL(offset_key)); + ZVAL_NULL(retval); + } + } + } else { + zend_error(E_NOTICE, "Undefined index: %s", ZSTR_VAL(offset_key)); + retval = zend_hash_update(ht, offset_key, &EG(uninitialized_zval)); + } + return retval; + +num_index: + ZEND_HASH_INDEX_FIND(ht, hval, retval, num_undef); + return retval; + +num_undef: + zend_error(E_NOTICE,"Undefined offset: " ZEND_LONG_FMT, hval); + retval = zend_hash_index_update(ht, hval, &EG(uninitialized_zval)); + return retval; +} + +static zval* ZEND_FASTCALL zend_jit_fetch_dim_w_helper(zend_array *ht, zval *dim) +{ + zend_long hval; + zend_string *offset_key; + zval *retval; + + if (Z_TYPE_P(dim) == IS_REFERENCE) { + dim = Z_REFVAL_P(dim); + } + + switch (Z_TYPE_P(dim)) { + case IS_UNDEF: + zend_jit_undefined_op_helper(EG(current_execute_data)->opline->op2.var); + /* break missing intentionally */ + case IS_NULL: + offset_key = ZSTR_EMPTY_ALLOC(); + goto str_index; + case IS_DOUBLE: + hval = zend_dval_to_lval(Z_DVAL_P(dim)); + goto num_index; + case IS_RESOURCE: + zend_error(E_NOTICE, "Resource ID#%d used as offset, casting to integer (%d)", Z_RES_HANDLE_P(dim), Z_RES_HANDLE_P(dim)); + hval = Z_RES_HANDLE_P(dim); + goto num_index; + case IS_FALSE: + hval = 0; + goto num_index; + case IS_TRUE: + hval = 1; + goto num_index; + default: + zend_error(E_WARNING, "Illegal offset type"); + return NULL; + } + +str_index: + retval = zend_hash_find(ht, offset_key); + if (retval) { + /* support for $GLOBALS[...] */ + if (UNEXPECTED(Z_TYPE_P(retval) == IS_INDIRECT)) { + retval = Z_INDIRECT_P(retval); + if (UNEXPECTED(Z_TYPE_P(retval) == IS_UNDEF)) { + ZVAL_NULL(retval); + } + } + } else { + retval = zend_hash_add_new(ht, offset_key, &EG(uninitialized_zval)); + } + return retval; + +num_index: + ZEND_HASH_INDEX_FIND(ht, hval, retval, num_undef); + return retval; + +num_undef: + retval = zend_hash_index_add_new(ht, hval, &EG(uninitialized_zval)); + return retval; +} + static void ZEND_FASTCALL zend_jit_fetch_dim_str_r_helper(zval *container, zval *dim, zval *result) { zend_long offset; @@ -353,6 +571,391 @@ static void ZEND_FASTCALL zend_jit_fetch_dim_obj_is_helper(zval *container, zval } } +static zval* ZEND_FASTCALL zend_jit_fetch_dimension_rw_long_helper(HashTable *ht, zend_long hval) +{ + zend_error(E_NOTICE,"Undefined offset: " ZEND_LONG_FMT, hval); + return zend_hash_index_update(ht, hval, &EG(uninitialized_zval)); +} + +static zend_never_inline zend_long zend_check_string_offset(zval *dim, int type) +{ + zend_long offset; + +try_again: + if (UNEXPECTED(Z_TYPE_P(dim) != IS_LONG)) { + switch(Z_TYPE_P(dim)) { + case IS_STRING: + if (IS_LONG == is_numeric_string(Z_STRVAL_P(dim), Z_STRLEN_P(dim), NULL, NULL, -1)) { + break; + } + if (type != BP_VAR_UNSET) { + zend_error(E_WARNING, "Illegal string offset '%s'", Z_STRVAL_P(dim)); + } + break; + case IS_UNDEF: + zend_jit_undefined_op_helper(EG(current_execute_data)->opline->op2.var); + case IS_DOUBLE: + case IS_NULL: + case IS_FALSE: + case IS_TRUE: + zend_error(E_NOTICE, "String offset cast occurred"); + break; + case IS_REFERENCE: + dim = Z_REFVAL_P(dim); + goto try_again; + default: + zend_error(E_WARNING, "Illegal offset type"); + break; + } + + offset = _zval_get_long_func(dim); + } else { + offset = Z_LVAL_P(dim); + } + + return offset; +} + +static zend_never_inline ZEND_COLD void zend_wrong_string_offset(void) +{ + const char *msg = NULL; + const zend_op *opline = EG(current_execute_data)->opline; + const zend_op *end; + uint32_t var; + + switch (opline->opcode) { + case ZEND_ASSIGN_ADD: + case ZEND_ASSIGN_SUB: + case ZEND_ASSIGN_MUL: + case ZEND_ASSIGN_DIV: + case ZEND_ASSIGN_MOD: + case ZEND_ASSIGN_SL: + case ZEND_ASSIGN_SR: + case ZEND_ASSIGN_CONCAT: + case ZEND_ASSIGN_BW_OR: + case ZEND_ASSIGN_BW_AND: + case ZEND_ASSIGN_BW_XOR: + case ZEND_ASSIGN_POW: + msg = "Cannot use assign-op operators with string offsets"; + break; + case ZEND_FETCH_DIM_W: + case ZEND_FETCH_DIM_RW: + case ZEND_FETCH_DIM_FUNC_ARG: + case ZEND_FETCH_DIM_UNSET: + /* TODO: Encode the "reason" into opline->extended_value??? */ + var = opline->result.var; + opline++; + end = EG(current_execute_data)->func->op_array.opcodes + + EG(current_execute_data)->func->op_array.last; + while (opline < end) { + if (opline->op1_type == IS_VAR && opline->op1.var == var) { + switch (opline->opcode) { + case ZEND_ASSIGN_ADD: + case ZEND_ASSIGN_SUB: + case ZEND_ASSIGN_MUL: + case ZEND_ASSIGN_DIV: + case ZEND_ASSIGN_MOD: + case ZEND_ASSIGN_SL: + case ZEND_ASSIGN_SR: + case ZEND_ASSIGN_CONCAT: + case ZEND_ASSIGN_BW_OR: + case ZEND_ASSIGN_BW_AND: + case ZEND_ASSIGN_BW_XOR: + case ZEND_ASSIGN_POW: + if (opline->extended_value == ZEND_ASSIGN_OBJ) { + msg = "Cannot use string offset as an object"; + } else if (opline->extended_value == ZEND_ASSIGN_DIM) { + msg = "Cannot use string offset as an array"; + } else { + msg = "Cannot use assign-op operators with string offsets"; + } + break; + case ZEND_PRE_INC_OBJ: + case ZEND_PRE_DEC_OBJ: + case ZEND_POST_INC_OBJ: + case ZEND_POST_DEC_OBJ: + case ZEND_PRE_INC: + case ZEND_PRE_DEC: + case ZEND_POST_INC: + case ZEND_POST_DEC: + msg = "Cannot increment/decrement string offsets"; + break; + case ZEND_FETCH_DIM_W: + case ZEND_FETCH_DIM_RW: + case ZEND_FETCH_DIM_FUNC_ARG: + case ZEND_FETCH_DIM_UNSET: + case ZEND_ASSIGN_DIM: + msg = "Cannot use string offset as an array"; + break; + case ZEND_FETCH_OBJ_W: + case ZEND_FETCH_OBJ_RW: + case ZEND_FETCH_OBJ_FUNC_ARG: + case ZEND_FETCH_OBJ_UNSET: + case ZEND_ASSIGN_OBJ: + msg = "Cannot use string offset as an object"; + break; + case ZEND_ASSIGN_REF: + case ZEND_ADD_ARRAY_ELEMENT: + case ZEND_INIT_ARRAY: + case ZEND_MAKE_REF: + msg = "Cannot create references to/from string offsets"; + break; + case ZEND_RETURN_BY_REF: + case ZEND_VERIFY_RETURN_TYPE: + msg = "Cannot return string offsets by reference"; + break; + case ZEND_UNSET_DIM: + case ZEND_UNSET_OBJ: + msg = "Cannot unset string offsets"; + break; + case ZEND_YIELD: + msg = "Cannot yield string offsets by reference"; + break; + case ZEND_SEND_REF: + case ZEND_SEND_VAR_EX: + msg = "Only variables can be passed by reference"; + break; + EMPTY_SWITCH_DEFAULT_CASE(); + } + break; + } + if (opline->op2_type == IS_VAR && opline->op2.var == var) { + ZEND_ASSERT(opline->opcode == ZEND_ASSIGN_REF); + msg = "Cannot create references to/from string offsets"; + break; + } + } + break; + EMPTY_SWITCH_DEFAULT_CASE(); + } + ZEND_ASSERT(msg != NULL); + zend_throw_error(NULL, msg); +} + +static void ZEND_FASTCALL zend_jit_assign_dim_add_helper(zval *container, zval *dim, zval *value) +{ + if (EXPECTED(Z_TYPE_P(container) == IS_OBJECT)) { + zval *object = container; + zval *property = dim; + zval *z; + zval rv, res; + + if (Z_OBJ_HT_P(object)->read_dimension && + (z = Z_OBJ_HT_P(object)->read_dimension(object, property, BP_VAR_R, &rv)) != NULL) { + + if (Z_TYPE_P(z) == IS_OBJECT && Z_OBJ_HT_P(z)->get) { + zval rv2; + zval *value = Z_OBJ_HT_P(z)->get(z, &rv2); + + if (z == &rv) { + zval_ptr_dtor(&rv); + } + ZVAL_COPY_VALUE(z, value); + } + add_function(&res, Z_ISREF_P(z) ? Z_REFVAL_P(z) : z, value); + Z_OBJ_HT_P(object)->write_dimension(object, property, &res); + if (z == &rv) { + zval_ptr_dtor(&rv); + } +//??? if (retval) { +//??? ZVAL_COPY(retval, &res); +//??? } + zval_ptr_dtor(&res); + } else { + zend_error(E_WARNING, "Attempt to assign property of non-object"); +//??? if (retval) { +//??? ZVAL_NULL(retval); +//??? } + } + } else { + if (UNEXPECTED(Z_TYPE_P(container) == IS_STRING)) { + if (!dim) { + zend_throw_error(NULL, "[] operator not supported for strings"); + } else { + zend_check_string_offset(dim, BP_VAR_RW); + zend_wrong_string_offset(); + } +//??? } else if (EXPECTED(Z_TYPE_P(container) <= IS_FALSE)) { +//??? ZEND_VM_C_GOTO(assign_dim_op_convert_to_array); + } else { +//??? if (UNEXPECTED(OP1_TYPE != IS_VAR || EXPECTED(!Z_ISERROR_P(container)))) { + zend_error(E_WARNING, "Cannot use a scalar value as an array"); +//??? } +//??? if (retval) { +//??? ZVAL_NULL(retval); +//??? } + } + } +} + +static void ZEND_FASTCALL zend_jit_assign_dim_sub_helper(zval *container, zval *dim, zval *value) +{ + if (EXPECTED(Z_TYPE_P(container) == IS_OBJECT)) { + zval *object = container; + zval *property = dim; + zval *z; + zval rv, res; + + if (Z_OBJ_HT_P(object)->read_dimension && + (z = Z_OBJ_HT_P(object)->read_dimension(object, property, BP_VAR_R, &rv)) != NULL) { + + if (Z_TYPE_P(z) == IS_OBJECT && Z_OBJ_HT_P(z)->get) { + zval rv2; + zval *value = Z_OBJ_HT_P(z)->get(z, &rv2); + + if (z == &rv) { + zval_ptr_dtor(&rv); + } + ZVAL_COPY_VALUE(z, value); + } + sub_function(&res, Z_ISREF_P(z) ? Z_REFVAL_P(z) : z, value); + Z_OBJ_HT_P(object)->write_dimension(object, property, &res); + if (z == &rv) { + zval_ptr_dtor(&rv); + } +//??? if (retval) { +//??? ZVAL_COPY(retval, &res); +//??? } + zval_ptr_dtor(&res); + } else { + zend_error(E_WARNING, "Attempt to assign property of non-object"); +//??? if (retval) { +//??? ZVAL_NULL(retval); +//??? } + } + } else { + if (UNEXPECTED(Z_TYPE_P(container) == IS_STRING)) { + if (!dim) { + zend_throw_error(NULL, "[] operator not supported for strings"); + } else { + zend_check_string_offset(dim, BP_VAR_RW); + zend_wrong_string_offset(); + } +//??? } else if (EXPECTED(Z_TYPE_P(container) <= IS_FALSE)) { +//??? ZEND_VM_C_GOTO(assign_dim_op_convert_to_array); + } else { +//??? if (UNEXPECTED(OP1_TYPE != IS_VAR || EXPECTED(!Z_ISERROR_P(container)))) { + zend_error(E_WARNING, "Cannot use a scalar value as an array"); +//??? } +//??? if (retval) { +//??? ZVAL_NULL(retval); +//??? } + } + } +} + +static void ZEND_FASTCALL zend_jit_assign_dim_mul_helper(zval *container, zval *dim, zval *value) +{ + if (EXPECTED(Z_TYPE_P(container) == IS_OBJECT)) { + zval *object = container; + zval *property = dim; + zval *z; + zval rv, res; + + if (Z_OBJ_HT_P(object)->read_dimension && + (z = Z_OBJ_HT_P(object)->read_dimension(object, property, BP_VAR_R, &rv)) != NULL) { + + if (Z_TYPE_P(z) == IS_OBJECT && Z_OBJ_HT_P(z)->get) { + zval rv2; + zval *value = Z_OBJ_HT_P(z)->get(z, &rv2); + + if (z == &rv) { + zval_ptr_dtor(&rv); + } + ZVAL_COPY_VALUE(z, value); + } + mul_function(&res, Z_ISREF_P(z) ? Z_REFVAL_P(z) : z, value); + Z_OBJ_HT_P(object)->write_dimension(object, property, &res); + if (z == &rv) { + zval_ptr_dtor(&rv); + } +//??? if (retval) { +//??? ZVAL_COPY(retval, &res); +//??? } + zval_ptr_dtor(&res); + } else { + zend_error(E_WARNING, "Attempt to assign property of non-object"); +//??? if (retval) { +//??? ZVAL_NULL(retval); +//??? } + } + } else { + if (UNEXPECTED(Z_TYPE_P(container) == IS_STRING)) { + if (!dim) { + zend_throw_error(NULL, "[] operator not supported for strings"); + } else { + zend_check_string_offset(dim, BP_VAR_RW); + zend_wrong_string_offset(); + } +//??? } else if (EXPECTED(Z_TYPE_P(container) <= IS_FALSE)) { +//??? ZEND_VM_C_GOTO(assign_dim_op_convert_to_array); + } else { +//??? if (UNEXPECTED(OP1_TYPE != IS_VAR || EXPECTED(!Z_ISERROR_P(container)))) { + zend_error(E_WARNING, "Cannot use a scalar value as an array"); +//??? } +//??? if (retval) { +//??? ZVAL_NULL(retval); +//??? } + } + } +} + +static void ZEND_FASTCALL zend_jit_assign_dim_div_helper(zval *container, zval *dim, zval *value) +{ + if (EXPECTED(Z_TYPE_P(container) == IS_OBJECT)) { + zval *object = container; + zval *property = dim; + zval *z; + zval rv, res; + + if (Z_OBJ_HT_P(object)->read_dimension && + (z = Z_OBJ_HT_P(object)->read_dimension(object, property, BP_VAR_R, &rv)) != NULL) { + + if (Z_TYPE_P(z) == IS_OBJECT && Z_OBJ_HT_P(z)->get) { + zval rv2; + zval *value = Z_OBJ_HT_P(z)->get(z, &rv2); + + if (z == &rv) { + zval_ptr_dtor(&rv); + } + ZVAL_COPY_VALUE(z, value); + } + div_function(&res, Z_ISREF_P(z) ? Z_REFVAL_P(z) : z, value); + Z_OBJ_HT_P(object)->write_dimension(object, property, &res); + if (z == &rv) { + zval_ptr_dtor(&rv); + } +//??? if (retval) { +//??? ZVAL_COPY(retval, &res); +//??? } + zval_ptr_dtor(&res); + } else { + zend_error(E_WARNING, "Attempt to assign property of non-object"); +//??? if (retval) { +//??? ZVAL_NULL(retval); +//??? } + } + } else { + if (UNEXPECTED(Z_TYPE_P(container) == IS_STRING)) { + if (!dim) { + zend_throw_error(NULL, "[] operator not supported for strings"); + } else { + zend_check_string_offset(dim, BP_VAR_RW); + zend_wrong_string_offset(); + } +//??? } else if (EXPECTED(Z_TYPE_P(container) <= IS_FALSE)) { +//??? ZEND_VM_C_GOTO(assign_dim_op_convert_to_array); + } else { +//??? if (UNEXPECTED(OP1_TYPE != IS_VAR || EXPECTED(!Z_ISERROR_P(container)))) { + zend_error(E_WARNING, "Cannot use a scalar value as an array"); +//??? } +//??? if (retval) { +//??? ZVAL_NULL(retval); +//??? } + } + } +} + static void ZEND_FASTCALL zend_jit_zval_copy_unref_helper(zval *dst, zval *src) { ZVAL_UNREF(src); diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 8c03b637e8919..6ebc72a79f0f1 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -1744,6 +1744,7 @@ static int zend_jit_math_helper(dasm_State **Dst, uint32_t res_reg, uint32_t res_offset, uint32_t res_info) +/* Labels: 1,2,3,4,5 */ { zend_bool same_ops = (op1_type == op2_type) && (op1.var == op2.var); zend_bool has_slow = @@ -1755,26 +1756,26 @@ static int zend_jit_math_helper(dasm_State **Dst, if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)) { if (op1_info & (MAY_BE_ANY-MAY_BE_LONG)) { if (op1_info & MAY_BE_DOUBLE) { - | IF_NOT_Z_TYPE Ra(op1_reg)+op1_offset, IS_LONG, >4 + | IF_NOT_Z_TYPE Ra(op1_reg)+op1_offset, IS_LONG, >3 } else { - | IF_NOT_Z_TYPE Ra(op1_reg)+op1_offset, IS_LONG, >9 + | IF_NOT_Z_TYPE Ra(op1_reg)+op1_offset, IS_LONG, >5 } } if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_LONG))) { if (op2_info & MAY_BE_DOUBLE) { - | IF_NOT_Z_TYPE Ra(op2_reg)+op2_offset, IS_LONG, >3 + | IF_NOT_Z_TYPE Ra(op2_reg)+op2_offset, IS_LONG, >1 |.cold_code - |3: + |1: if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { - | IF_NOT_Z_TYPE Ra(op2_reg)+op2_offset, IS_DOUBLE, >9 + | IF_NOT_Z_TYPE Ra(op2_reg)+op2_offset, IS_DOUBLE, >5 } if (!zend_jit_math_long_double(Dst, opline, op1_type, op1, op1_reg, op1_offset, op2_type, op2, op2_reg, op2_offset, res_reg, res_offset)) { return 0; } - | jmp >6 + | jmp >4 |.code } else { - | IF_NOT_Z_TYPE Ra(op2_reg)+op2_offset, IS_LONG, >9 + | IF_NOT_Z_TYPE Ra(op2_reg)+op2_offset, IS_LONG, >5 } } if (!zend_jit_math_long_long(Dst, opline, op_array, ssa, op1_type, op1, op1_reg, op1_offset, op2_type, op2, op2_reg, op2_offset, res_reg, res_offset, res_info)) { @@ -1782,32 +1783,32 @@ static int zend_jit_math_helper(dasm_State **Dst, } if (op1_info & MAY_BE_DOUBLE) { |.cold_code - |4: + |3: if (op1_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { - | IF_NOT_Z_TYPE Ra(op1_reg)+op1_offset, IS_DOUBLE, >9 + | IF_NOT_Z_TYPE Ra(op1_reg)+op1_offset, IS_DOUBLE, >5 } if (op2_info & MAY_BE_DOUBLE) { if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { if (!same_ops) { - | IF_NOT_Z_TYPE, Ra(op2_reg)+op2_offset, IS_DOUBLE, >5 + | IF_NOT_Z_TYPE, Ra(op2_reg)+op2_offset, IS_DOUBLE, >1 } else { - | IF_NOT_Z_TYPE, Ra(op2_reg)+op2_offset, IS_DOUBLE, >9 + | IF_NOT_Z_TYPE, Ra(op2_reg)+op2_offset, IS_DOUBLE, >5 } } if (!zend_jit_math_double_double(Dst, opline, op1_type, op1, op1_reg, op1_offset, op2_type, op2, op2_reg, op2_offset, res_reg, res_offset)) { return 0; } - | jmp >6 + | jmp >4 } if (!same_ops) { - |5: + |1: if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { - | IF_NOT_Z_TYPE Ra(op2_reg)+op2_offset, IS_LONG, >9 + | IF_NOT_Z_TYPE Ra(op2_reg)+op2_offset, IS_LONG, >5 } if (!zend_jit_math_double_long(Dst, opline, op1_type, op1, op1_reg, op1_offset, op2_type, op2, op2_reg, op2_offset, res_reg, res_offset)) { return 0; } - | jmp >6 + | jmp >4 } |.code } @@ -1815,14 +1816,14 @@ static int zend_jit_math_helper(dasm_State **Dst, !(op1_info & MAY_BE_LONG) && (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { if (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) { - | IF_NOT_Z_TYPE Ra(op1_reg)+op1_offset, IS_DOUBLE, >9 + | IF_NOT_Z_TYPE Ra(op1_reg)+op1_offset, IS_DOUBLE, >5 } if (op2_info & MAY_BE_DOUBLE) { if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { if (!same_ops && (op2_info & MAY_BE_LONG)) { - | IF_NOT_Z_TYPE Ra(op2_reg)+op2_offset, IS_DOUBLE, >3 + | IF_NOT_Z_TYPE Ra(op2_reg)+op2_offset, IS_DOUBLE, >1 } else { - | IF_NOT_Z_TYPE Ra(op2_reg)+op2_offset, IS_DOUBLE, >9 + | IF_NOT_Z_TYPE Ra(op2_reg)+op2_offset, IS_DOUBLE, >5 } } if (!zend_jit_math_double_double(Dst, opline, op1_type, op1, op1_reg, op1_offset, op2_type, op2, op2_reg, op2_offset, res_reg, res_offset)) { @@ -1833,15 +1834,15 @@ static int zend_jit_math_helper(dasm_State **Dst, if (op2_info & MAY_BE_DOUBLE) { |.cold_code } - |3: + |1: if (op2_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) { - | IF_NOT_Z_TYPE Ra(op2_reg)+op2_offset, IS_LONG, >9 + | IF_NOT_Z_TYPE Ra(op2_reg)+op2_offset, IS_LONG, >5 } if (!zend_jit_math_double_long(Dst, opline, op1_type, op1, op1_reg, op1_offset, op2_type, op2, op2_reg, op2_offset, res_reg, res_offset)) { return 0; } if (op2_info & MAY_BE_DOUBLE) { - | jmp >6 + | jmp >4 |.code } } @@ -1849,14 +1850,14 @@ static int zend_jit_math_helper(dasm_State **Dst, !(op2_info & MAY_BE_LONG) && (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { if (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) { - | IF_NOT_Z_TYPE Ra(op2_reg)+op2_offset, IS_DOUBLE, >9 + | IF_NOT_Z_TYPE Ra(op2_reg)+op2_offset, IS_DOUBLE, >5 } if (op1_info & MAY_BE_DOUBLE) { if (!same_ops && (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { if (!same_ops && (op1_info & MAY_BE_LONG)) { - | IF_NOT_Z_TYPE Ra(op1_reg)+op1_offset, IS_DOUBLE, >3 + | IF_NOT_Z_TYPE Ra(op1_reg)+op1_offset, IS_DOUBLE, >1 } else { - | IF_NOT_Z_TYPE Ra(op1_reg)+op1_offset, IS_DOUBLE, >9 + | IF_NOT_Z_TYPE Ra(op1_reg)+op1_offset, IS_DOUBLE, >5 } } if (!zend_jit_math_double_double(Dst, opline, op1_type, op1, op1_reg, op1_offset, op2_type, op2, op2_reg, op2_offset, res_reg, res_offset)) { @@ -1867,25 +1868,25 @@ static int zend_jit_math_helper(dasm_State **Dst, if (op1_info & MAY_BE_DOUBLE) { |.cold_code } - |3: + |1: if (op1_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) { - | IF_NOT_Z_TYPE Ra(op1_reg)+op1_offset, IS_LONG, >9 + | IF_NOT_Z_TYPE Ra(op1_reg)+op1_offset, IS_LONG, >5 } if (!zend_jit_math_long_double(Dst, opline, op1_type, op1, op1_reg, op1_offset, op2_type, op2, op2_reg, op2_offset, res_reg, res_offset)) { return 0; } if (op1_info & MAY_BE_DOUBLE) { - | jmp >6 + | jmp >4 |.code } } } - |6: + |4: if (has_slow) { |.cold_code - |9: + |5: | SAVE_VALID_OPLINE opline if (res_reg != FCARG1a || res_offset != 0) { | lea FCARG1a, [Ra(res_reg)+res_offset] @@ -1925,7 +1926,7 @@ static int zend_jit_math_helper(dasm_State **Dst, || if (zend_may_throw(opline, op_array, ssa)) { || zend_jit_check_exception(Dst); || } - | jmp <6 + | jmp <4 |.code } @@ -1970,6 +1971,7 @@ fallback: } static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, uint32_t type, uint32_t op2_info, uint32_t found, uint32_t not_found) +/* Labels: 1,2,3 */ { if (op2_info & MAY_BE_LONG) { if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_LONG)) { @@ -2056,26 +2058,30 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o case BP_VAR_RW: | SAVE_VALID_OPLINE opline | // zend_error(E_NOTICE,"Undefined offset: " ZEND_LONG_FMT, hval); + | //retval = zend_hash_index_update(ht, hval, &EG(uninitialized_zval)); + | EXT_CALL zend_jit_fetch_dimension_rw_long_helper, r0 + | jmp >8 + break; + case BP_VAR_W: + | //retval = zend_hash_index_add_new(ht, hval, &EG(uninitialized_zval)); + | LOAD_ADDR FCARG2a, &EG(uninitialized_zval) + || if (ZEND_DEBUG) { + || const char *filename = op_array->filename ? op_array->filename->val : NULL; |.if X64 - | mov CARG1, E_NOTICE - | LOAD_ADDR CARG2, "Undefined offset: " ZEND_LONG_FMT - | LONG_LOAD CARG3, opline->op2_type, opline->op2 - | EXT_CALL zend_error, r0 + | LOAD_ADDR CARG4, filename + | mov CARG5d, opline->lineno |.else - | sub r4, 4 - | LONG_LOAD r0, opline->op2_type, opline->op2 - | push r0 - | push "Undefined offset: " ZEND_LONG_FMT - | push E_NOTICE - | EXT_CALL zend_error, r0 - | add r4, 16 + | push opline->lineno + | push filename |.endif - |//retval = zend_hash_index_update(ht, hval, &EG(uninitialized_zval)); - |//???? - break; - case BP_VAR_W: - |//retval = zend_hash_index_add_new(ht, hval, &EG(uninitialized_zval)); - |//???? + || } + |.if X64 + | LOAD_ADDR CARG3, &EG(uninitialized_zval) + |.else + | PUSH_ADDR &EG(uninitialized_zval), r0 + |.endif + | EXT_CALL _zend_hash_index_add_new, r0 + | jmp >8 break; default: ZEND_ASSERT(0); @@ -2097,78 +2103,76 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o | // offset_key = Z_STR_P(dim); | LONG_LOAD FCARG2a, opline->op2_type, opline->op2 | // retval = zend_hash_find(ht, offset_key); - if (opline->op2_type != IS_CONST) { - | EXT_CALL zend_jit_symtable_find, r0 - } else { - | EXT_CALL zend_hash_find, r0 - } - | test r0, r0 - | jz >2 // NOT_FOUND - | // if (UNEXPECTED(Z_TYPE_P(retval) == IS_INDIRECT)) - | IF_Z_TYPE r0, IS_INDIRECT, >1 // SLOW - |.cold_code - |1: - | // retval = Z_INDIRECT_P(retval); - | GET_Z_PTR r0, r0 - | IF_NOT_Z_TYPE r0, IS_UNDEF, >8 - |2: switch (type) { case BP_VAR_R: - // zend_error(E_NOTICE, "Undefined index: %s", ZSTR_VAL(offset_key)); - | SAVE_VALID_OPLINE opline - |.if X64 - | mov CARG1, E_NOTICE - | LOAD_ADDR CARG2, "Undefined index: %s" - | LONG_LOAD CARG3, opline->op2_type, opline->op2 - | add CARG3, offsetof(zend_string, val) - | EXT_CALL zend_error, r0 - |.else - | sub r4, 4 - | LONG_LOAD r0, opline->op2_type, opline->op2 - | add r0, offsetof(zend_string, val) - | push r0 - | push "Undefined index: %s" - | push E_NOTICE - | EXT_CALL zend_error, r0 - | add r4, 16 - |.endif - /* break missing intentionally */ case BP_VAR_IS: case BP_VAR_UNSET: - | // retval = &EG(uninitialized_zval); - | SET_Z_TYPE_INFO FP + opline->result.var, IS_NULL - | jmp >9 + if (opline->op2_type != IS_CONST) { + | EXT_CALL zend_jit_symtable_find, r0 + } else { + | EXT_CALL zend_hash_find, r0 + } + | test r0, r0 + | jz >2 // NOT_FOUND + | // if (UNEXPECTED(Z_TYPE_P(retval) == IS_INDIRECT)) + | IF_Z_TYPE r0, IS_INDIRECT, >1 // SLOW + |.cold_code + |1: + | // retval = Z_INDIRECT_P(retval); + | GET_Z_PTR r0, r0 + | IF_NOT_Z_TYPE r0, IS_UNDEF, >8 + |2: + switch (type) { + case BP_VAR_R: + // zend_error(E_NOTICE, "Undefined index: %s", ZSTR_VAL(offset_key)); + | SAVE_VALID_OPLINE opline + |.if X64 + | mov CARG1, E_NOTICE + | LOAD_ADDR CARG2, "Undefined index: %s" + | LONG_LOAD CARG3, opline->op2_type, opline->op2 + | add CARG3, offsetof(zend_string, val) + | EXT_CALL zend_error, r0 + |.else + | sub r4, 4 + | LONG_LOAD r0, opline->op2_type, opline->op2 + | add r0, offsetof(zend_string, val) + | push r0 + | push "Undefined index: %s" + | push E_NOTICE + | EXT_CALL zend_error, r0 + | add r4, 16 + |.endif + /* break missing intentionally */ + case BP_VAR_IS: + case BP_VAR_UNSET: + | // retval = &EG(uninitialized_zval); + | SET_Z_TYPE_INFO FP + opline->result.var, IS_NULL + | jmp >9 + break; + default: + ZEND_ASSERT(0); + } + |.code break; case BP_VAR_RW: - // zend_error(E_NOTICE, "Undefined index: %s", ZSTR_VAL(offset_key)); | SAVE_VALID_OPLINE opline - |.if X64 - | mov CARG1, E_NOTICE - | LOAD_ADDR CARG2, "Undefined index: %s" - | LONG_LOAD CARG3, opline->op2_type, opline->op2 - | add CARG3, offsetof(zend_string, val) - | EXT_CALL zend_error, r0 - |.else - | sub r4, 4 - | LONG_LOAD r0, opline->op2_type, opline->op2 - | add r0, offsetof(zend_string, val) - | push r0 - | push "Undefined index: %s" - | push E_NOTICE - | EXT_CALL zend_error, r0 - | add r4, 16 - |.endif - |//retval = zend_hash_update(ht, offset_key, &EG(uninitialized_zval)); - |//???? or ZVAL_NULL(retval); + if (opline->op2_type != IS_CONST) { + | EXT_CALL zend_jit_symtable_lookup_rw, r0 + } else { + | EXT_CALL zend_jit_hash_lookup_rw, r0 + } break; case BP_VAR_W: - |//retval = zend_hash_add_new(ht, offset_key, &EG(uninitialized_zval)); - |//???? or ZVAL_NULL(retval); + | SAVE_VALID_OPLINE opline + if (opline->op2_type != IS_CONST) { + | EXT_CALL zend_jit_symtable_lookup_w, r0 + } else { + | EXT_CALL zend_jit_hash_lookup_w, r0 + } break; default: ZEND_ASSERT(0); } - |.code if (op2_info & (MAY_BE_ANY - (MAY_BE_LONG|MAY_BE_STRING))) { | jmp >8 } @@ -2181,30 +2185,43 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o } | SAVE_VALID_OPLINE opline | LOAD_ZVAL_ADDR FCARG2a, opline->op2_type, opline->op2 - |.if X64 - | lea CARG3, [FP + opline->result.var] - |.else - | lea r0, [FP + opline->result.var] - | push r0 - |.endif switch (type) { case BP_VAR_R: + |.if X64 + | lea CARG3, [FP + opline->result.var] + |.else + | lea r0, [FP + opline->result.var] + | push r0 + |.endif | EXT_CALL zend_jit_fetch_dim_r_helper, r0 + | jmp >9 break; case BP_VAR_IS: case BP_VAR_UNSET: + |.if X64 + | lea CARG3, [FP + opline->result.var] + |.else + | lea r0, [FP + opline->result.var] + | push r0 + |.endif | EXT_CALL zend_jit_fetch_dim_is_helper, r0 + | jmp >9 break; case BP_VAR_RW: - |//????EXT_CALL zend_jit_fetch_dim_rw_helper, r0 + | EXT_CALL zend_jit_fetch_dim_rw_helper, r0 + | test r0, r0 + | jne >8 + | jmp >9 break; case BP_VAR_W: - |//????EXT_CALL zend_jit_fetch_dim_w_helper, r0 + | EXT_CALL zend_jit_fetch_dim_w_helper, r0 + | test r0, r0 + | jne >8 + | jmp >9 break; default: ZEND_ASSERT(0); } - | jmp >9 if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) { |.code } @@ -2236,41 +2253,174 @@ static int zend_jit_assign_dim_math(dasm_State **Dst, const zend_op *opline, zen if (op1_info & MAY_BE_ARRAY) { if (op1_info & MAY_BE_REF) { | IF_NOT_Z_TYPE FCARG1a, IS_ARRAY, >7 + | // SEPARATE_ARRAY(var_ptr); + | GET_Z_PTR r0, FCARG1a + | cmp dword [r0], 1 // if (GC_REFCOUNTED() > 1) + | ja >1 + |.cold_code + |1: + | IF_Z_FLAGS FCARG1a, IS_TYPE_IMMUTABLE, >2 + | GC_DELREF r0 + |2: + | mov aword [r4], FCARG1a // save + | ZVAL_COPY_CTOR_FUNC (op_array->filename ? ZSTR_VAL(op_array->filename) : NULL), opline->lineno + | mov FCARG1a, aword [r4] // restore | GET_Z_PTR FCARG1a, FCARG1a + | jmp >3 + |.code + | mov FCARG1a, r0 + |3: } else { if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { | IF_NOT_Z_TYPE FP + opline->op1.var, IS_ARRAY, >7 } - | LONG_LOAD FCARG1a, opline->op1_type, opline->op1 - } - if (!zend_jit_fetch_dimension_address_inner(Dst, opline, op_array, BP_VAR_RW, op2_info, 8, 8)) { - return 0; - } - - |8: - - if (op1_info & (MAY_BE_ARRAY_OF_REF)) { - | ZVAL_DEREF r0, MAY_BE_REF - } - if (op1_info & (MAY_BE_ARRAY_OF_STRING|MAY_BE_ARRAY_OF_ARRAY)) { - // SEPARATE_ZVAL_NOREF(var_ptr); - | IF_Z_FLAGS FCARG1a, IS_TYPE_COPYABLE + IS_TYPE_IMMUTABLE, >1 + | // SEPARATE_ARRAY(var_ptr); + | LONG_LOAD r0, opline->op1_type, opline->op1 + | cmp dword [r0], 1 // if (GC_REFCOUNTED() > 1) + | ja >1 |.cold_code |1: - | GET_Z_PTR r0, FCARG1a - | cmp dword [r0], 1 // if (GC_REFCOUNTED() > 1) - | jbe >3 - | IF_Z_FLAGS FCARG1a, IS_TYPE_IMMUTABLE, >2 + | IF_Z_FLAGS FP + opline->op1.var, IS_TYPE_IMMUTABLE, >2 | GC_DELREF r0 |2: - | mov aword [r4], FCARG1a // save + | LOAD_ZVAL_ADDR FCARG1a, opline->op1_type, opline->op1 | ZVAL_COPY_CTOR_FUNC (op_array->filename ? ZSTR_VAL(op_array->filename) : NULL), opline->lineno - | mov FCARG1a, aword [r4] // restore | jmp >3 |.code |3: + | LONG_LOAD FCARG1a, opline->op1_type, opline->op1 + } + } else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE)) { + if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { + if (op1_info & (MAY_BE_NULL|MAY_BE_FALSE)) { + if (op1_info & MAY_BE_REF) { + | cmp byte [FCARG1a + 8], IS_FALSE + } else { + | cmp byte [FP + opline->op1.var + 8], IS_FALSE + } + | jg >2 + } + } + if (op1_info & MAY_BE_UNDEF) { + | IF_NOT_Z_TYPE FP + opline->op1.var, IS_UNDEF, >1 + | SAVE_VALID_OPLINE opline + | mov FCARG1a, opline->op1.var + | EXT_CALL zend_jit_undefined_op_helper, r0 + |1: + } + |2: + | // ZVAL_NEW_ARR(container); + | // zend_hash_init(Z_ARRVAL_P(container), 8, NULL, ZVAL_PTR_DTOR, 0); + if (op1_info & MAY_BE_REF) { + | mov [r4], FCARG1a // save + } + || if (ZEND_DEBUG) { + || const char *filename = op_array->filename ? op_array->filename->val : NULL; + |.if X64 + | LOAD_ADDR CARG3, filename + | xor CARG4d, opline->lineno + |.else + | push opline->lineno + | push filename + |.endif + || } else { + |.if not X64 + | sub r4, 8 + |.endif + || } + |.if X64 + | LOAD_ADDR CARG2, 8 + || if (!(op1_info & MAY_BE_REF)) { + | LOAD_ZVAL_ADDR FCARG1a, opline->op1_type, opline->op1 + || } + |.else + | push 8 + || if (!(op1_info & MAY_BE_REF)) { + | LOAD_ZVAL_ADDR FCARG1a, opline->op1_type, opline->op1 + || } + | push FCARG1a + |.endif + | EXT_CALL _array_init, r0 + |.if not X64 + | add r4, 16 + |.endif + if (op1_info & MAY_BE_REF) { + | mov FCARG1a, [r4] // restore + | GET_Z_PTR FCARG1a, FCARG1a + } else { + | LONG_LOAD FCARG1a, opline->op1_type, opline->op1 + } + } + + if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { + |6: + if (opline->op2_type == IS_UNUSED) { + | // var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval)); + | LOAD_ADDR FCARG2a, &EG(uninitialized_zval) + || if (ZEND_DEBUG) { + || const char *filename = op_array->filename ? op_array->filename->val : NULL; + |.if X64 + | LOAD_ADDR CARG3, filename + | mov CARG4d, opline->lineno + |.else + | push opline->lineno + | push filename + |.endif + || } + | EXT_CALL _zend_hash_next_index_insert, r0 + | // if (UNEXPECTED(!var_ptr)) { + | test r0, r0 + | jz >1 + |.cold_code + |1: + | // zend_error(E_WARNING, "Cannot add element to the array as the next element is already occupied"); + | SAVE_VALID_OPLINE opline + |.if X64 + | mov CARG1, E_WARNING + | LOAD_ADDR CARG2, "Cannot add element to the array as the next element is already occupied" + | EXT_CALL zend_error, r0 + |.else + | sub r4, 8 + | push "Cannot add element to the array as the next element is already occupied" + | push E_WARNING + | EXT_CALL zend_error, r0 + | add r4, 16 + |.endif + | //ZEND_VM_C_GOTO(assign_dim_op_ret_null); + | jmp >9 + |.code + | mov FCARG1a, r0 + } else { + if (!zend_jit_fetch_dimension_address_inner(Dst, opline, op_array, BP_VAR_RW, op2_info, 8, 8)) { + return 0; + } + + |8: + | mov FCARG1a, r0 + if (op1_info & (MAY_BE_ARRAY_OF_REF)) { + | ZVAL_DEREF FCARG1a, MAY_BE_REF + } + if (op1_info & (MAY_BE_ARRAY_OF_STRING|MAY_BE_ARRAY_OF_ARRAY)) { + // SEPARATE_ZVAL_NOREF(var_ptr); + | IF_Z_FLAGS FCARG1a, IS_TYPE_COPYABLE + IS_TYPE_IMMUTABLE, >1 + |.cold_code + |1: + | GET_Z_PTR r0, FCARG1a + | cmp dword [r0], 1 // if (GC_REFCOUNTED() > 1) + | jbe >3 + | IF_Z_FLAGS FCARG1a, IS_TYPE_IMMUTABLE, >2 + | GC_DELREF r0 + |2: + | mov aword [r4], FCARG1a // save + | ZVAL_COPY_CTOR_FUNC (op_array->filename ? ZSTR_VAL(op_array->filename) : NULL), opline->lineno + | mov FCARG1a, aword [r4] // restore + | jmp >3 + |.code + |3: + } } - if (!zend_jit_math_helper(Dst, opline, op_array, ssa, IS_CV, opline->op1, FCARG1a, 0, op1_info, (opline+1)->op2_type, (opline+1)->op2, FP, (opline+1)->op2.var, OP2_DATA_INFO(), FCARG1a, 0, OP1_DEF_INFO())) { + + if (!zend_jit_math_helper(Dst, opline, op_array, ssa, IS_CV, opline->op1, FCARG1a, 0, zend_array_element_type(op1_info, 0, 0), (opline+1)->op1_type, (opline+1)->op1, FP, (opline+1)->op1.var, OP1_DATA_INFO(), FCARG1a, 0, OP1_DEF_INFO())) { return 0; } } @@ -2280,93 +2430,107 @@ static int zend_jit_assign_dim_math(dasm_State **Dst, const zend_op *opline, zen |.cold_code |7: } - | int3 -#if 0 - if (op1_info & MAY_BE_STRING) { - if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_STRING))) { - if (op1_info & MAY_BE_REF) { - | IF_NOT_Z_TYPE FCARG1a, IS_STRING, >6 - } else { - | IF_NOT_Z_TYPE FP + opline->op1.var, IS_STRING, >6 - } + + if ((op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE)) && + (op1_info & MAY_BE_ARRAY)) { + if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { + if (op1_info & (MAY_BE_NULL|MAY_BE_FALSE)) { + if (op1_info & MAY_BE_REF) { + | cmp byte [FCARG1a + 8], IS_FALSE + } else { + | cmp byte [FP + opline->op1.var + 8], IS_FALSE + } + | jg >2 + } } - | SAVE_VALID_OPLINE opline - if (!(op1_info & MAY_BE_REF)) { - | LOAD_ZVAL_ADDR FCARG1a, opline->op1_type, opline->op1 - } - | LOAD_ZVAL_ADDR FCARG2a, opline->op2_type, opline->op2 + if (op1_info & MAY_BE_UNDEF) { + | IF_NOT_Z_TYPE FP + opline->op1.var, IS_UNDEF, >1 + | SAVE_VALID_OPLINE opline + | mov FCARG1a, opline->op1.var + | EXT_CALL zend_jit_undefined_op_helper, r0 + |1: + } + | // ZVAL_NEW_ARR(container); + | // zend_hash_init(Z_ARRVAL_P(container), 8, NULL, ZVAL_PTR_DTOR, 0); + if (op1_info & MAY_BE_REF) { + | mov [r4], FCARG1a // save + } + || if (ZEND_DEBUG) { + || const char *filename = op_array->filename ? op_array->filename->val : NULL; |.if X64 - | lea CARG3, [FP + opline->result.var] + | LOAD_ADDR CARG3, filename + | xor CARG4d, opline->lineno |.else - | lea r0, [FP + opline->result.var] - | push r0 + | push opline->lineno + | push filename |.endif - if (opline->opcode == ZEND_FETCH_DIM_R) { - | EXT_CALL zend_jit_fetch_dim_str_r_helper, r0 - } else if (opline->opcode == ZEND_FETCH_DIM_IS) { - | EXT_CALL zend_jit_fetch_dim_str_is_helper, r0 + || } else { + |.if not X64 + | sub r4, 8 + |.endif + || } + |.if X64 + | LOAD_ADDR CARG2, 8 + || if (!(op1_info & MAY_BE_REF)) { + | LOAD_ZVAL_ADDR FCARG1a, opline->op1_type, opline->op1 + || } + |.else + | push 8 + || if (!(op1_info & MAY_BE_REF)) { + | LOAD_ZVAL_ADDR FCARG1a, opline->op1_type, opline->op1 + || } + | push FCARG1a + |.endif + | EXT_CALL _array_init, r0 + |.if not X64 + | add r4, 16 + |.endif + if (op1_info & MAY_BE_REF) { + | mov FCARG1a, [r4] // restore + | GET_Z_PTR FCARG1a, FCARG1a } else { - ZEND_ASSERT(0); + | LONG_LOAD FCARG1a, opline->op1_type, opline->op1 } - | jmp >9 // END - |6: + | // ZEND_VM_C_GOTO(assign_dim_op_new_array); + | jmp <6 + |2: } - if (op1_info & MAY_BE_OBJECT) { - if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_STRING|MAY_BE_OBJECT))) { - if (op1_info & MAY_BE_REF) { - | IF_NOT_Z_TYPE FCARG1a, IS_OBJECT, >6 - } else { - | IF_NOT_Z_TYPE FP + opline->op1.var, IS_OBJECT, >6 - } - } + if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { | SAVE_VALID_OPLINE opline if (!(op1_info & MAY_BE_REF)) { | LOAD_ZVAL_ADDR FCARG1a, opline->op1_type, opline->op1 - } - | LOAD_ZVAL_ADDR FCARG2a, opline->op2_type, opline->op2 + } + if (opline->op2_type == IS_UNUSED) { + | xor FCARG2a, FCARG2a + } else { + | LOAD_ZVAL_ADDR FCARG2a, opline->op2_type, opline->op2 + } |.if X64 - | lea CARG3, [FP + opline->result.var] + | LOAD_ZVAL_ADDR CARG3, (opline+1)->op1_type, (opline+1)->op1 |.else - | lea r0, [FP + opline->result.var] - | push r0 + | PUSH_ZVAL_ADDR (opline+1)->op1_type, (opline+1)->op1, r0 |.endif - if (opline->opcode == ZEND_FETCH_DIM_R) { - | EXT_CALL zend_jit_fetch_dim_obj_r_helper, r0 - } else if (opline->opcode == ZEND_FETCH_DIM_IS) { - | EXT_CALL zend_jit_fetch_dim_obj_is_helper, r0 - } else { - ZEND_ASSERT(0); + switch (opline->opcode) { + case ZEND_ASSIGN_ADD: + | EXT_CALL zend_jit_assign_dim_add_helper, r0 + break; + case ZEND_ASSIGN_SUB: + | EXT_CALL zend_jit_assign_dim_sub_helper, r0 + break; + case ZEND_ASSIGN_MUL: + | EXT_CALL zend_jit_assign_dim_mul_helper, r0 + break; + case ZEND_ASSIGN_DIV: + | EXT_CALL zend_jit_assign_dim_div_helper, r0 + break; + default: + ZEND_ASSERT(0); } - | jmp >9 // END - |6: } - if ((opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) || - (op2_info & MAY_BE_UNDEF)) { - | SAVE_VALID_OPLINE opline - } - if (opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) { - if (op1_info & MAY_BE_REF) { - | IF_NOT_Z_TYPE FCARG1a, IS_UNDEF, >1 - } else { - | IF_NOT_Z_TYPE FP + opline->op1.var, IS_UNDEF, >1 - } - | // zend_error(E_NOTICE, "Undefined variable: %s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); - | mov FCARG1d, opline->op1.var - | EXT_CALL zend_jit_undefined_op_helper, r0 - |1: - } - if (op2_info & MAY_BE_UNDEF) { - | IF_NOT_Z_TYPE FP + opline->op2.var, IS_UNDEF, >1 - | mov FCARG1d, opline->op2.var - | EXT_CALL zend_jit_undefined_op_helper, r0 - |1: - } - | SET_Z_TYPE_INFO FP + opline->result.var, IS_NULL - | jmp >9 // END -#endif if (op1_info & MAY_BE_ARRAY) { + | jmp >9 // END |.code } } @@ -2384,9 +2548,6 @@ static int zend_jit_assign_math(dasm_State **Dst, const zend_op *opline, zend_op uint32_t op1_info, op2_info; if (opline->extended_value == ZEND_ASSIGN_DIM) { -#if 1 - goto fallback; -#endif return zend_jit_assign_dim_math(Dst, opline, op_array, ssa); } else if (opline->extended_value == ZEND_ASSIGN_OBJ) { goto fallback; From 992ded2c4c687c50d1791c547b345a5028ab55d0 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 28 Sep 2016 14:31:54 +0300 Subject: [PATCH 250/569] Added missing helpers --- ext/opcache/jit/zend_jit_disasm_x86.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/ext/opcache/jit/zend_jit_disasm_x86.c b/ext/opcache/jit/zend_jit_disasm_x86.c index 0ee2a0ab2e1bd..fb0075e74997b 100644 --- a/ext/opcache/jit/zend_jit_disasm_x86.c +++ b/ext/opcache/jit/zend_jit_disasm_x86.c @@ -388,6 +388,11 @@ static int zend_jit_disasm_init(void) REGISTER_HELPER(zend_jit_leave_nested_func_helper); REGISTER_HELPER(zend_jit_leave_top_func_helper); REGISTER_HELPER(zend_jit_symtable_find); + REGISTER_HELPER(zend_jit_hash_lookup_rw); + REGISTER_HELPER(zend_jit_hash_lookup_w); + REGISTER_HELPER(zend_jit_symtable_lookup_rw); + REGISTER_HELPER(zend_jit_symtable_lookup_w); + REGISTER_HELPER(zend_jit_fetch_dimension_rw_long_helper); REGISTER_HELPER(zend_jit_undefined_op_helper); REGISTER_HELPER(zend_jit_fetch_dim_r_helper); REGISTER_HELPER(zend_jit_fetch_dim_is_helper); @@ -395,6 +400,12 @@ static int zend_jit_disasm_init(void) REGISTER_HELPER(zend_jit_fetch_dim_str_is_helper); REGISTER_HELPER(zend_jit_fetch_dim_obj_r_helper); REGISTER_HELPER(zend_jit_fetch_dim_obj_is_helper); + REGISTER_HELPER(zend_jit_fetch_dim_rw_helper); + REGISTER_HELPER(zend_jit_fetch_dim_w_helper); + REGISTER_HELPER(zend_jit_assign_dim_add_helper); + REGISTER_HELPER(zend_jit_assign_dim_sub_helper); + REGISTER_HELPER(zend_jit_assign_dim_mul_helper); + REGISTER_HELPER(zend_jit_assign_dim_div_helper); REGISTER_HELPER(zend_jit_zval_copy_unref_helper); REGISTER_HELPER(zend_jit_new_ref_helper); REGISTER_HELPER(zend_jit_fetch_global_helper); From 11989edebb4170d3646e5d2c4c2db93ee5469110 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 28 Sep 2016 14:32:18 +0300 Subject: [PATCH 251/569] ASSIGN_ADD/SUB/MUL optimization --- ext/opcache/jit/zend_jit_x86.dasc | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 6ebc72a79f0f1..61239264e3395 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -2087,15 +2087,13 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o ZEND_ASSERT(0); } |.code - if (op2_info & (MAY_BE_ANY - MAY_BE_LONG)) { + if (op2_info & MAY_BE_STRING) { | jmp >8 } - if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_LONG)) { - |3: - } } if (op2_info & MAY_BE_STRING) { + |3: if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) { | // if (EXPECTED(Z_TYPE_P(dim) == IS_STRING)) | IF_NOT_Z_TYPE FP + opline->op2.var, IS_STRING, >3 @@ -2275,20 +2273,20 @@ static int zend_jit_assign_dim_math(dasm_State **Dst, const zend_op *opline, zen | IF_NOT_Z_TYPE FP + opline->op1.var, IS_ARRAY, >7 } | // SEPARATE_ARRAY(var_ptr); - | LONG_LOAD r0, opline->op1_type, opline->op1 - | cmp dword [r0], 1 // if (GC_REFCOUNTED() > 1) + | LONG_LOAD FCARG1a, opline->op1_type, opline->op1 + | cmp dword [FCARG1a], 1 // if (GC_REFCOUNTED() > 1) | ja >1 |.cold_code |1: | IF_Z_FLAGS FP + opline->op1.var, IS_TYPE_IMMUTABLE, >2 - | GC_DELREF r0 + | GC_DELREF FCARG1a |2: | LOAD_ZVAL_ADDR FCARG1a, opline->op1_type, opline->op1 | ZVAL_COPY_CTOR_FUNC (op_array->filename ? ZSTR_VAL(op_array->filename) : NULL), opline->lineno + | LONG_LOAD FCARG1a, opline->op1_type, opline->op1 | jmp >3 |.code |3: - | LONG_LOAD FCARG1a, opline->op1_type, opline->op1 } } else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE)) { if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { From 2e61e365b9f10695455f3dd1b576a3479bcd82bd Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 28 Sep 2016 18:45:34 +0300 Subject: [PATCH 252/569] JIT for ASSIGN_DIM (incomplete) --- ext/opcache/jit/zend_jit.c | 7 + ext/opcache/jit/zend_jit_disasm_x86.c | 3 + ext/opcache/jit/zend_jit_helpers.c | 120 ++++ ext/opcache/jit/zend_jit_x86.dasc | 802 ++++++++++++++++++-------- 4 files changed, 698 insertions(+), 234 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 5a233cfe9886f..25f2f0aac90d3 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -1000,6 +1000,13 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) goto jit_failure; } break; +#if 0 + case ZEND_ASSIGN_DIM: + if (!zend_jit_assign_dim(&dasm_state, opline, op_array, ssa)) { + goto jit_failure; + } + break; +#endif case ZEND_ASSIGN: if (!zend_jit_assign(&dasm_state, opline, op_array, ssa)) { goto jit_failure; diff --git a/ext/opcache/jit/zend_jit_disasm_x86.c b/ext/opcache/jit/zend_jit_disasm_x86.c index fb0075e74997b..4adf4f6c58790 100644 --- a/ext/opcache/jit/zend_jit_disasm_x86.c +++ b/ext/opcache/jit/zend_jit_disasm_x86.c @@ -388,6 +388,8 @@ static int zend_jit_disasm_init(void) REGISTER_HELPER(zend_jit_leave_nested_func_helper); REGISTER_HELPER(zend_jit_leave_top_func_helper); REGISTER_HELPER(zend_jit_symtable_find); + REGISTER_HELPER(zend_jit_hash_index_lookup_rw); + REGISTER_HELPER(zend_jit_hash_index_lookup_w); REGISTER_HELPER(zend_jit_hash_lookup_rw); REGISTER_HELPER(zend_jit_hash_lookup_w); REGISTER_HELPER(zend_jit_symtable_lookup_rw); @@ -402,6 +404,7 @@ static int zend_jit_disasm_init(void) REGISTER_HELPER(zend_jit_fetch_dim_obj_is_helper); REGISTER_HELPER(zend_jit_fetch_dim_rw_helper); REGISTER_HELPER(zend_jit_fetch_dim_w_helper); + REGISTER_HELPER(zend_jit_assign_dim_helper); REGISTER_HELPER(zend_jit_assign_dim_add_helper); REGISTER_HELPER(zend_jit_assign_dim_sub_helper); REGISTER_HELPER(zend_jit_assign_dim_mul_helper); diff --git a/ext/opcache/jit/zend_jit_helpers.c b/ext/opcache/jit/zend_jit_helpers.c index dff3ee9f197aa..667695c2d1fc3 100644 --- a/ext/opcache/jit/zend_jit_helpers.c +++ b/ext/opcache/jit/zend_jit_helpers.c @@ -68,6 +68,27 @@ static zval* ZEND_FASTCALL zend_jit_symtable_find(HashTable *ht, zend_string *st return zend_hash_find(ht, str); } +static zval* ZEND_FASTCALL zend_jit_hash_index_lookup_rw(HashTable *ht, zend_long idx) +{ + zval *retval = zend_hash_index_find(ht, idx); + + if (!retval) { + zend_error(E_NOTICE,"Undefined index: %s", idx); + retval = zend_hash_index_update(ht, idx, &EG(uninitialized_zval)); + } + return retval; +} + +static zval* ZEND_FASTCALL zend_jit_hash_index_lookup_w(HashTable *ht, zend_long idx) +{ + zval *retval = zend_hash_index_find(ht, idx); + + if (!retval) { + retval = zend_hash_index_add_new(ht, idx, &EG(uninitialized_zval)); + } + return retval; +} + static zval* ZEND_FASTCALL zend_jit_hash_lookup_rw(HashTable *ht, zend_string *str) { zval *retval = zend_hash_find(ht, str); @@ -732,6 +753,105 @@ static zend_never_inline ZEND_COLD void zend_wrong_string_offset(void) zend_throw_error(NULL, msg); } +static zend_never_inline void zend_assign_to_string_offset(zval *str, zval *dim, zval *value, zval *result) +{ + zend_string *old_str; + zend_uchar c; + size_t string_len; + zend_long offset; + + offset = zend_check_string_offset(dim, BP_VAR_W); + if (offset < -(zend_long)Z_STRLEN_P(str)) { + /* Error on negative offset */ + zend_error(E_WARNING, "Illegal string offset: " ZEND_LONG_FMT, offset); + if (result) { + ZVAL_NULL(result); + } + return; + } + + if (Z_TYPE_P(value) != IS_STRING) { + /* Convert to string, just the time to pick the 1st byte */ + zend_string *tmp = zval_get_string(value); + + string_len = ZSTR_LEN(tmp); + c = (zend_uchar)ZSTR_VAL(tmp)[0]; + zend_string_release(tmp); + } else { + string_len = Z_STRLEN_P(value); + c = (zend_uchar)Z_STRVAL_P(value)[0]; + } + + if (string_len == 0) { + /* Error on empty input string */ + zend_error(E_WARNING, "Cannot assign an empty string to a string offset"); + if (result) { + ZVAL_NULL(result); + } + return; + } + + if (offset < 0) { /* Handle negative offset */ + offset += (zend_long)Z_STRLEN_P(str); + } + + if ((size_t)offset >= Z_STRLEN_P(str)) { + /* Extend string if needed */ + zend_long old_len = Z_STRLEN_P(str); + Z_STR_P(str) = zend_string_extend(Z_STR_P(str), offset + 1, 0); + Z_TYPE_INFO_P(str) = IS_STRING_EX; + memset(Z_STRVAL_P(str) + old_len, ' ', offset - old_len); + Z_STRVAL_P(str)[offset+1] = 0; + } else if (!Z_REFCOUNTED_P(str)) { + old_str = Z_STR_P(str); + Z_STR_P(str) = zend_string_init(Z_STRVAL_P(str), Z_STRLEN_P(str), 0); + Z_TYPE_INFO_P(str) = IS_STRING_EX; + zend_string_release(old_str); + } else { + SEPARATE_STRING(str); + zend_string_forget_hash_val(Z_STR_P(str)); + } + + Z_STRVAL_P(str)[offset] = c; + + if (result) { + /* Return the new character */ + if (CG(one_char_string)[c]) { + ZVAL_INTERNED_STR(result, CG(one_char_string)[c]); + } else { + ZVAL_NEW_STR(result, zend_string_init(Z_STRVAL_P(str) + offset, 1, 0)); + } + } +} + +static void ZEND_FASTCALL zend_jit_assign_dim_helper(zval *object_ptr, zval *dim, zval *value) +{ + if (EXPECTED(Z_TYPE_P(object_ptr) == IS_OBJECT)) { + + if (UNEXPECTED(!Z_OBJ_HT_P(object_ptr)->write_dimension)) { + zend_throw_error(NULL, "Cannot use object as array"); + } else { + Z_OBJ_HT_P(object_ptr)->write_dimension(object_ptr, dim, value); +//??? if (UNEXPECTED(RETURN_VALUE_USED(opline)) && EXPECTED(!EG(exception))) { +//??? ZVAL_COPY(EX_VAR(opline->result.var), value); +//??? } + } + } else if (EXPECTED(Z_TYPE_P(object_ptr) == IS_STRING)) { + if (dim) { + zend_throw_error(NULL, "[] operator not supported for strings"); + } else { + zend_assign_to_string_offset(object_ptr, dim, value, /*???UNEXPECTED(RETURN_VALUE_USED(opline)) ? EX_VAR(opline->result.var) :*/ NULL); + } + } else { +//??? if (OP1_TYPE != IS_VAR || EXPECTED(!Z_ISERROR_P(object_ptr))) { + zend_error(E_WARNING, "Cannot use a scalar value as an array"); +//??? } +//??? if (UNEXPECTED(RETURN_VALUE_USED(opline))) { +//??? ZVAL_NULL(EX_VAR(opline->result.var)); +//??? } + } +} + static void ZEND_FASTCALL zend_jit_assign_dim_add_helper(zval *container, zval *dim, zval *value) { if (EXPECTED(Z_TYPE_P(container) == IS_OBJECT)) { diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 61239264e3395..9a84c1d8858c4 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -1980,7 +1980,6 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o } | // hval = Z_LVAL_P(dim); | LONG_LOAD FCARG2a, opline->op2_type, opline->op2 - if (opline->op2_type == IS_CONST) { zend_long val = Z_LVAL_P(RT_CONSTANT(op_array, opline->op2)); if (val >= 0 && val < HT_MAX_SIZE) { @@ -2025,46 +2024,63 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o | jmp >2 // NOT_FOUND } |1: - | EXT_CALL zend_hash_index_find, r0 - | test r0, r0 - | jz >2 // NOT_FOUND - |.cold_code - |2: switch (type) { case BP_VAR_R: - | SAVE_VALID_OPLINE opline - | // zend_error(E_NOTICE,"Undefined offset: " ZEND_LONG_FMT, hval); - |.if X64 - | mov CARG1, E_NOTICE - | LOAD_ADDR CARG2, "Undefined offset: " ZEND_LONG_FMT - | LONG_LOAD CARG3, opline->op2_type, opline->op2 - | EXT_CALL zend_error, r0 - |.else - | sub r4, 4 - | LONG_LOAD r0, opline->op2_type, opline->op2 - | push r0 - | push "Undefined offset: " ZEND_LONG_FMT - | push E_NOTICE - | EXT_CALL zend_error, r0 - | add r4, 16 - |.endif - /* break missing intentionally */ case BP_VAR_IS: case BP_VAR_UNSET: - | // retval = &EG(uninitialized_zval); - | SET_Z_TYPE_INFO FP + opline->result.var, IS_NULL - | jmp >9 + | EXT_CALL zend_hash_index_find, r0 + | test r0, r0 + | jz >2 // NOT_FOUND + |.cold_code + |2: + switch (type) { + case BP_VAR_R: + | SAVE_VALID_OPLINE opline + | // zend_error(E_NOTICE,"Undefined offset: " ZEND_LONG_FMT, hval); + |.if X64 + | mov CARG1, E_NOTICE + | LOAD_ADDR CARG2, "Undefined offset: " ZEND_LONG_FMT + | LONG_LOAD CARG3, opline->op2_type, opline->op2 + | EXT_CALL zend_error, r0 + |.else + | sub r4, 4 + | LONG_LOAD r0, opline->op2_type, opline->op2 + | push r0 + | push "Undefined offset: " ZEND_LONG_FMT + | push E_NOTICE + | EXT_CALL zend_error, r0 + | add r4, 16 + |.endif + /* break missing intentionally */ + case BP_VAR_IS: + case BP_VAR_UNSET: + | // retval = &EG(uninitialized_zval); + | SET_Z_TYPE_INFO FP + opline->result.var, IS_NULL + | jmp >9 + break; + default: + ZEND_ASSERT(0); + } + |.code break; case BP_VAR_RW: + | SAVE_VALID_OPLINE opline + | EXT_CALL zend_jit_hash_index_lookup_rw, r0 + |.cold_code + |2: | SAVE_VALID_OPLINE opline | // zend_error(E_NOTICE,"Undefined offset: " ZEND_LONG_FMT, hval); | //retval = zend_hash_index_update(ht, hval, &EG(uninitialized_zval)); | EXT_CALL zend_jit_fetch_dimension_rw_long_helper, r0 | jmp >8 + |.code break; case BP_VAR_W: + | SAVE_VALID_OPLINE opline + | EXT_CALL zend_jit_hash_index_lookup_w, r0 + |.cold_code + |2: | //retval = zend_hash_index_add_new(ht, hval, &EG(uninitialized_zval)); - | LOAD_ADDR FCARG2a, &EG(uninitialized_zval) || if (ZEND_DEBUG) { || const char *filename = op_array->filename ? op_array->filename->val : NULL; |.if X64 @@ -2082,11 +2098,12 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o |.endif | EXT_CALL _zend_hash_index_add_new, r0 | jmp >8 + |.code break; default: ZEND_ASSERT(0); } - |.code + if (op2_info & MAY_BE_STRING) { | jmp >8 } @@ -2167,65 +2184,534 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o } else { | EXT_CALL zend_jit_hash_lookup_w, r0 } - break; - default: - ZEND_ASSERT(0); - } - if (op2_info & (MAY_BE_ANY - (MAY_BE_LONG|MAY_BE_STRING))) { - | jmp >8 + break; + default: + ZEND_ASSERT(0); + } + if (op2_info & (MAY_BE_ANY - (MAY_BE_LONG|MAY_BE_STRING))) { + | jmp >8 + } + } + + if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) { + if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) { + |.cold_code + |3: + } + | SAVE_VALID_OPLINE opline + | LOAD_ZVAL_ADDR FCARG2a, opline->op2_type, opline->op2 + switch (type) { + case BP_VAR_R: + |.if X64 + | lea CARG3, [FP + opline->result.var] + |.else + | lea r0, [FP + opline->result.var] + | push r0 + |.endif + | EXT_CALL zend_jit_fetch_dim_r_helper, r0 + | jmp >9 + break; + case BP_VAR_IS: + case BP_VAR_UNSET: + |.if X64 + | lea CARG3, [FP + opline->result.var] + |.else + | lea r0, [FP + opline->result.var] + | push r0 + |.endif + | EXT_CALL zend_jit_fetch_dim_is_helper, r0 + | jmp >9 + break; + case BP_VAR_RW: + | EXT_CALL zend_jit_fetch_dim_rw_helper, r0 + | test r0, r0 + | jne >8 + | jmp >9 + break; + case BP_VAR_W: + | EXT_CALL zend_jit_fetch_dim_w_helper, r0 + | test r0, r0 + | jne >8 + | jmp >9 + break; + default: + ZEND_ASSERT(0); + } + if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) { + |.code + } + } + + return 1; +} + +static int zend_jit_simple_assign(dasm_State **Dst, + const zend_op *opline, + zend_op_array *op_array, + zend_ssa *ssa, + uint32_t var_reg, + uint32_t var_offset, + uint32_t var_info, + zend_uchar val_type, + znode_op val, + uint32_t val_info, + uint32_t var2, + int in_cold) +/* Labels: 1,2,3 */ +{ + if (val_type == IS_CONST) { + zval *zv = RT_CONSTANT(op_array, val); + if (var2 == (uint32_t)-1) { + | ZVAL_COPY_CONST Ra(var_reg)+var_offset, var_info, zv, r0 + } else { + | ZVAL_COPY_CONST_2 Ra(var_reg)+var_offset, FP + var2, var_info, zv, r0 + } + || if (Z_REFCOUNTED_P(zv)) { +// TODO: += 2 + | ADDREF_CONST zv, r0 + || } + } else { + if (val_info & MAY_BE_UNDEF) { + if (in_cold) { + | IF_NOT_Z_TYPE FP + val.var, IS_UNDEF, >2 + } else { + | IF_Z_TYPE FP + val.var, IS_UNDEF, >1 + |.cold_code + |1: + } + | // zend_error(E_NOTICE, "Undefined variable: %s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); + | SAVE_VALID_OPLINE opline + | mov FCARG1d, val.var + | EXT_CALL zend_jit_undefined_op_helper, r0 + | SET_Z_TYPE_INFO FP + val.var, IS_NULL + | // ???? val or var ^ + | jmp >3 + if (in_cold) { + |2: + } else { + |.code + } + } + if (val_info & MAY_BE_REF) { + if (val_type == IS_CV) { + | lea r2, [FP + val.var] + | ZVAL_DEREF r2, val_info + if (var2 == (uint32_t)-1) { + | ZVAL_COPY_VALUE Ra(var_reg)+var_offset, r2, val_info, r0, eax, r1 + } else { + | ZVAL_COPY_VALUE_2 Ra(var_reg)+var_offset, FP + var2, r2, val_info, r0, eax, r1 + } + } else { + if (in_cold) { + | IF_NOT_Z_TYPE FP + val.var, IS_REFERENCE, >1 + } else { + | IF_Z_TYPE FP + val.var, IS_REFERENCE, >1 + |.cold_code + |1: + } + | // zend_refcounted *ref = Z_COUNTED_P(retval_ptr); + | GET_Z_PTR r0, FP + val.var + | // ZVAL_COPY_VALUE(return_value, &ref->value); + if (var2 == (uint32_t)-1) { + | ZVAL_COPY_VALUE Ra(var_reg)+var_offset, r0 + 8, val_info, r2, edx, r1 + } else { + | ZVAL_COPY_VALUE_2 Ra(var_reg)+var_offset, FP + var2, r0 + 8, val_info, r2, edx, r1 + } + | GC_DELREF r0 + | je >2 + | IF_NOT_REFCOUNTED dh, >3 + if (var2 == (uint32_t)-1) { + | GC_ADDREF r1 + } else { + | add dword [r1], 2 + } + | jmp >3 + |2: + if (var2 != (uint32_t)-1) { + | IF_NOT_REFCOUNTED dh, >2 + | GC_ADDREF r1 + |2: + } + | EFREE_SIZE r0, sizeof(zend_reference), op_array, opline + | jmp >3 + if (in_cold) { + |1: + } else { + |.code + } + if (var2 == (uint32_t)-1) { + | ZVAL_COPY_VALUE Ra(var_reg)+var_offset, FP + val.var, val_info, r0, eax, r1 + } else { + | ZVAL_COPY_VALUE_2 Ra(var_reg)+var_offset, FP + var2, FP + val.var, val_info, r0, eax, r1 + } + } + } else { + if (var2 == (uint32_t)-1) { + | ZVAL_COPY_VALUE Ra(var_reg)+var_offset, FP + val.var, val_info, r0, eax, r1 + } else { + | ZVAL_COPY_VALUE_2 Ra(var_reg)+var_offset, FP + var2, FP + val.var, val_info, r0, eax, r1 + } + } + if (val_type == IS_CV) { + if (var2 == (uint32_t)-1) { + | TRY_ADDREF val_info, ah, r1 + } else { + | TRY_ADDREF_2 val_info, ah, r1 + } + } else { + if (var2 != (uint32_t)-1) { + | TRY_ADDREF val_info, ah, r1 + } + } + |3: + } + return 1; +} + +static int zend_jit_assign_to_variable(dasm_State **Dst, + const zend_op *opline, + zend_op_array *op_array, + zend_ssa *ssa, + uint32_t var_reg, + uint32_t var_offset, + uint32_t var_info, + zend_uchar val_type, + znode_op val, + uint32_t val_info, + uint32_t var2) +/* Labels: 1,2,3,4,5 */ +{ + if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { + int in_cold = 0; + + if (var_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { + | IF_Z_REFCOUNTED Ra(var_reg)+var_offset, >1 + |.cold_code + |1: + in_cold = 1; + } + | // TODO: support for object->set + | // TODO: support for assignment to itself + | GET_Z_PTR r0, Ra(var_reg)+var_offset + | GC_DELREF r0 + | jnz >4 + | mov aword [r4], r0 // save + if (!zend_jit_simple_assign(Dst, opline, op_array, ssa, var_reg, var_offset, var_info, val_type, val, val_info, var2, in_cold)) { + return 0; + } + | mov FCARG1a, aword [r4] // restore + | ZVAL_DTOR_FUNC (op_array->filename ? ZSTR_VAL(op_array->filename) : NULL), opline->lineno + | jmp >3 + |4: + | IF_NOT_Z_FLAGS Ra(var_reg)+var_offset, IS_TYPE_COLLECTABLE, >5 + if (var_reg == FP) { + | GET_Z_PTR FCARG1a, Ra(var_reg)+var_offset + | IF_GC_INFO FCARG1a, >5 + } else if (var_reg != FCARG1a) { + | GET_Z_PTR FCARG1a, Ra(var_reg)+var_offset + | IF_GC_INFO FCARG1a, >5 + | mov [r4], Ra(var_reg) // save + } else { + | GET_Z_PTR r0, Ra(var_reg)+var_offset + | IF_GC_INFO r0, >5 + | mov [r4], Ra(var_reg) // save + | mov FCARG1a, r0 + } + | EXT_CALL gc_possible_root, r0 + if (var_reg != FP) { + | mov [r4], Ra(var_reg) // restore + } + if (in_cold) { + | jmp >5 + |.code + } + |5: + } + + if (!zend_jit_simple_assign(Dst, opline, op_array, ssa, var_reg, var_offset, var_info, val_type, val, val_info, var2, 0)) { + return 0; + } + |3: + + return 1; +} + +static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) +{ + uint32_t op1_info, op2_info, val_info; + + if (opline->op1_type != IS_CV || opline->result_type != IS_UNUSED) { + goto fallback; + } + + if (!ssa->ops || !ssa->var_info) { + goto fallback; + } + + op1_info = OP1_INFO(); + op2_info = OP2_INFO(); + val_info = OP1_DATA_INFO(); + + if (op1_info & MAY_BE_REF) { + | LOAD_ZVAL_ADDR FCARG1a, opline->op1_type, opline->op1 + | ZVAL_DEREF FCARG1a, op1_info + } + + if (op1_info & MAY_BE_ARRAY) { + if (op1_info & MAY_BE_REF) { + | IF_NOT_Z_TYPE FCARG1a, IS_ARRAY, >7 + | // SEPARATE_ARRAY(var_ptr); + | GET_Z_PTR r0, FCARG1a + | cmp dword [r0], 1 // if (GC_REFCOUNTED() > 1) + | ja >1 + |.cold_code + |1: + | IF_Z_FLAGS FCARG1a, IS_TYPE_IMMUTABLE, >2 + | GC_DELREF r0 + |2: + | mov aword [r4], FCARG1a // save + | ZVAL_COPY_CTOR_FUNC (op_array->filename ? ZSTR_VAL(op_array->filename) : NULL), opline->lineno + | mov FCARG1a, aword [r4] // restore + | GET_Z_PTR FCARG1a, FCARG1a + | jmp >3 + |.code + | mov FCARG1a, r0 + |3: + } else { + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { + | IF_NOT_Z_TYPE FP + opline->op1.var, IS_ARRAY, >7 + } + | // SEPARATE_ARRAY(var_ptr); + | LONG_LOAD FCARG1a, opline->op1_type, opline->op1 + | cmp dword [FCARG1a], 1 // if (GC_REFCOUNTED() > 1) + | ja >1 + |.cold_code + |1: + | IF_Z_FLAGS FP + opline->op1.var, IS_TYPE_IMMUTABLE, >2 + | GC_DELREF FCARG1a + |2: + | LOAD_ZVAL_ADDR FCARG1a, opline->op1_type, opline->op1 + | ZVAL_COPY_CTOR_FUNC (op_array->filename ? ZSTR_VAL(op_array->filename) : NULL), opline->lineno + | LONG_LOAD FCARG1a, opline->op1_type, opline->op1 + | jmp >3 + |.code + |3: + } + } else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE)) { + if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { + if (op1_info & (MAY_BE_NULL|MAY_BE_FALSE)) { + if (op1_info & MAY_BE_REF) { + | cmp byte [FCARG1a + 8], IS_FALSE + } else { + | cmp byte [FP + opline->op1.var + 8], IS_FALSE + } + | jg >2 + } + } + if (op1_info & MAY_BE_UNDEF) { + | IF_NOT_Z_TYPE FP + opline->op1.var, IS_UNDEF, >1 + | SAVE_VALID_OPLINE opline + | mov FCARG1a, opline->op1.var + | EXT_CALL zend_jit_undefined_op_helper, r0 + |1: + } + |2: + | // ZVAL_NEW_ARR(container); + | // zend_hash_init(Z_ARRVAL_P(container), 8, NULL, ZVAL_PTR_DTOR, 0); + if (op1_info & MAY_BE_REF) { + | mov [r4], FCARG1a // save + } + || if (ZEND_DEBUG) { + || const char *filename = op_array->filename ? op_array->filename->val : NULL; + |.if X64 + | LOAD_ADDR CARG3, filename + | xor CARG4d, opline->lineno + |.else + | push opline->lineno + | push filename + |.endif + || } else { + |.if not X64 + | sub r4, 8 + |.endif + || } + |.if X64 + | LOAD_ADDR CARG2, 8 + || if (!(op1_info & MAY_BE_REF)) { + | LOAD_ZVAL_ADDR FCARG1a, opline->op1_type, opline->op1 + || } + |.else + | push 8 + || if (!(op1_info & MAY_BE_REF)) { + | LOAD_ZVAL_ADDR FCARG1a, opline->op1_type, opline->op1 + || } + | push FCARG1a + |.endif + | EXT_CALL _array_init, r0 + |.if not X64 + | add r4, 16 + |.endif + if (op1_info & MAY_BE_REF) { + | mov FCARG1a, [r4] // restore + | GET_Z_PTR FCARG1a, FCARG1a + } else { + | LONG_LOAD FCARG1a, opline->op1_type, opline->op1 + } + } + + if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { + |6: + if (opline->op2_type == IS_UNUSED) { + | // var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval)); + | LOAD_ADDR FCARG2a, &EG(uninitialized_zval) + || if (ZEND_DEBUG) { + || const char *filename = op_array->filename ? op_array->filename->val : NULL; + |.if X64 + | LOAD_ADDR CARG3, filename + | mov CARG4d, opline->lineno + |.else + | push opline->lineno + | push filename + |.endif + || } + | EXT_CALL _zend_hash_next_index_insert, r0 + | // if (UNEXPECTED(!var_ptr)) { + | test r0, r0 + | jz >1 + |.cold_code + |1: + | // zend_error(E_WARNING, "Cannot add element to the array as the next element is already occupied"); + | SAVE_VALID_OPLINE opline + |.if X64 + | mov CARG1, E_WARNING + | LOAD_ADDR CARG2, "Cannot add element to the array as the next element is already occupied" + | EXT_CALL zend_error, r0 + |.else + | sub r4, 8 + | push "Cannot add element to the array as the next element is already occupied" + | push E_WARNING + | EXT_CALL zend_error, r0 + | add r4, 16 + |.endif + | //ZEND_VM_C_GOTO(assign_dim_op_ret_null); + | jmp >9 + |.code + | mov FCARG1a, r0 + } else { + if (!zend_jit_fetch_dimension_address_inner(Dst, opline, op_array, BP_VAR_W, op2_info, 8, 8)) { + return 0; + } + + |8: + | mov FCARG1a, r0 + } + + do { + uint32_t var_info = zend_array_element_type(op1_info, 0, 0); + + | // value = zend_assign_to_variable(variable_ptr, value, OP_DATA_TYPE); + | ZVAL_DEREF FCARG1a, var_info + if (!zend_jit_assign_to_variable(Dst, opline, op_array, ssa, FCARG1a, 0, var_info, (opline+1)->op1_type, (opline+1)->op1, val_info, + opline->result_type == IS_UNUSED ? -1 : opline->result.var)) { + return 0; + } + } while (0); + } + + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY)) { + if (op1_info & MAY_BE_ARRAY) { + |.cold_code + |7: + } + + if ((op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE)) && + (op1_info & MAY_BE_ARRAY)) { + if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { + if (op1_info & MAY_BE_REF) { + | cmp byte [FCARG1a + 8], IS_FALSE + } else { + | cmp byte [FP + opline->op1.var + 8], IS_FALSE + } + | jg >2 + } + | // ZVAL_NEW_ARR(container); + | // zend_hash_init(Z_ARRVAL_P(container), 8, NULL, ZVAL_PTR_DTOR, 0); + if (op1_info & MAY_BE_REF) { + | mov [r4], FCARG1a // save + } + || if (ZEND_DEBUG) { + || const char *filename = op_array->filename ? op_array->filename->val : NULL; + |.if X64 + | LOAD_ADDR CARG3, filename + | xor CARG4d, opline->lineno + |.else + | push opline->lineno + | push filename + |.endif + || } else { + |.if not X64 + | sub r4, 8 + |.endif + || } + |.if X64 + | LOAD_ADDR CARG2, 8 + || if (!(op1_info & MAY_BE_REF)) { + | LOAD_ZVAL_ADDR FCARG1a, opline->op1_type, opline->op1 + || } + |.else + | push 8 + || if (!(op1_info & MAY_BE_REF)) { + | LOAD_ZVAL_ADDR FCARG1a, opline->op1_type, opline->op1 + || } + | push FCARG1a + |.endif + | EXT_CALL _array_init, r0 + |.if not X64 + | add r4, 16 + |.endif + if (op1_info & MAY_BE_REF) { + | mov FCARG1a, [r4] // restore + | GET_Z_PTR FCARG1a, FCARG1a + } else { + | LONG_LOAD FCARG1a, opline->op1_type, opline->op1 + } + | // ZEND_VM_C_GOTO(assign_dim_op_new_array); + | jmp <6 + |2: } - } - if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) { - if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) { - |.cold_code - |3: - } - | SAVE_VALID_OPLINE opline - | LOAD_ZVAL_ADDR FCARG2a, opline->op2_type, opline->op2 - switch (type) { - case BP_VAR_R: - |.if X64 - | lea CARG3, [FP + opline->result.var] - |.else - | lea r0, [FP + opline->result.var] - | push r0 - |.endif - | EXT_CALL zend_jit_fetch_dim_r_helper, r0 - | jmp >9 - break; - case BP_VAR_IS: - case BP_VAR_UNSET: - |.if X64 - | lea CARG3, [FP + opline->result.var] - |.else - | lea r0, [FP + opline->result.var] - | push r0 - |.endif - | EXT_CALL zend_jit_fetch_dim_is_helper, r0 - | jmp >9 - break; - case BP_VAR_RW: - | EXT_CALL zend_jit_fetch_dim_rw_helper, r0 - | test r0, r0 - | jne >8 - | jmp >9 - break; - case BP_VAR_W: - | EXT_CALL zend_jit_fetch_dim_w_helper, r0 - | test r0, r0 - | jne >8 - | jmp >9 - break; - default: - ZEND_ASSERT(0); + if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { + | SAVE_VALID_OPLINE opline + if (!(op1_info & MAY_BE_REF)) { + | LOAD_ZVAL_ADDR FCARG1a, opline->op1_type, opline->op1 + } + if (opline->op2_type == IS_UNUSED) { + | xor FCARG2a, FCARG2a + } else { + | LOAD_ZVAL_ADDR FCARG2a, opline->op2_type, opline->op2 + } + |.if X64 + | LOAD_ZVAL_ADDR CARG3, (opline+1)->op1_type, (opline+1)->op1 + |.else + | PUSH_ZVAL_ADDR (opline+1)->op1_type, (opline+1)->op1, r0 + |.endif + | EXT_CALL zend_jit_assign_dim_helper, r0 } - if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) { + + if (op1_info & MAY_BE_ARRAY) { + | jmp >9 // END |.code } } + |9: + | FREE_OP opline->op2_type, opline->op2, op2_info, op_array, opline->lineno + | FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, op_array, opline->lineno + return 1; + +fallback: + return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); } static int zend_jit_assign_dim_math(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) @@ -3414,164 +3900,11 @@ static int zend_jit_jmpznz(dasm_State **Dst, const zend_op *opline, int b, zend_ return 1; } -static int zend_jit_simple_assign(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, uint32_t var, uint32_t var_info, zend_uchar val_type, znode_op val, uint32_t val_info, uint32_t var2, int in_cold) -{ - if (val_type == IS_CONST) { - zval *zv = RT_CONSTANT(op_array, val); - if (var2 == (uint32_t)-1) { - | ZVAL_COPY_CONST FP + var, var_info, zv, r0 - } else { - | ZVAL_COPY_CONST_2 FP + var, FP + var2, var_info, zv, r0 - } - || if (Z_REFCOUNTED_P(zv)) { -// TODO: += 2 - | ADDREF_CONST zv, r0 - || } - } else { - if (val_info & MAY_BE_UNDEF) { - if (in_cold) { - | IF_NOT_Z_TYPE FP + val.var, IS_UNDEF, >2 - } else { - | IF_Z_TYPE FP + val.var, IS_UNDEF, >1 - |.cold_code - |1: - } - | // zend_error(E_NOTICE, "Undefined variable: %s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); - | SAVE_VALID_OPLINE opline - | mov FCARG1d, val.var - | EXT_CALL zend_jit_undefined_op_helper, r0 - | SET_Z_TYPE_INFO FP + val.var, IS_NULL - | jmp >3 - if (in_cold) { - |2: - } else { - |.code - } - } - if (val_info & MAY_BE_REF) { - if (val_type == IS_CV) { - | lea r2, [FP + val.var] - | ZVAL_DEREF r2, val_info - if (var2 == (uint32_t)-1) { - | ZVAL_COPY_VALUE FP + var, r2, val_info, r0, eax, r1 - } else { - | ZVAL_COPY_VALUE_2 FP + var, FP + var2, r2, val_info, r0, eax, r1 - } - } else { - if (in_cold) { - | IF_NOT_Z_TYPE FP + val.var, IS_REFERENCE, >1 - } else { - | IF_Z_TYPE FP + val.var, IS_REFERENCE, >1 - |.cold_code - |1: - } - | // zend_refcounted *ref = Z_COUNTED_P(retval_ptr); - | GET_Z_PTR r0, FP + val.var - | // ZVAL_COPY_VALUE(return_value, &ref->value); - if (var2 == (uint32_t)-1) { - | ZVAL_COPY_VALUE FP + var, r0 + 8, val_info, r2, edx, r1 - } else { - | ZVAL_COPY_VALUE_2 FP + var, FP + var2, r0 + 8, val_info, r2, edx, r1 - } - | GC_DELREF r0 - | je >2 - | IF_NOT_REFCOUNTED dh, >3 - if (var2 == (uint32_t)-1) { - | GC_ADDREF r1 - } else { - | add dword [r1], 2 - } - | jmp >3 - |2: - if (var2 != (uint32_t)-1) { - | IF_NOT_REFCOUNTED dh, >2 - | GC_ADDREF r1 - |2: - } - | EFREE_SIZE r0, sizeof(zend_reference), op_array, opline - | jmp >3 - if (in_cold) { - |1: - } else { - |.code - } - if (var2 == (uint32_t)-1) { - | ZVAL_COPY_VALUE FP + var, FP + val.var, val_info, r0, eax, r1 - } else { - | ZVAL_COPY_VALUE_2 FP + var, FP + var2, FP + val.var, val_info, r0, eax, r1 - } - } - } else { - if (var2 == (uint32_t)-1) { - | ZVAL_COPY_VALUE FP + var, FP + val.var, val_info, r0, eax, r1 - } else { - | ZVAL_COPY_VALUE_2 FP + var, FP + var2, FP + val.var, val_info, r0, eax, r1 - } - } - if (val_type == IS_CV) { - if (var2 == (uint32_t)-1) { - | TRY_ADDREF val_info, ah, r1 - } else { - | TRY_ADDREF_2 val_info, ah, r1 - } - } else { - if (var2 != (uint32_t)-1) { - | TRY_ADDREF val_info, ah, r1 - } - } - |3: - } - return 1; -} - static int zend_jit_qm_assign(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) { uint32_t op1_info = OP1_INFO(); - return zend_jit_simple_assign(Dst, opline, op_array, ssa, opline->result.var, -1, opline->op1_type, opline->op1, op1_info, -1, 0); -} - -static int zend_jit_assign_to_variable(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, uint32_t var, uint32_t var_info, zend_uchar val_type, znode_op val, uint32_t val_info, uint32_t var2) -{ - if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { - int in_cold = 0; - - if (var_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { - | IF_Z_REFCOUNTED FP + var, >1 - |.cold_code - |1: - in_cold = 1; - } - | // TODO: support for object->set - | // TODO: support for assignment to itself - | GET_Z_PTR r0, FP + var - | GC_DELREF r0 - | jnz >4 - | mov aword [r4], r0 // save - if (!zend_jit_simple_assign(Dst, opline, op_array, ssa, var, var_info, val_type, val, val_info, var2, in_cold)) { - return 0; - } - | mov FCARG1a, aword [r4] // restore - | ZVAL_DTOR_FUNC (op_array->filename ? ZSTR_VAL(op_array->filename) : NULL), opline->lineno - | jmp >6 - |4: - | IF_NOT_Z_FLAGS FP + var, IS_TYPE_COLLECTABLE, >5 - | GET_Z_PTR FCARG1a, FP + var - | IF_GC_INFO FCARG1a, >5 - | EXT_CALL gc_possible_root, r0 - if (in_cold) { - | jmp >5 - |.code - } - |5: - } - - if (!zend_jit_simple_assign(Dst, opline, op_array, ssa, var, var_info, val_type, val, val_info, var2, 0)) { - return 0; - } - |6: - - return 1; + return zend_jit_simple_assign(Dst, opline, op_array, ssa, FP, opline->result.var, -1, opline->op1_type, opline->op1, op1_info, -1, 0); } static int zend_jit_assign(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) @@ -3586,10 +3919,11 @@ static int zend_jit_assign(dasm_State **Dst, const zend_op *opline, zend_op_arra op2_info = OP2_INFO(); if (op1_info & MAY_BE_REF) { + // TODO: support for references ??? goto fallback; } - if (!zend_jit_assign_to_variable(Dst, opline, op_array, ssa, opline->op1.var, op1_info, opline->op2_type, opline->op2, op2_info, + if (!zend_jit_assign_to_variable(Dst, opline, op_array, ssa, FP, opline->op1.var, op1_info, opline->op2_type, opline->op2, op2_info, opline->result_type == IS_UNUSED ? -1 : opline->result.var)) { return 0; } From 97ee3f4cc57aea4034b8cc60a50f2ab97e3b696f Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Thu, 29 Sep 2016 15:02:29 +0800 Subject: [PATCH 253/569] Add info to get_status about JIT --- ext/opcache/jit/zend_jit.c | 33 ++++++++++++++++++++------- ext/opcache/jit/zend_jit.h | 3 ++- ext/opcache/zend_accelerator_module.c | 7 ++++++ 3 files changed, 34 insertions(+), 9 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 25f2f0aac90d3..5e0af97e65eda 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -52,8 +52,8 @@ typedef struct _zend_jit_stub { {JIT_STUB_PREFIX #name, zend_jit_ ## name ## _stub} static void *dasm_buf = NULL; -static void *dasm_ptr = NULL; static void *dasm_end = NULL; +static void **dasm_ptr = NULL; static int zend_may_throw(const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa); static int zend_may_overflow(const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa); @@ -79,6 +79,20 @@ static int zend_may_overflow(const zend_op *opline, zend_op_array *op_array, zen #define DASM_ALIGNMENT 16 +ZEND_API void zend_jit_status(zval *ret) +{ + zval stats; + array_init(&stats); + if (dasm_buf) { + add_assoc_long(&stats, "buffer_size", (char*)dasm_end - (char*)dasm_buf); + add_assoc_long(&stats, "buffer_free", (char*)dasm_end - (char*)*dasm_ptr); + } else { + add_assoc_long(&stats, "buffer_size", 0); + add_assoc_long(&stats, "buffer_free", 0); + } + add_assoc_zval(ret, "jit", &stats); +} + static zend_string *zend_jit_func_name(zend_op_array *op_array) { smart_str buf = {0}; @@ -124,21 +138,21 @@ static void *dasm_link_and_encode(dasm_State **dasm_state, return NULL; } - if ((void*)((char*)dasm_ptr + size) > dasm_end) { - dasm_ptr = dasm_end; //prevent further try + if ((void*)((char*)*dasm_ptr + size) > dasm_end) { + *dasm_ptr = dasm_end; //prevent further try // TODO: jit_buffer_size overflow ??? return NULL; } - ret = dasm_encode(dasm_state, dasm_ptr); + ret = dasm_encode(dasm_state, *dasm_ptr); if (ret != DASM_S_OK) { // TODO: dasm_encode() failed ??? return NULL; } - entry = dasm_ptr; - dasm_ptr = (void*)((char*)dasm_ptr + ZEND_MM_ALIGNED_SIZE_EX(size, DASM_ALIGNMENT)); + entry = *dasm_ptr; + *dasm_ptr = (void*)((char*)*dasm_ptr + ZEND_MM_ALIGNED_SIZE_EX(size, DASM_ALIGNMENT)); if (op_array && cfg) { int b; @@ -1268,7 +1282,7 @@ static int zend_real_jit_func(zend_op_array *op_array, zend_script *script) zend_ssa ssa; void *checkpoint; - if (dasm_ptr == dasm_end) { + if (*dasm_ptr == dasm_end) { return FAILURE; } @@ -1375,7 +1389,7 @@ ZEND_API int zend_jit_script(zend_script *script) zend_func_info *info; int i; - if (dasm_ptr == dasm_end) { + if (dasm_ptr == NULL || *dasm_ptr == dasm_end) { return FAILURE; } @@ -1517,6 +1531,9 @@ ZEND_API int zend_jit_startup(size_t size) dasm_buf = dasm_ptr = buf; dasm_end = (void*)(((char*)dasm_buf) + size); + zend_jit_unprotect(); + *dasm_ptr = dasm_buf + sizeof(*dasm_ptr); + zend_jit_protect(); #ifdef HAVE_DISASM if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_ASM) { diff --git a/ext/opcache/jit/zend_jit.h b/ext/opcache/jit/zend_jit.h index 3e1bd920b7284..188736e093558 100644 --- a/ext/opcache/jit/zend_jit.h +++ b/ext/opcache/jit/zend_jit.h @@ -43,6 +43,7 @@ ZEND_API void zend_jit_unprotect(void); ZEND_API void zend_jit_protect(void); ZEND_API int zend_jit_startup(size_t size); ZEND_API void zend_jit_shutdown(void); +ZEND_API void zend_jit_status(zval *ret); #endif /* HAVE_JIT_H */ @@ -52,4 +53,4 @@ ZEND_API void zend_jit_shutdown(void); * c-basic-offset: 4 * indent-tabs-mode: t * End: - */ \ No newline at end of file + */ diff --git a/ext/opcache/zend_accelerator_module.c b/ext/opcache/zend_accelerator_module.c index 9b8863da28c9f..4ba0fee87b347 100644 --- a/ext/opcache/zend_accelerator_module.c +++ b/ext/opcache/zend_accelerator_module.c @@ -32,6 +32,10 @@ #include "ext/standard/info.h" #include "ext/standard/php_filestat.h" +#if HAVE_JIT +#include "jit/zend_jit.h" +#endif + #define STRING_NOT_NULL(s) (NULL == (s)?"":s) #define MIN_ACCEL_FILES 200 #define MAX_ACCEL_FILES 1000000 @@ -665,6 +669,9 @@ static ZEND_FUNCTION(opcache_get_status) add_assoc_zval(return_value, "scripts", &scripts); } } +#if HAVE_JIT + zend_jit_status(return_value); +#endif } static int add_blacklist_path(zend_blacklist_entry *p, zval *return_value) From c1b3cd7b7dc83a1cebf49d4f5ca55a80d90dbac1 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 29 Sep 2016 13:03:31 +0300 Subject: [PATCH 254/569] Fixed condition --- ext/opcache/jit/zend_jit_helpers.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_helpers.c b/ext/opcache/jit/zend_jit_helpers.c index 667695c2d1fc3..4e5f2f1643d4c 100644 --- a/ext/opcache/jit/zend_jit_helpers.c +++ b/ext/opcache/jit/zend_jit_helpers.c @@ -837,7 +837,7 @@ static void ZEND_FASTCALL zend_jit_assign_dim_helper(zval *object_ptr, zval *dim //??? } } } else if (EXPECTED(Z_TYPE_P(object_ptr) == IS_STRING)) { - if (dim) { + if (!dim) { zend_throw_error(NULL, "[] operator not supported for strings"); } else { zend_assign_to_string_offset(object_ptr, dim, value, /*???UNEXPECTED(RETURN_VALUE_USED(opline)) ? EX_VAR(opline->result.var) :*/ NULL); From 1bcde6e3b6a8b61d896524400e967c0847eda9d0 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 29 Sep 2016 15:54:02 +0300 Subject: [PATCH 255/569] Fixed few ASSIGN_DIM related bugs --- ext/opcache/jit/zend_jit_x86.dasc | 70 +++++++++++++++---------------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 9a84c1d8858c4..c9335b8c16b0e 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -2013,13 +2013,13 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o |.endif | jle >2 // NOT_FOUND | // _ret = &_ht->arData[_h].val; + | mov r0, FCARG2a |.if X64 - | shl FCARG2a, 5 + | shl r0, 5 |.else - | imul FCARG2a, sizeof(Bucket) + | imul r0, sizeof(Bucket) |.endif - | mov r0, aword [FCARG1a + offsetof(zend_array, arData)] - | add r0, FCARG2a + | add r0, aword [FCARG1a + offsetof(zend_array, arData)] | IF_NOT_Z_TYPE r0, IS_UNDEF, >8 | jmp >2 // NOT_FOUND } @@ -2499,23 +2499,13 @@ static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, zend_op_ } } else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE)) { if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { - if (op1_info & (MAY_BE_NULL|MAY_BE_FALSE)) { - if (op1_info & MAY_BE_REF) { - | cmp byte [FCARG1a + 8], IS_FALSE - } else { - | cmp byte [FP + opline->op1.var + 8], IS_FALSE - } - | jg >2 + if (op1_info & MAY_BE_REF) { + | cmp byte [FCARG1a + 8], IS_FALSE + } else { + | cmp byte [FP + opline->op1.var + 8], IS_FALSE } + | jg >7 } - if (op1_info & MAY_BE_UNDEF) { - | IF_NOT_Z_TYPE FP + opline->op1.var, IS_UNDEF, >1 - | SAVE_VALID_OPLINE opline - | mov FCARG1a, opline->op1.var - | EXT_CALL zend_jit_undefined_op_helper, r0 - |1: - } - |2: | // ZVAL_NEW_ARR(container); | // zend_hash_init(Z_ARRVAL_P(container), 8, NULL, ZVAL_PTR_DTOR, 0); if (op1_info & MAY_BE_REF) { @@ -2606,7 +2596,14 @@ static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, zend_op_ | mov FCARG1a, r0 } - do { + if (opline->op2_type == IS_UNUSED) { + uint32_t var_info = zend_array_element_type(op1_info, 0, 0); + + if (!zend_jit_simple_assign(Dst, opline, op_array, ssa, FCARG1a, 0, var_info, (opline+1)->op1_type, (opline+1)->op1, val_info, + opline->result_type == IS_UNUSED ? -1 : opline->result.var, 0)) { + return 0; + } + } else { uint32_t var_info = zend_array_element_type(op1_info, 0, 0); | // value = zend_assign_to_variable(variable_ptr, value, OP_DATA_TYPE); @@ -2615,11 +2612,13 @@ static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, zend_op_ opline->result_type == IS_UNUSED ? -1 : opline->result.var)) { return 0; } - } while (0); + } } - if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY)) { - if (op1_info & MAY_BE_ARRAY) { + if (((op1_info & MAY_BE_ARRAY) && + (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE))) || + (op1_info & (MAY_BE_ANY-(MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)))) { + if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { |.cold_code |7: } @@ -2696,6 +2695,7 @@ static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, zend_op_ | PUSH_ZVAL_ADDR (opline+1)->op1_type, (opline+1)->op1, r0 |.endif | EXT_CALL zend_jit_assign_dim_helper, r0 + | FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, op_array, opline->lineno } if (op1_info & MAY_BE_ARRAY) { @@ -2706,7 +2706,6 @@ static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, zend_op_ |9: | FREE_OP opline->op2_type, opline->op2, op2_info, op_array, opline->lineno - | FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, op_array, opline->lineno return 1; @@ -2776,23 +2775,22 @@ static int zend_jit_assign_dim_math(dasm_State **Dst, const zend_op *opline, zen } } else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE)) { if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { - if (op1_info & (MAY_BE_NULL|MAY_BE_FALSE)) { - if (op1_info & MAY_BE_REF) { - | cmp byte [FCARG1a + 8], IS_FALSE - } else { - | cmp byte [FP + opline->op1.var + 8], IS_FALSE - } - | jg >2 + if (op1_info & MAY_BE_REF) { + | cmp byte [FCARG1a + 8], IS_FALSE + } else { + | cmp byte [FP + opline->op1.var + 8], IS_FALSE } + | jg >7 } if (op1_info & MAY_BE_UNDEF) { - | IF_NOT_Z_TYPE FP + opline->op1.var, IS_UNDEF, >1 + if (op1_info & (MAY_BE_NULL|MAY_BE_FALSE)) { + | IF_NOT_Z_TYPE FP + opline->op1.var, IS_UNDEF, >1 + } | SAVE_VALID_OPLINE opline | mov FCARG1a, opline->op1.var | EXT_CALL zend_jit_undefined_op_helper, r0 |1: } - |2: | // ZVAL_NEW_ARR(container); | // zend_hash_init(Z_ARRVAL_P(container), 8, NULL, ZVAL_PTR_DTOR, 0); if (op1_info & MAY_BE_REF) { @@ -2909,8 +2907,10 @@ static int zend_jit_assign_dim_math(dasm_State **Dst, const zend_op *opline, zen } } - if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY)) { - if (op1_info & MAY_BE_ARRAY) { + if (((op1_info & MAY_BE_ARRAY) && + (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE))) || + (op1_info & (MAY_BE_ANY-(MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)))) { + if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { |.cold_code |7: } From ef41317168d485c6d78ca0f42ba94e09b8224fcf Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 29 Sep 2016 16:44:58 +0300 Subject: [PATCH 256/569] Fixed operands order --- ext/opcache/jit/zend_jit_x86.dasc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index c9335b8c16b0e..266ca6f36c2ec 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -2419,7 +2419,7 @@ static int zend_jit_assign_to_variable(dasm_State **Dst, } | EXT_CALL gc_possible_root, r0 if (var_reg != FP) { - | mov [r4], Ra(var_reg) // restore + | mov Ra(var_reg), [r4] // restore } if (in_cold) { | jmp >5 From cf2f53914ef2720e6add65f5c0e1fdfb67939a60 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 29 Sep 2016 17:34:33 +0300 Subject: [PATCH 257/569] Fixed incorrect code placement --- ext/opcache/jit/zend_jit_x86.dasc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 266ca6f36c2ec..6ed9936f4e8db 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -2698,7 +2698,7 @@ static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, zend_op_ | FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, op_array, opline->lineno } - if (op1_info & MAY_BE_ARRAY) { + if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { | jmp >9 // END |.code } @@ -3013,7 +3013,7 @@ static int zend_jit_assign_dim_math(dasm_State **Dst, const zend_op *opline, zen } } - if (op1_info & MAY_BE_ARRAY) { + if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { | jmp >9 // END |.code } From 5174435ea6bafb327ddcfa8324d3bd906e11af93 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 29 Sep 2016 20:29:33 +0300 Subject: [PATCH 258/569] Implemented JIT for ASSIGN_DIM --- ext/opcache/jit/zend_jit.c | 2 - ext/opcache/jit/zend_jit_x86.dasc | 154 +++++++++++++++++++++++++----- 2 files changed, 131 insertions(+), 25 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 5e0af97e65eda..c2b5ef7e6809e 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -1014,13 +1014,11 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) goto jit_failure; } break; -#if 0 case ZEND_ASSIGN_DIM: if (!zend_jit_assign_dim(&dasm_state, opline, op_array, ssa)) { goto jit_failure; } break; -#endif case ZEND_ASSIGN: if (!zend_jit_assign(&dasm_state, opline, op_array, ssa)) { goto jit_failure; diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 6ed9936f4e8db..11b3b1270b4fc 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -598,6 +598,42 @@ static void* dasm_labels[zend_lb_MAX]; ||} |.endmacro +/* the same as above, but "src" may overlap with "tmp_reg1d" */ +|.macro ZVAL_COPY_VALUE_clobber_src, dst, src, src_info, tmp_reg1, tmp_reg1d, tmp_reg2 +||if (src_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) { +|| if (!(src_info & MAY_BE_DOUBLE)) { +| GET_Z_PTR tmp_reg2, src +| SET_Z_PTR dst, tmp_reg2 +|| } else if ((src_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { +|.if X64 or SSE +| movsd xmm0, qword [src] +| movsd qword [dst], xmm0 +|.else +| fld qword [src] +| fstp qword [dst] +|.endif +|| } else { +|.if X64 +| GET_Z_PTR tmp_reg2, src +| SET_Z_PTR dst, tmp_reg2 +|.else +| GET_Z_W2 tmp_reg2, src +| SET_Z_W2 dst, tmp_reg2 +| GET_Z_PTR tmp_reg2, src +| SET_Z_PTR dst, tmp_reg2 +|.endif +|| } +||} +||if ((src_info & (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE)) && +|| has_concrete_type(src_info & MAY_BE_ANY)) { +|| zend_uchar type = concrete_type(src_info); +| SET_Z_TYPE_INFO dst, type +||} else { +| GET_Z_TYPE_INFO tmp_reg1d, src +| SET_Z_TYPE_INFO dst, tmp_reg1d +||} +|.endmacro + |.macro ZVAL_COPY_VALUE_2, dst, dst2, src, src_info, tmp_reg1, tmp_reg1d, tmp_reg2 ||if (src_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) { || if (!(src_info & MAY_BE_DOUBLE)) { @@ -642,6 +678,51 @@ static void* dasm_labels[zend_lb_MAX]; ||} |.endmacro +/* the same as above, but "src" may overlap with "tmp_reg1d" */ +|.macro ZVAL_COPY_VALUE_clobber_src_2, dst, dst2, src, src_info, tmp_reg1, tmp_reg1d, tmp_reg2 +||if (src_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) { +|| if (!(src_info & MAY_BE_DOUBLE)) { +| GET_Z_PTR tmp_reg2, src +| SET_Z_PTR dst, tmp_reg2 +| SET_Z_PTR dst2, tmp_reg2 +|| } else if ((src_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { +|.if X64 or SSE +| movsd xmm0, qword [src] +| movsd qword [dst], xmm0 +| movsd qword [dst2], xmm0 +|.else +| fld qword [src] +| fstp qword [dst] +| fld qword [src] +| fstp qword [dst2] +|.endif +|| } else { +|.if X64 +| GET_Z_PTR tmp_reg2, src +| SET_Z_PTR dst, tmp_reg2 +| SET_Z_PTR dst2, tmp_reg2 +|.else +| GET_Z_W2 tmp_reg2, src +| SET_Z_W2 dst, tmp_reg2 +| SET_Z_W2 dst2, tmp_reg2 +| GET_Z_PTR tmp_reg2, src +| SET_Z_PTR dst, tmp_reg2 +| SET_Z_PTR dst2, tmp_reg2 +|.endif +|| } +||} +||if ((src_info & (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE)) && +|| has_concrete_type(src_info & MAY_BE_ANY)) { +|| zend_uchar type = concrete_type(src_info); +| SET_Z_TYPE_INFO dst, type +| SET_Z_TYPE_INFO dst2, type +||} else { +| GET_Z_TYPE_INFO tmp_reg1d, src +| SET_Z_TYPE_INFO dst, tmp_reg1d +| SET_Z_TYPE_INFO dst2, tmp_reg1d +||} +|.endmacro + |.macro IF_TYPE, type, val, label | cmp type, val | je label @@ -725,6 +806,19 @@ static void* dasm_labels[zend_lb_MAX]; |.endif |.endmacro +|.macro ADDREF_CONST_2, zv, tmp_reg +|.if X64 +|| if (!IS_SIGNED_32BIT(Z_LVAL_P(zv))) { +| mov64 tmp_reg, ((uintptr_t)Z_LVAL_P(zv)) +| add dword [tmp_reg], 2 +|| } else { +| add dword [Z_LVAL_P(zv)], 2 +|| } +|.else +| inc dword [Z_LVAL_P(zv)] +|.endif +|.endmacro + |.macro TRY_ADDREF, val_info, type_flags_reg, value_ptr_reg ||if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { || if (val_info & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { @@ -2259,6 +2353,7 @@ static int zend_jit_simple_assign(dasm_State **Dst, int in_cold) /* Labels: 1,2,3 */ { + ZEND_ASSERT(var_reg != r0); if (val_type == IS_CONST) { zval *zv = RT_CONSTANT(op_array, val); if (var2 == (uint32_t)-1) { @@ -2266,10 +2361,13 @@ static int zend_jit_simple_assign(dasm_State **Dst, } else { | ZVAL_COPY_CONST_2 Ra(var_reg)+var_offset, FP + var2, var_info, zv, r0 } - || if (Z_REFCOUNTED_P(zv)) { -// TODO: += 2 - | ADDREF_CONST zv, r0 - || } + if (Z_REFCOUNTED_P(zv)) { + if (var2 == (uint32_t)-1) { + | ADDREF_CONST zv, r0 + } else { + | ADDREF_CONST_2 zv, r0 + } + } } else { if (val_info & MAY_BE_UNDEF) { if (in_cold) { @@ -2280,11 +2378,19 @@ static int zend_jit_simple_assign(dasm_State **Dst, |1: } | // zend_error(E_NOTICE, "Undefined variable: %s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); + if (var_reg != FP) { + | mov aword [r4], Ra(var_reg) // save + } | SAVE_VALID_OPLINE opline | mov FCARG1d, val.var | EXT_CALL zend_jit_undefined_op_helper, r0 - | SET_Z_TYPE_INFO FP + val.var, IS_NULL - | // ???? val or var ^ + if (var_reg != FP) { + | mov Ra(var_reg), aword [r4] // restore + } + | SET_Z_TYPE_INFO Ra(var_reg)+var_offset, IS_NULL + if (var2 != (uint32_t)-1) { + | SET_Z_TYPE_INFO FP + var2, IS_NULL + } | jmp >3 if (in_cold) { |2: @@ -2294,12 +2400,13 @@ static int zend_jit_simple_assign(dasm_State **Dst, } if (val_info & MAY_BE_REF) { if (val_type == IS_CV) { + ZEND_ASSERT(var_reg != r2); | lea r2, [FP + val.var] | ZVAL_DEREF r2, val_info if (var2 == (uint32_t)-1) { - | ZVAL_COPY_VALUE Ra(var_reg)+var_offset, r2, val_info, r0, eax, r1 + | ZVAL_COPY_VALUE_clobber_src Ra(var_reg)+var_offset, r2, val_info, r2, edx, r0 } else { - | ZVAL_COPY_VALUE_2 Ra(var_reg)+var_offset, FP + var2, r2, val_info, r0, eax, r1 + | ZVAL_COPY_VALUE_clobber_src_2 Ra(var_reg)+var_offset, FP + var2, r2, val_info, r2, edx, r0 } } else { if (in_cold) { @@ -2310,29 +2417,29 @@ static int zend_jit_simple_assign(dasm_State **Dst, |1: } | // zend_refcounted *ref = Z_COUNTED_P(retval_ptr); - | GET_Z_PTR r0, FP + val.var + | GET_Z_PTR r2, FP + val.var + | GC_DELREF r2 | // ZVAL_COPY_VALUE(return_value, &ref->value); if (var2 == (uint32_t)-1) { - | ZVAL_COPY_VALUE Ra(var_reg)+var_offset, r0 + 8, val_info, r2, edx, r1 + | ZVAL_COPY_VALUE_clobber_src Ra(var_reg)+var_offset, r2 + 8, val_info, r2, edx, r0 } else { - | ZVAL_COPY_VALUE_2 Ra(var_reg)+var_offset, FP + var2, r0 + 8, val_info, r2, edx, r1 + | ZVAL_COPY_VALUE_clobber_src_2 Ra(var_reg)+var_offset, FP + var2, r2 + 8, val_info, r2, edx, r0 } - | GC_DELREF r0 | je >2 | IF_NOT_REFCOUNTED dh, >3 if (var2 == (uint32_t)-1) { - | GC_ADDREF r1 + | GC_ADDREF r0 } else { - | add dword [r1], 2 + | add dword [r0], 2 } | jmp >3 |2: if (var2 != (uint32_t)-1) { | IF_NOT_REFCOUNTED dh, >2 - | GC_ADDREF r1 + | GC_ADDREF r0 |2: } - | EFREE_SIZE r0, sizeof(zend_reference), op_array, opline + | EFREE_SIZE aword [FP + val.var], sizeof(zend_reference), op_array, opline | jmp >3 if (in_cold) { |1: @@ -2340,27 +2447,27 @@ static int zend_jit_simple_assign(dasm_State **Dst, |.code } if (var2 == (uint32_t)-1) { - | ZVAL_COPY_VALUE Ra(var_reg)+var_offset, FP + val.var, val_info, r0, eax, r1 + | ZVAL_COPY_VALUE Ra(var_reg)+var_offset, FP + val.var, val_info, r2, edx, r0 } else { - | ZVAL_COPY_VALUE_2 Ra(var_reg)+var_offset, FP + var2, FP + val.var, val_info, r0, eax, r1 + | ZVAL_COPY_VALUE_2 Ra(var_reg)+var_offset, FP + var2, FP + val.var, val_info, r2, edx, r0 } } } else { if (var2 == (uint32_t)-1) { - | ZVAL_COPY_VALUE Ra(var_reg)+var_offset, FP + val.var, val_info, r0, eax, r1 + | ZVAL_COPY_VALUE Ra(var_reg)+var_offset, FP + val.var, val_info, r2, edx, r0 } else { - | ZVAL_COPY_VALUE_2 Ra(var_reg)+var_offset, FP + var2, FP + val.var, val_info, r0, eax, r1 + | ZVAL_COPY_VALUE_2 Ra(var_reg)+var_offset, FP + var2, FP + val.var, val_info, r2, edx, r0 } } if (val_type == IS_CV) { if (var2 == (uint32_t)-1) { - | TRY_ADDREF val_info, ah, r1 + | TRY_ADDREF val_info, dh, r0 } else { - | TRY_ADDREF_2 val_info, ah, r1 + | TRY_ADDREF_2 val_info, dh, r0 } } else { if (var2 != (uint32_t)-1) { - | TRY_ADDREF val_info, ah, r1 + | TRY_ADDREF val_info, dh, r0 } } |3: @@ -2384,6 +2491,7 @@ static int zend_jit_assign_to_variable(dasm_State **Dst, if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { int in_cold = 0; + ZEND_ASSERT(var_reg != r0); if (var_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { | IF_Z_REFCOUNTED Ra(var_reg)+var_offset, >1 |.cold_code From e248b47b6fb8530ae429a4aa8c0a34f77e281491 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 29 Sep 2016 22:56:30 +0300 Subject: [PATCH 259/569] Added JIT support for few more cases --- ext/opcache/jit/zend_jit_helpers.c | 16 +++++------ ext/opcache/jit/zend_jit_x86.dasc | 44 ++++++++++++++++++++++++------ 2 files changed, 44 insertions(+), 16 deletions(-) diff --git a/ext/opcache/jit/zend_jit_helpers.c b/ext/opcache/jit/zend_jit_helpers.c index 4e5f2f1643d4c..0ade38df3620e 100644 --- a/ext/opcache/jit/zend_jit_helpers.c +++ b/ext/opcache/jit/zend_jit_helpers.c @@ -824,7 +824,7 @@ static zend_never_inline void zend_assign_to_string_offset(zval *str, zval *dim, } } -static void ZEND_FASTCALL zend_jit_assign_dim_helper(zval *object_ptr, zval *dim, zval *value) +static void ZEND_FASTCALL zend_jit_assign_dim_helper(zval *object_ptr, zval *dim, zval *value, zval *result) { if (EXPECTED(Z_TYPE_P(object_ptr) == IS_OBJECT)) { @@ -832,23 +832,23 @@ static void ZEND_FASTCALL zend_jit_assign_dim_helper(zval *object_ptr, zval *dim zend_throw_error(NULL, "Cannot use object as array"); } else { Z_OBJ_HT_P(object_ptr)->write_dimension(object_ptr, dim, value); -//??? if (UNEXPECTED(RETURN_VALUE_USED(opline)) && EXPECTED(!EG(exception))) { -//??? ZVAL_COPY(EX_VAR(opline->result.var), value); -//??? } + if (result && EXPECTED(!EG(exception))) { + ZVAL_COPY(result, value); + } } } else if (EXPECTED(Z_TYPE_P(object_ptr) == IS_STRING)) { if (!dim) { zend_throw_error(NULL, "[] operator not supported for strings"); } else { - zend_assign_to_string_offset(object_ptr, dim, value, /*???UNEXPECTED(RETURN_VALUE_USED(opline)) ? EX_VAR(opline->result.var) :*/ NULL); + zend_assign_to_string_offset(object_ptr, dim, value, result); } } else { //??? if (OP1_TYPE != IS_VAR || EXPECTED(!Z_ISERROR_P(object_ptr))) { zend_error(E_WARNING, "Cannot use a scalar value as an array"); //??? } -//??? if (UNEXPECTED(RETURN_VALUE_USED(opline))) { -//??? ZVAL_NULL(EX_VAR(opline->result.var)); -//??? } + if (result) { + ZVAL_NULL(result); + } } } diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 11b3b1270b4fc..47a6712668d09 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -2548,7 +2548,7 @@ static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, zend_op_ { uint32_t op1_info, op2_info, val_info; - if (opline->op1_type != IS_CV || opline->result_type != IS_UNUSED) { + if (opline->op1_type != IS_CV) { goto fallback; } @@ -2691,6 +2691,9 @@ static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, zend_op_ | EXT_CALL zend_error, r0 | add r4, 16 |.endif + if (opline->result_type != IS_UNUSED) { + | SET_Z_TYPE_INFO FP + opline->result.var, IS_NULL + } | //ZEND_VM_C_GOTO(assign_dim_op_ret_null); | jmp >9 |.code @@ -2725,7 +2728,7 @@ static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, zend_op_ if (((op1_info & MAY_BE_ARRAY) && (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE))) || - (op1_info & (MAY_BE_ANY-(MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)))) { + (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)))) { if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { |.cold_code |7: @@ -2797,6 +2800,19 @@ static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, zend_op_ } else { | LOAD_ZVAL_ADDR FCARG2a, opline->op2_type, opline->op2 } + if (opline->result_type == IS_UNUSED) { + |.if X64 + | xor CARG4, CARG4 + |.else + | push 0 + |.endif + } else { + |.if X64 + | LOAD_ZVAL_ADDR CARG4, opline->result_type, opline->result + |.else + | PUSH_ZVAL_ADDR opline->result_type, opline->result, r0 + |.endif + } |.if X64 | LOAD_ZVAL_ADDR CARG3, (opline+1)->op1_type, (opline+1)->op1 |.else @@ -2815,6 +2831,10 @@ static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, zend_op_ |9: | FREE_OP opline->op2_type, opline->op2, op2_info, op_array, opline->lineno + if (zend_may_throw(opline, op_array, ssa)) { + zend_jit_check_exception(Dst); + } + return 1; fallback: @@ -3017,7 +3037,7 @@ static int zend_jit_assign_dim_math(dasm_State **Dst, const zend_op *opline, zen if (((op1_info & MAY_BE_ARRAY) && (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE))) || - (op1_info & (MAY_BE_ANY-(MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)))) { + (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)))) { if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { |.cold_code |7: @@ -4027,13 +4047,21 @@ static int zend_jit_assign(dasm_State **Dst, const zend_op *opline, zend_op_arra op2_info = OP2_INFO(); if (op1_info & MAY_BE_REF) { - // TODO: support for references ??? - goto fallback; + | lea FCARG1a, [FP + opline->op1.var] + | ZVAL_DEREF FCARG1a, op1_info + if (!zend_jit_assign_to_variable(Dst, opline, op_array, ssa, FCARG1a, 0, op1_info, opline->op2_type, opline->op2, op2_info, + opline->result_type == IS_UNUSED ? -1 : opline->result.var)) { + return 0; + } + } else { + if (!zend_jit_assign_to_variable(Dst, opline, op_array, ssa, FP, opline->op1.var, op1_info, opline->op2_type, opline->op2, op2_info, + opline->result_type == IS_UNUSED ? -1 : opline->result.var)) { + return 0; + } } - if (!zend_jit_assign_to_variable(Dst, opline, op_array, ssa, FP, opline->op1.var, op1_info, opline->op2_type, opline->op2, op2_info, - opline->result_type == IS_UNUSED ? -1 : opline->result.var)) { - return 0; + if (zend_may_throw(opline, op_array, ssa)) { + zend_jit_check_exception(Dst); } return 1; From 7dce00b903b44d5b15b9e553633d544b0dd385d5 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 30 Sep 2016 01:57:15 +0300 Subject: [PATCH 260/569] Improved code layout and removed dead jumps --- ext/opcache/Optimizer/zend_inference.c | 12 +-- ext/opcache/jit/zend_jit_x86.dasc | 110 ++++++++++++++----------- 2 files changed, 69 insertions(+), 53 deletions(-) diff --git a/ext/opcache/Optimizer/zend_inference.c b/ext/opcache/Optimizer/zend_inference.c index 331112f387401..bea0ac150ac8b 100644 --- a/ext/opcache/Optimizer/zend_inference.c +++ b/ext/opcache/Optimizer/zend_inference.c @@ -2181,7 +2181,7 @@ uint32_t zend_array_element_type(uint32_t t1, int write, int insert) } if (t1 & MAY_BE_ARRAY) { if (insert) { - tmp |= MAY_BE_NULL | MAY_BE_RCN; + tmp |= MAY_BE_NULL; } else { tmp |= MAY_BE_NULL | ((t1 & MAY_BE_ARRAY_OF_ANY) >> MAY_BE_ARRAY_SHIFT); if (tmp & MAY_BE_ARRAY) { @@ -2201,7 +2201,7 @@ uint32_t zend_array_element_type(uint32_t t1, int write, int insert) } } if (t1 & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE)) { - tmp |= MAY_BE_NULL | MAY_BE_RCN; + tmp |= MAY_BE_NULL; if (t1 & MAY_BE_ERROR) { if (write) { tmp |= MAY_BE_ERROR; @@ -2209,7 +2209,7 @@ uint32_t zend_array_element_type(uint32_t t1, int write, int insert) } } if (t1 & (MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE|MAY_BE_RESOURCE)) { - tmp |= MAY_BE_NULL | MAY_BE_RCN; + tmp |= MAY_BE_NULL; if (write) { tmp |= MAY_BE_ERROR; } @@ -2222,8 +2222,8 @@ static uint32_t assign_dim_result_type( uint32_t tmp = arr_type & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_ARRAY_KEY_ANY|MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF) & ~(MAY_BE_NULL|MAY_BE_FALSE); - if (arr_type & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_STRING)) { - tmp |= MAY_BE_ARRAY; + if (arr_type & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE)) { + tmp |= MAY_BE_ARRAY|MAY_BE_RC1; } if (arr_type & (MAY_BE_RC1|MAY_BE_RCN)) { tmp |= MAY_BE_RC1; @@ -4106,7 +4106,7 @@ int zend_ssa_inference(zend_arena **arena, const zend_op_array *op_array, const } } else { for (i = 0; i < op_array->last_var; i++) { - ssa_var_info[i].type = MAY_BE_UNDEF | MAY_BE_RCN; + ssa_var_info[i].type = MAY_BE_UNDEF; ssa_var_info[i].has_range = 0; } } diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 47a6712668d09..1b16fd45b0693 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -2064,7 +2064,7 @@ fallback: return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); } -static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, uint32_t type, uint32_t op2_info, uint32_t found, uint32_t not_found) +static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, uint32_t type, uint32_t op1_info, uint32_t op2_info, uint32_t found, uint32_t not_found) /* Labels: 1,2,3 */ { if (op2_info & MAY_BE_LONG) { @@ -2074,54 +2074,64 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o } | // hval = Z_LVAL_P(dim); | LONG_LOAD FCARG2a, opline->op2_type, opline->op2 - if (opline->op2_type == IS_CONST) { - zend_long val = Z_LVAL_P(RT_CONSTANT(op_array, opline->op2)); - if (val >= 0 && val < HT_MAX_SIZE) { + if (op1_info & MAY_BE_ARRAY_KEY_LONG) { + if (opline->op2_type == IS_CONST) { + zend_long val = Z_LVAL_P(RT_CONSTANT(op_array, opline->op2)); + if (val >= 0 && val < HT_MAX_SIZE) { + | // ZEND_HASH_INDEX_FIND(ht, hval, retval, num_undef); + | test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED + | jz >1 // HASH_FIND + | // if (EXPECTED((zend_ulong)(_h) < (zend_ulong)(_ht)->nNumUsed)) + |.if X64 + | movsxd r0, dword [FCARG1a + offsetof(zend_array, nNumUsed)] + | cmp r0, val + |.else + | cmp dword [FCARG1a + offsetof(zend_array, nNumUsed)], val + |.endif + | jle >2 // NOT_FOUND + | // _ret = &_ht->arData[_h].val; + | mov r0, aword [FCARG1a + offsetof(zend_array, arData)] + | add r0, val * sizeof(Bucket) + | IF_NOT_Z_TYPE r0, IS_UNDEF, >8 + } + } else { | // ZEND_HASH_INDEX_FIND(ht, hval, retval, num_undef); | test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED | jz >1 // HASH_FIND | // if (EXPECTED((zend_ulong)(_h) < (zend_ulong)(_ht)->nNumUsed)) |.if X64 | movsxd r0, dword [FCARG1a + offsetof(zend_array, nNumUsed)] - | cmp r0, val + | cmp r0, FCARG2a |.else - | cmp dword [FCARG1a + offsetof(zend_array, nNumUsed)], val + | cmp dword [FCARG1a + offsetof(zend_array, nNumUsed)], FCARG2a |.endif | jle >2 // NOT_FOUND | // _ret = &_ht->arData[_h].val; - | mov r0, aword [FCARG1a + offsetof(zend_array, arData)] - | add r0, val * sizeof(Bucket) + | mov r0, FCARG2a + |.if X64 + | shl r0, 5 + |.else + | imul r0, sizeof(Bucket) + |.endif + | add r0, aword [FCARG1a + offsetof(zend_array, arData)] | IF_NOT_Z_TYPE r0, IS_UNDEF, >8 - | jmp >2 // NOT_FOUND } - } else { - | // ZEND_HASH_INDEX_FIND(ht, hval, retval, num_undef); - | test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED - | jz >1 // HASH_FIND - | // if (EXPECTED((zend_ulong)(_h) < (zend_ulong)(_ht)->nNumUsed)) - |.if X64 - | movsxd r0, dword [FCARG1a + offsetof(zend_array, nNumUsed)] - | cmp r0, FCARG2a - |.else - | cmp dword [FCARG1a + offsetof(zend_array, nNumUsed)], FCARG2a - |.endif - | jle >2 // NOT_FOUND - | // _ret = &_ht->arData[_h].val; - | mov r0, FCARG2a - |.if X64 - | shl r0, 5 - |.else - | imul r0, sizeof(Bucket) - |.endif - | add r0, aword [FCARG1a + offsetof(zend_array, arData)] - | IF_NOT_Z_TYPE r0, IS_UNDEF, >8 - | jmp >2 // NOT_FOUND } - |1: switch (type) { case BP_VAR_R: case BP_VAR_IS: case BP_VAR_UNSET: + if (op1_info & MAY_BE_ARRAY_KEY_LONG) { + if (opline->op2_type == IS_CONST) { + zend_long val = Z_LVAL_P(RT_CONSTANT(op_array, opline->op2)); + if (val >= 0 && val < HT_MAX_SIZE) { + | jmp >2 // NOT_FOUND + } + } else { + | jmp >2 // NOT_FOUND + } + |1: + } | EXT_CALL zend_hash_index_find, r0 | test r0, r0 | jz >2 // NOT_FOUND @@ -2158,21 +2168,19 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o |.code break; case BP_VAR_RW: - | SAVE_VALID_OPLINE opline - | EXT_CALL zend_jit_hash_index_lookup_rw, r0 - |.cold_code |2: | SAVE_VALID_OPLINE opline | // zend_error(E_NOTICE,"Undefined offset: " ZEND_LONG_FMT, hval); | //retval = zend_hash_index_update(ht, hval, &EG(uninitialized_zval)); | EXT_CALL zend_jit_fetch_dimension_rw_long_helper, r0 - | jmp >8 - |.code + if (op1_info & MAY_BE_ARRAY_KEY_LONG) { + | jmp >8 + |1: + | SAVE_VALID_OPLINE opline + | EXT_CALL zend_jit_hash_index_lookup_rw, r0 + } break; case BP_VAR_W: - | SAVE_VALID_OPLINE opline - | EXT_CALL zend_jit_hash_index_lookup_w, r0 - |.cold_code |2: | //retval = zend_hash_index_add_new(ht, hval, &EG(uninitialized_zval)); || if (ZEND_DEBUG) { @@ -2191,8 +2199,12 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o | PUSH_ADDR &EG(uninitialized_zval), r0 |.endif | EXT_CALL _zend_hash_index_add_new, r0 - | jmp >8 - |.code + if (op1_info & MAY_BE_ARRAY_KEY_LONG) { + | jmp >8 + |1: + | SAVE_VALID_OPLINE opline + | EXT_CALL zend_jit_hash_index_lookup_w, r0 + } break; default: ZEND_ASSERT(0); @@ -2699,7 +2711,7 @@ static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, zend_op_ |.code | mov FCARG1a, r0 } else { - if (!zend_jit_fetch_dimension_address_inner(Dst, opline, op_array, BP_VAR_W, op2_info, 8, 8)) { + if (!zend_jit_fetch_dimension_address_inner(Dst, opline, op_array, BP_VAR_W, op1_info, op2_info, 8, 8)) { return 0; } @@ -2823,7 +2835,9 @@ static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, zend_op_ } if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { - | jmp >9 // END + if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { + | jmp >9 // END + } |.code } } @@ -3001,7 +3015,7 @@ static int zend_jit_assign_dim_math(dasm_State **Dst, const zend_op *opline, zen |.code | mov FCARG1a, r0 } else { - if (!zend_jit_fetch_dimension_address_inner(Dst, opline, op_array, BP_VAR_RW, op2_info, 8, 8)) { + if (!zend_jit_fetch_dimension_address_inner(Dst, opline, op_array, BP_VAR_RW, op1_info, op2_info, 8, 8)) { return 0; } @@ -3142,7 +3156,9 @@ static int zend_jit_assign_dim_math(dasm_State **Dst, const zend_op *opline, zen } if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { - | jmp >9 // END + if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { + | jmp >9 // END + } |.code } } @@ -4740,7 +4756,7 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, const zend_op *opline, zend } | LONG_LOAD FCARG1a, opline->op1_type, opline->op1 } - if (!zend_jit_fetch_dimension_address_inner(Dst, opline, op_array, (opline->opcode == ZEND_FETCH_DIM_R) ? BP_VAR_R : BP_VAR_IS, op2_info, 8, 9)) { + if (!zend_jit_fetch_dimension_address_inner(Dst, opline, op_array, (opline->opcode == ZEND_FETCH_DIM_R) ? BP_VAR_R : BP_VAR_IS, op1_info, op2_info, 8, 9)) { return 0; } } From db3fb4876e15804216bd75ceb36fd052aff90388 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 30 Sep 2016 02:44:39 +0300 Subject: [PATCH 261/569] Reverted changes committed by mistake --- ext/opcache/Optimizer/zend_inference.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ext/opcache/Optimizer/zend_inference.c b/ext/opcache/Optimizer/zend_inference.c index bea0ac150ac8b..331112f387401 100644 --- a/ext/opcache/Optimizer/zend_inference.c +++ b/ext/opcache/Optimizer/zend_inference.c @@ -2181,7 +2181,7 @@ uint32_t zend_array_element_type(uint32_t t1, int write, int insert) } if (t1 & MAY_BE_ARRAY) { if (insert) { - tmp |= MAY_BE_NULL; + tmp |= MAY_BE_NULL | MAY_BE_RCN; } else { tmp |= MAY_BE_NULL | ((t1 & MAY_BE_ARRAY_OF_ANY) >> MAY_BE_ARRAY_SHIFT); if (tmp & MAY_BE_ARRAY) { @@ -2201,7 +2201,7 @@ uint32_t zend_array_element_type(uint32_t t1, int write, int insert) } } if (t1 & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE)) { - tmp |= MAY_BE_NULL; + tmp |= MAY_BE_NULL | MAY_BE_RCN; if (t1 & MAY_BE_ERROR) { if (write) { tmp |= MAY_BE_ERROR; @@ -2209,7 +2209,7 @@ uint32_t zend_array_element_type(uint32_t t1, int write, int insert) } } if (t1 & (MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE|MAY_BE_RESOURCE)) { - tmp |= MAY_BE_NULL; + tmp |= MAY_BE_NULL | MAY_BE_RCN; if (write) { tmp |= MAY_BE_ERROR; } @@ -2222,8 +2222,8 @@ static uint32_t assign_dim_result_type( uint32_t tmp = arr_type & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_ARRAY_KEY_ANY|MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF) & ~(MAY_BE_NULL|MAY_BE_FALSE); - if (arr_type & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE)) { - tmp |= MAY_BE_ARRAY|MAY_BE_RC1; + if (arr_type & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_STRING)) { + tmp |= MAY_BE_ARRAY; } if (arr_type & (MAY_BE_RC1|MAY_BE_RCN)) { tmp |= MAY_BE_RC1; @@ -4106,7 +4106,7 @@ int zend_ssa_inference(zend_arena **arena, const zend_op_array *op_array, const } } else { for (i = 0; i < op_array->last_var; i++) { - ssa_var_info[i].type = MAY_BE_UNDEF; + ssa_var_info[i].type = MAY_BE_UNDEF | MAY_BE_RCN; ssa_var_info[i].has_range = 0; } } From e2f47bf00471a30c2f99e77d5e4b6166f93daa8f Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Fri, 30 Sep 2016 11:50:13 +0800 Subject: [PATCH 262/569] Better to put in at end of buf --- ext/opcache/jit/zend_jit.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index c2b5ef7e6809e..593b9a22916b3 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -1495,7 +1495,6 @@ ZEND_API int zend_jit_startup(size_t size) { size_t page_size = jit_page_size(); int shared = 1; - void *buf; int ret; #ifdef HAVE_GDB @@ -1521,16 +1520,15 @@ ZEND_API int zend_jit_startup(size_t size) size = ZEND_MM_ALIGNED_SIZE_EX(size, page_size); - buf = jit_alloc(size, shared); + dasm_buf = jit_alloc(size, shared); - if (!buf) { + if (!dasm_buf) { return FAILURE; } - dasm_buf = dasm_ptr = buf; - dasm_end = (void*)(((char*)dasm_buf) + size); + dasm_ptr = dasm_end = (void*)(((char*)dasm_buf) + size - sizeof(*dasm_ptr)); zend_jit_unprotect(); - *dasm_ptr = dasm_buf + sizeof(*dasm_ptr); + *dasm_ptr = dasm_buf; zend_jit_protect(); #ifdef HAVE_DISASM From f8dbf69ce335ffb4c2db35ff9a367b07e3963b36 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Fri, 30 Sep 2016 14:06:34 +0800 Subject: [PATCH 263/569] Implemented FETCH_OBJ_R (incomplete) --- ext/opcache/jit/zend_jit.c | 7 +++ ext/opcache/jit/zend_jit_x86.dasc | 100 ++++++++++++++++++++++++++++++ 2 files changed, 107 insertions(+) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 593b9a22916b3..11387ff933a81 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -1102,6 +1102,13 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) goto jit_failure; } break; + /* + case ZEND_FETCH_OBJ_R: + if (!zend_jit_fetch_obj_r(&dasm_state, opline, op_array, ssa)) { + goto jit_failure; + } + break; + */ case ZEND_BIND_GLOBAL: if (!zend_jit_bind_global(&dasm_state, opline, op_array, ssa)) { goto jit_failure; diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 1b16fd45b0693..85ed0b993495b 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -5163,6 +5163,106 @@ static int zend_jit_recv_init(dasm_State **Dst, const zend_op *opline, zend_op_a return 1; } + +static int zend_jit_fetch_obj_r(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) +{ + uint32_t op1_info, res_info; + zend_ssa_var_info *op1_ssa; + zval *member, *zv; + zend_property_info *info; + + if ((opline->op1_type != IS_CV && opline->op1_type != IS_UNUSED) || + opline->op2_type != IS_CONST || !ssa->ops || !ssa->var_info) { + goto fallback; + } + + op1_info = OP1_INFO(); + if ((op1_info & MAY_BE_ANY) != MAY_BE_OBJECT) { + goto fallback; + } + + op1_ssa = &ssa->var_info[ssa->ops[opline - op_array->opcodes].op1_use]; + if (!op1_ssa->ce || op1_ssa->is_instanceof || op1_ssa->ce->create_object) { + goto fallback; + } + + member = RT_CONSTANT(op_array, opline->op2); + if (Z_TYPE_P(member) != IS_STRING || Z_STRVAL_P(member)[0] == '\0') { + goto fallback; + } + + zv = zend_hash_find(&op1_ssa->ce->properties_info, Z_STR_P(member)); + if (zv == NULL) { + goto fallback; + } + + info = (zend_property_info*)Z_PTR_P(zv); + if (!(info->flags & ZEND_ACC_PUBLIC) || info->offset == ZEND_WRONG_PROPERTY_OFFSET) { + goto fallback; + } + + if (opline->op1_type == IS_CV) { + | mov r0, [FP + opline->op1.var] + | ZVAL_DEREF r0, op1_info + if (op1_info & MAY_BE_REF) { + | IF_NOT_Z_TYPE, r0, IS_REFERENCE, >1 + | GET_Z_PTR r0, r0 + | add r0, offsetof(zend_reference, val) + |1: + } + } else { + | mov r0, EX->This + } + if (op1_info & MAY_BE_UNDEF) { + | IF_Z_TYPE r0, IS_UNDEF, >8 + } + | mov r0, [r0 + info->offset] + | // ZVAL_COPY_UNREF + if (res_info & MAY_BE_REF) { + | // ZVAL_COPY_UNREF + | IF_NOT_Z_REFCOUNTED r0, >2 + | GET_Z_PTR r1, r0 + | IF_NOT_Z_TYPE r0, IS_REFERENCE, >1 + | cmp dword [r1], 1 + | jne >1 + | lea FCARG1a, [FP + opline->result.var] + | mov FCARG2a, r0 + | EXT_CALL zend_jit_zval_copy_unref_helper, r0 + | jmp >9 + |1: + | GC_ADDREF r1 + |2: + | ZVAL_COPY_VALUE FP + opline->result.var, r0, MAY_BE_ANY, r1, ecx, r2 + } else { + | // ZVAL_COPY + | ZVAL_COPY_VALUE FP + opline->result.var, r0, MAY_BE_ANY, r1, ecx, r2 + | TRY_ADDREF res_info, ch, r2 + } + |9: // END + + |.cold_code + |8: + |.if X64 + | mov CARG1, 0 + | LOAD_ADDR CARG2, "Using $this when not in object context" + | EXT_CALL zend_throw_error, r0 + |.else + | sub r4, 8 + | push "Using $this when not in object context" + | push 0 + | EXT_CALL zend_throw_error, r0 + | add r4, 16 + |.endif + | jmp <9 + |.code + + return 1; + +fallback: + /* fallback to subroutine threading */ + return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); +} + #endif /* From c48b381c36ce2c52da2047a0aa1353452fc28252 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 30 Sep 2016 15:11:16 +0300 Subject: [PATCH 264/569] Impemented JIT for DO_ICALL --- ext/opcache/jit/zend_jit.c | 1 + ext/opcache/jit/zend_jit_disasm_x86.c | 1 + ext/opcache/jit/zend_jit_helpers.c | 5 + ext/opcache/jit/zend_jit_x86.dasc | 257 ++++++++++++++++++-------- 4 files changed, 191 insertions(+), 73 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 11387ff933a81..9a33f9ad7a71c 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -1046,6 +1046,7 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) } break; case ZEND_DO_UCALL: + case ZEND_DO_ICALL: if (!zend_jit_do_fcall(&dasm_state, opline, op_array, ssa, call_level)) { goto jit_failure; } diff --git a/ext/opcache/jit/zend_jit_disasm_x86.c b/ext/opcache/jit/zend_jit_disasm_x86.c index 4adf4f6c58790..a9760f3526d8a 100644 --- a/ext/opcache/jit/zend_jit_disasm_x86.c +++ b/ext/opcache/jit/zend_jit_disasm_x86.c @@ -409,6 +409,7 @@ static int zend_jit_disasm_init(void) REGISTER_HELPER(zend_jit_assign_dim_sub_helper); REGISTER_HELPER(zend_jit_assign_dim_mul_helper); REGISTER_HELPER(zend_jit_assign_dim_div_helper); + REGISTER_HELPER(zend_jit_free_call_frame); REGISTER_HELPER(zend_jit_zval_copy_unref_helper); REGISTER_HELPER(zend_jit_new_ref_helper); REGISTER_HELPER(zend_jit_fetch_global_helper); diff --git a/ext/opcache/jit/zend_jit_helpers.c b/ext/opcache/jit/zend_jit_helpers.c index 0ade38df3620e..ded3ab23d0dfc 100644 --- a/ext/opcache/jit/zend_jit_helpers.c +++ b/ext/opcache/jit/zend_jit_helpers.c @@ -1076,6 +1076,11 @@ static void ZEND_FASTCALL zend_jit_assign_dim_div_helper(zval *container, zval * } } +static void ZEND_FASTCALL zend_jit_free_call_frame(zend_execute_data *call) +{ + zend_vm_stack_free_call_frame(call); +} + static void ZEND_FASTCALL zend_jit_zval_copy_unref_helper(zval *dst, zval *src) { ZVAL_UNREF(src); diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 85ed0b993495b..d50be7fbcf0af 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -900,7 +900,7 @@ static void* dasm_labels[zend_lb_MAX]; |1: |.endmacro -|.macro ZVAL_PTR_DTOR, zv, info, cold, filename, lineno +|.macro ZVAL_PTR_DTOR, zv, info, gc, cold, safe, filename, lineno || if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { || if (info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { | // if (Z_REFCOUNTED_P(cv)) { @@ -916,15 +916,17 @@ static void* dasm_labels[zend_lb_MAX]; | GET_Z_PTR FCARG1a, zv | GC_DELREF FCARG1a | jnz >1 -| // ZVAL_NULL(cv); -| SET_Z_TYPE_INFO zv, IS_NULL +|| if (safe) { +| // ZVAL_NULL(cv); +| SET_Z_TYPE_INFO zv, IS_NULL +|| } | // zval_dtor_func(r); | ZVAL_DTOR_FUNC filename, lineno -|| if (info & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) { +|| if (gc && (info & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT))) { | jmp >3 || } |1: -|| if (info & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) { +|| if (gc && (info & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT))) { || if (info & MAY_BE_REF) { | lea r1, [zv] | // ZVAL_DEREF(z); @@ -947,7 +949,7 @@ static void* dasm_labels[zend_lb_MAX]; | jmp >3 |.code || } -|| } else if (cold && (info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE)))) { +|| } else if (cold && (!gc || (info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))))) { | jmp >3 |.code || } @@ -1151,7 +1153,7 @@ static int zend_jit_leave_throw_stub(dasm_State **Dst) | mov r0, OP:IP->result.var |.endif | add r0, FP - | ZVAL_PTR_DTOR r0, MAY_BE_ANY, 0, NULL, 0 + | ZVAL_PTR_DTOR r0, MAY_BE_ANY, 1, 0, 0, NULL, 0 |5: | // opline = EG(exception_op); | LOAD_ADDR IP, &EG(exception_op) @@ -3175,7 +3177,7 @@ static int zend_jit_assign_math(dasm_State **Dst, const zend_op *opline, zend_op { uint32_t op1_info, op2_info; - if (opline->extended_value == ZEND_ASSIGN_DIM) { + if (opline->extended_value == ZEND_ASSIGN_DIM) { return zend_jit_assign_dim_math(Dst, opline, op_array, ssa); } else if (opline->extended_value == ZEND_ASSIGN_OBJ) { goto fallback; @@ -4201,12 +4203,26 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar func = call_info->callee_func; } } - if (!func || - func->type != ZEND_USER_FUNCTION || - call_info->num_args > func->op_array.num_args || - (opline-1)->opcode == ZEND_SEND_UNPACK || - (func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) != 0) { + if (!func) { + // TODO: add support for unknown functions ??? goto fallback; + } else if (func->type == ZEND_USER_FUNCTION) { + if (call_info->num_args > func->op_array.num_args || + (opline-1)->opcode == ZEND_SEND_UNPACK || + (func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) != 0) { + goto fallback; + } + } else if (func->type == ZEND_INTERNAL_FUNCTION) { +#if ZEND_DEBUG + if (func->common.fn_flags & ZEND_ACC_HAS_RETURN_TYPE) { + goto fallback; + } +#endif + if ((opline-1)->opcode == ZEND_SEND_UNPACK) { + goto fallback; + } + } else { + ZEND_ASSERT(0); } | // call = EX(call); @@ -4215,73 +4231,168 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar | // mov r2, EX:FCARG1a->func ??? | // SAVE_OPLINE(); | SAVE_VALID_OPLINE opline - || if (call_level == 1) { - | mov aword EX->call, 0 - || } else { - | //EX(call) = call->prev_execute_data; - | mov r0, EX:FCARG1a->prev_execute_data - | mov EX->call, r0 - || } + if (call_level == 1) { + | mov aword EX->call, 0 + } else { + | //EX(call) = call->prev_execute_data; + | mov r0, EX:FCARG1a->prev_execute_data + | mov EX->call, r0 + } | //call->prev_execute_data = execute_data; | mov EX:FCARG1a->prev_execute_data, EX | - | // EX(call) = NULL; - | mov aword EX:FCARG1a->call, 0 - || if (RETURN_VALUE_USED(opline)) { - | // ZVAL_NULL(EX_VAR(opline->result.var)); - | SET_Z_TYPE_INFO FP + opline->result.var, IS_NULL - | // EX(return_value) = EX_VAR(opline->result.var); - | lea r0, aword [FP + opline->result.var] - | mov aword EX:FCARG1a->return_value, r0 - || } else { - | // EX(return_value) = 0; - | mov aword EX:FCARG1a->return_value, 0 - || } - | - || for (i = call_info->num_args; i < func->op_array.last_var; i++) { - || uint32_t n = (uint32_t)(uintptr_t)ZEND_CALL_VAR_NUM(NULL, i); - | SET_Z_TYPE_INFO FCARG1a + n, IS_UNDEF - || } - | - | //EX_LOAD_RUN_TIME_CACHE(op_array); - || if (!func || func->op_array.cache_size) { - || if (func && op_array == &func->op_array) { - || /* recursive call */ - || if (func->op_array.cache_size > sizeof(void*)) { - | mov r0, EX->run_time_cache - | mov EX:FCARG1a->run_time_cache, r0 - || } - || } else { - | mov r2, EX:FCARG1a->func - | mov r0, aword [r2 + offsetof(zend_op_array, run_time_cache)] - | mov EX:FCARG1a->run_time_cache, r0 - || } - || } - | //EX_LOAD_LITERALS(op_array); - |.if X64 - | LOAD_ADDR r0, func->op_array.literals - | mov EX:FCARG1a->literals, r0 - |.endif - | // EG(current_execute_data) = execute_data; - | mov aword [&EG(current_execute_data)], FCARG1a - | mov FP, FCARG1a - | // opline = op_array->opcodes; - | LOAD_ADDR IP, (func->op_array.opcodes + call_info->num_args) - || if (func && op_array == &func->op_array) { - || /* recursive call */ + + if (func && func->type == ZEND_USER_FUNCTION) { + | // EX(call) = NULL; + | mov aword EX:FCARG1a->call, 0 + } + + if (func && func->type == ZEND_USER_FUNCTION) { + + if (RETURN_VALUE_USED(opline)) { + | // ZVAL_NULL(EX_VAR(opline->result.var)); + | SET_Z_TYPE_INFO FP + opline->result.var, IS_NULL + | // EX(return_value) = EX_VAR(opline->result.var); + | lea r0, aword [FP + opline->result.var] + | mov aword EX:FCARG1a->return_value, r0 + } else { + | // EX(return_value) = 0; + | mov aword EX:FCARG1a->return_value, 0 + } + + for (i = call_info->num_args; i < func->op_array.last_var; i++) { + uint32_t n = (uint32_t)(uintptr_t)ZEND_CALL_VAR_NUM(NULL, i); + | SET_Z_TYPE_INFO FCARG1a + n, IS_UNDEF + } + + //EX_LOAD_RUN_TIME_CACHE(op_array); + if (!func || func->op_array.cache_size) { + if (func && op_array == &func->op_array) { + /* recursive call */ + if (func->op_array.cache_size > sizeof(void*)) { + | mov r0, EX->run_time_cache + | mov EX:FCARG1a->run_time_cache, r0 + } + } else { + | mov r2, EX:FCARG1a->func + | mov r0, aword [r2 + offsetof(zend_op_array, run_time_cache)] + | mov EX:FCARG1a->run_time_cache, r0 + } + } + + | //EX_LOAD_LITERALS(op_array); + |.if X64 + | LOAD_ADDR r0, func->op_array.literals + | mov EX:FCARG1a->literals, r0 + |.endif + + | // EG(current_execute_data) = execute_data; + | mov aword [&EG(current_execute_data)], FCARG1a + | mov FP, FCARG1a + + | // opline = op_array->opcodes; + | LOAD_ADDR IP, (func->op_array.opcodes + call_info->num_args) + + if (func && op_array == &func->op_array) { + /* recursive call */ #ifdef CONTEXT_THREADED_JIT - | call =>(call_info->num_args+ssa->cfg.blocks_count) + | call =>(call_info->num_args+ssa->cfg.blocks_count) #else - | jmp =>call_info->num_args + | jmp =>call_info->num_args #endif - || } else { + } else { #ifdef CONTEXT_THREADED_JIT - | call aword [IP] + | call aword [IP] #else - | add r4, SPAD // stack alignment - | jmp aword [IP] + | add r4, SPAD // stack alignment + | jmp aword [IP] #endif - || } + } + } else if (func && func->type == ZEND_INTERNAL_FUNCTION) { + if (RETURN_VALUE_USED(opline)) { + | // ZVAL_NULL(EX_VAR(opline->result.var)); + | lea FCARG2a, aword [FP + opline->result.var] + | SET_Z_TYPE_INFO FCARG2a, IS_NULL + } else { + | sub r4, 16 /* alloca() */ + | lea FCARG2a, aword [r4 + 8] + | SET_Z_TYPE_INFO FCARG2a, IS_NULL + } + + | // EG(current_execute_data) = execute_data; + | mov aword [&EG(current_execute_data)], FCARG1a + + | // fbc->internal_function.handler(call, ret); + | mov aword [r4], FCARG1a // save + |.if X64 + | EXT_CALL func->internal_function.handler, r0 + |.else + | sub r4, 8 + | push FCARG2a + | push FCARG1a + | EXT_CALL func->internal_function.handler, r0 + | add r4, 16 + |.endif + + | // EG(current_execute_data) = execute_data; + | mov aword [&EG(current_execute_data)], FP + + | // zend_vm_stack_free_args(call); + for (i = 0; i < call_info->num_args; i++ ) { + uint32_t offset = (uint32_t)(uintptr_t)ZEND_CALL_VAR_NUM(NULL, i); + | mov r0, aword [r4] // restore + | ZVAL_PTR_DTOR r0 + offset, -1, 0, 1, 1, (op_array->filename ? ZSTR_VAL(op_array->filename) : NULL), opline->lineno + } + + | // zend_vm_stack_free_call_frame(call); + | mov FCARG1a, aword [r4] // restore + | test byte [FCARG1a + offsetof(zend_execute_data, This.u1.type_info) + 2], ZEND_CALL_ALLOCATED + | jnz >1 + |.cold_code + |1: + | EXT_CALL zend_jit_free_call_frame, r0 + | jmp >1 + |.code + | mov aword [&EG(vm_stack_top)], FCARG1a + |1: + + if (opline->result_type == IS_UNUSED) { + uint32_t func_info = zend_get_func_info(call_info, ssa); + + if (func_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { + | lea r0, aword [r4 + 8] + } + | add r4, 16 /* revert alloca() */ + if (func_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { + | ZVAL_PTR_DTOR r0, func_info, 1, 1, 0, (op_array->filename ? ZSTR_VAL(op_array->filename) : NULL), opline->lineno + } + } + + | // if (UNEXPECTED(EG(exception) != NULL)) { + | cmp aword [&EG(exception)], 0 + | jne >1 + |.cold_code + |1: + | // zend_throw_exception_internal(NULL); + |.if X64 + | xor CARG1, CARG1 + | EXT_CALL zend_throw_exception_internal, r0 + |.else + | sub r4, 12 + | push 0 + | EXT_CALL zend_throw_exception_internal, r0 + | add r4, 16 + |.endif + if (opline->result_type != IS_UNUSED) { + | // zval_ptr_dtor(EX_VAR(opline->result.var)); + | ZVAL_PTR_DTOR FP + opline->result.var, RES_INFO(), 1, 0, 0, (op_array->filename ? ZSTR_VAL(op_array->filename) : NULL), opline->lineno + } + | // HANDLE_EXCEPTION(); + | jmp ->exception_handler + |.code + // TODO: Interrupt handling ??? + //return zend_jit_check_timeout(Dst, opline + 1); + } return 1; @@ -4464,7 +4575,7 @@ static int zend_jit_free_compiled_variables(dasm_State **Dst, const zend_op *opl if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { uint32_t offset = (uint32_t)(uintptr_t)ZEND_CALL_VAR_NUM(NULL, i); - | ZVAL_PTR_DTOR FP + offset, info, 1, (op_array->filename ? ZSTR_VAL(op_array->filename) : NULL), opline->lineno + | ZVAL_PTR_DTOR FP + offset, info, 1, 1, 1, (op_array->filename ? ZSTR_VAL(op_array->filename) : NULL), opline->lineno } } return 1; @@ -5134,7 +5245,7 @@ static int zend_jit_recv_init(dasm_State **Dst, const zend_op *opline, zend_op_a |.cold_code if (has_slow & 1) { |7: - | ZVAL_PTR_DTOR FP + opline->result.var, -1, 0, (op_array->filename ? ZSTR_VAL(op_array->filename) : NULL), opline->lineno + | ZVAL_PTR_DTOR FP + opline->result.var, -1, 1, 0, 0, (op_array->filename ? ZSTR_VAL(op_array->filename) : NULL), opline->lineno | SET_Z_TYPE_INFO FP + opline->result.var, IS_UNDEF | jmp <4 } From 1c188e49e504bf6b06f3d75dc0fe3089e5449e7c Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Sat, 1 Oct 2016 15:46:25 +0800 Subject: [PATCH 265/569] Tiny cleanup --- ext/opcache/jit/zend_jit_x86.dasc | 52 ++++++++++++++++++------------- 1 file changed, 30 insertions(+), 22 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index d50be7fbcf0af..41257d9f84567 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -842,7 +842,11 @@ static void* dasm_labels[zend_lb_MAX]; // zval should be in FCARG1a |.macro ZVAL_COPY_CTOR_FUNC, filename, lineno // arg1 must be in FCARG1a || if (ZEND_DEBUG) { -| LOAD_ADDR FCARG2a, filename +|| if (filename) { +| LOAD_ADDR FCARG2a, ZSTR_VAL((zend_string*)filename) +|| } else { +| mov FCARG2a, 0 +|| } |.if X64 | mov CARG3d, lineno |.else @@ -881,7 +885,11 @@ static void* dasm_labels[zend_lb_MAX]; // zval should be in FCARG1a |.macro ZVAL_DTOR_FUNC, filename, lineno // arg1 must be in FCARG1a || if (ZEND_DEBUG) { -| LOAD_ADDR FCARG2a, filename +|| if (filename) { +| LOAD_ADDR FCARG2a, ZSTR_VAL((zend_string*)filename) +|| } else { +| mov FCARG2a, 0 +|| } |.if X64 | mov CARG3d, lineno |.else @@ -960,7 +968,7 @@ static void* dasm_labels[zend_lb_MAX]; |.macro FREE_OP, op_type, op, op_info, op_array, lineno ||if ((op_type & (IS_VAR|IS_TMP_VAR)) && || (op_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { -| ZVAL_PTR_DTOR_NOGC FP + op.var, (op_array->filename ? ZSTR_VAL(op_array->filename) : NULL), lineno +| ZVAL_PTR_DTOR_NOGC FP + op.var, op_array->filename, lineno ||} |.endmacro @@ -1435,7 +1443,7 @@ static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, zend_op_arr || if ((opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC)) { || if (opline->result_type != IS_UNUSED) { | ZVAL_COPY_VALUE FP + opline->result.var, FCARG1a, op1_info, r0, eax, r2 - | //ZVAL_COPY_CTOR op1_info, ah, r2, (op_array->filename ? ZSTR_VAL(op_array->filename) : NULL), opline->lineno + | //ZVAL_COPY_CTOR op1_info, ah, r2, op_array->filename, opline->lineno || if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { || if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY)) { | IF_NOT_FLAGS ah, IS_TYPE_REFCOUNTED + IS_TYPE_IMMUTABLE, >2 @@ -1444,7 +1452,7 @@ static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, zend_op_arr | jmp >2 |1: | mov aword [r4], FCARG1a // save - | ZVAL_COPY_CTOR_FUNC (op_array->filename ? ZSTR_VAL(op_array->filename) : NULL), opline->lineno + | ZVAL_COPY_CTOR_FUNC op_array->filename, opline->lineno | mov FCARG1a, aword [r4] // restore |2: || } else { @@ -1463,7 +1471,7 @@ static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, zend_op_arr | GC_DELREF r0 |1: | mov aword [r4], FCARG1a // save - | ZVAL_COPY_CTOR_FUNC (op_array->filename ? ZSTR_VAL(op_array->filename) : NULL), opline->lineno + | ZVAL_COPY_CTOR_FUNC op_array->filename, opline->lineno | mov FCARG1a, aword [r4] // restore |2: || } @@ -2522,7 +2530,7 @@ static int zend_jit_assign_to_variable(dasm_State **Dst, return 0; } | mov FCARG1a, aword [r4] // restore - | ZVAL_DTOR_FUNC (op_array->filename ? ZSTR_VAL(op_array->filename) : NULL), opline->lineno + | ZVAL_DTOR_FUNC op_array->filename, opline->lineno | jmp >3 |4: | IF_NOT_Z_FLAGS Ra(var_reg)+var_offset, IS_TYPE_COLLECTABLE, >5 @@ -2592,7 +2600,7 @@ static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, zend_op_ | GC_DELREF r0 |2: | mov aword [r4], FCARG1a // save - | ZVAL_COPY_CTOR_FUNC (op_array->filename ? ZSTR_VAL(op_array->filename) : NULL), opline->lineno + | ZVAL_COPY_CTOR_FUNC op_array->filename, opline->lineno | mov FCARG1a, aword [r4] // restore | GET_Z_PTR FCARG1a, FCARG1a | jmp >3 @@ -2613,7 +2621,7 @@ static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, zend_op_ | GC_DELREF FCARG1a |2: | LOAD_ZVAL_ADDR FCARG1a, opline->op1_type, opline->op1 - | ZVAL_COPY_CTOR_FUNC (op_array->filename ? ZSTR_VAL(op_array->filename) : NULL), opline->lineno + | ZVAL_COPY_CTOR_FUNC op_array->filename, opline->lineno | LONG_LOAD FCARG1a, opline->op1_type, opline->op1 | jmp >3 |.code @@ -2890,7 +2898,7 @@ static int zend_jit_assign_dim_math(dasm_State **Dst, const zend_op *opline, zen | GC_DELREF r0 |2: | mov aword [r4], FCARG1a // save - | ZVAL_COPY_CTOR_FUNC (op_array->filename ? ZSTR_VAL(op_array->filename) : NULL), opline->lineno + | ZVAL_COPY_CTOR_FUNC op_array->filename, opline->lineno | mov FCARG1a, aword [r4] // restore | GET_Z_PTR FCARG1a, FCARG1a | jmp >3 @@ -2911,7 +2919,7 @@ static int zend_jit_assign_dim_math(dasm_State **Dst, const zend_op *opline, zen | GC_DELREF FCARG1a |2: | LOAD_ZVAL_ADDR FCARG1a, opline->op1_type, opline->op1 - | ZVAL_COPY_CTOR_FUNC (op_array->filename ? ZSTR_VAL(op_array->filename) : NULL), opline->lineno + | ZVAL_COPY_CTOR_FUNC op_array->filename, opline->lineno | LONG_LOAD FCARG1a, opline->op1_type, opline->op1 | jmp >3 |.code @@ -3038,7 +3046,7 @@ static int zend_jit_assign_dim_math(dasm_State **Dst, const zend_op *opline, zen | GC_DELREF r0 |2: | mov aword [r4], FCARG1a // save - | ZVAL_COPY_CTOR_FUNC (op_array->filename ? ZSTR_VAL(op_array->filename) : NULL), opline->lineno + | ZVAL_COPY_CTOR_FUNC op_array->filename, opline->lineno | mov FCARG1a, aword [r4] // restore | jmp >3 |.code @@ -3218,7 +3226,7 @@ static int zend_jit_assign_math(dasm_State **Dst, const zend_op *opline, zend_op | GC_DELREF r0 |2: | mov aword [r4], FCARG1a // save - | ZVAL_COPY_CTOR_FUNC (op_array->filename ? ZSTR_VAL(op_array->filename) : NULL), opline->lineno + | ZVAL_COPY_CTOR_FUNC op_array->filename, opline->lineno | mov FCARG1a, aword [r4] // restore | jmp >3 |.code @@ -3238,7 +3246,7 @@ static int zend_jit_assign_math(dasm_State **Dst, const zend_op *opline, zend_op | GC_DELREF r0 |2: | lea FCARG1a, [FP + opline->op1.var] - | ZVAL_COPY_CTOR_FUNC (op_array->filename ? ZSTR_VAL(op_array->filename) : NULL), opline->lineno + | ZVAL_COPY_CTOR_FUNC op_array->filename, opline->lineno | jmp >3 |.code |3: @@ -4010,7 +4018,7 @@ static int zend_jit_jmpznz(dasm_State **Dst, const zend_op *opline, int b, zend_ | dec dword [FCARG1a] | jnz >1 | mov aword [r4], r0 // save - | ZVAL_DTOR_FUNC (op_array->filename ? ZSTR_VAL(op_array->filename) : NULL), opline->lineno + | ZVAL_DTOR_FUNC op_array->filename, opline->lineno | mov r0, aword [r4] // restore |1: } @@ -4341,7 +4349,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar for (i = 0; i < call_info->num_args; i++ ) { uint32_t offset = (uint32_t)(uintptr_t)ZEND_CALL_VAR_NUM(NULL, i); | mov r0, aword [r4] // restore - | ZVAL_PTR_DTOR r0 + offset, -1, 0, 1, 1, (op_array->filename ? ZSTR_VAL(op_array->filename) : NULL), opline->lineno + | ZVAL_PTR_DTOR r0 + offset, -1, 0, 1, 1, op_array->filename, opline->lineno } | // zend_vm_stack_free_call_frame(call); @@ -4364,7 +4372,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar } | add r4, 16 /* revert alloca() */ if (func_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { - | ZVAL_PTR_DTOR r0, func_info, 1, 1, 0, (op_array->filename ? ZSTR_VAL(op_array->filename) : NULL), opline->lineno + | ZVAL_PTR_DTOR r0, func_info, 1, 1, 0, op_array->filename, opline->lineno } } @@ -4385,7 +4393,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar |.endif if (opline->result_type != IS_UNUSED) { | // zval_ptr_dtor(EX_VAR(opline->result.var)); - | ZVAL_PTR_DTOR FP + opline->result.var, RES_INFO(), 1, 0, 0, (op_array->filename ? ZSTR_VAL(op_array->filename) : NULL), opline->lineno + | ZVAL_PTR_DTOR FP + opline->result.var, RES_INFO(), 1, 0, 0, op_array->filename, opline->lineno } | // HANDLE_EXCEPTION(); | jmp ->exception_handler @@ -4575,7 +4583,7 @@ static int zend_jit_free_compiled_variables(dasm_State **Dst, const zend_op *opl if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { uint32_t offset = (uint32_t)(uintptr_t)ZEND_CALL_VAR_NUM(NULL, i); - | ZVAL_PTR_DTOR FP + offset, info, 1, 1, 1, (op_array->filename ? ZSTR_VAL(op_array->filename) : NULL), opline->lineno + | ZVAL_PTR_DTOR FP + offset, info, 1, 1, 1, op_array->filename, opline->lineno } } return 1; @@ -4732,7 +4740,7 @@ static int zend_jit_return(dasm_State **Dst, const zend_op *opline, zend_op_arra || } | //SAVE_OPLINE() | SAVE_VALID_OPLINE opline - | ZVAL_DTOR_FUNC (op_array->filename ? ZSTR_VAL(op_array->filename) : NULL), opline->lineno + | ZVAL_DTOR_FUNC op_array->filename, opline->lineno | mov r1, EX->return_value // reload ??? || if (jit_return_label >= 0) { | jmp =>jit_return_label @@ -5121,7 +5129,7 @@ static int zend_jit_bind_global(dasm_State **Dst, const zend_op *opline, zend_op | jnz >3 } | mov aword [r4], r0 // save - | ZVAL_DTOR_FUNC (op_array->filename ? ZSTR_VAL(op_array->filename) : NULL), opline->lineno + | ZVAL_DTOR_FUNC op_array->filename, opline->lineno | mov r0, aword [r4] // restore | jmp >5 if (op1_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) { @@ -5245,7 +5253,7 @@ static int zend_jit_recv_init(dasm_State **Dst, const zend_op *opline, zend_op_a |.cold_code if (has_slow & 1) { |7: - | ZVAL_PTR_DTOR FP + opline->result.var, -1, 1, 0, 0, (op_array->filename ? ZSTR_VAL(op_array->filename) : NULL), opline->lineno + | ZVAL_PTR_DTOR FP + opline->result.var, -1, 1, 0, 0, op_array->filename, opline->lineno | SET_Z_TYPE_INFO FP + opline->result.var, IS_UNDEF | jmp <4 } From dc4ddbb7e339c165e7c5c251e7890998bb5a2006 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Sun, 2 Oct 2016 00:07:10 +0800 Subject: [PATCH 266/569] Fixed IS_CV support --- ext/opcache/jit/zend_jit_x86.dasc | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 41257d9f84567..f025736124ac6 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -5290,8 +5290,8 @@ static int zend_jit_fetch_obj_r(dasm_State **Dst, zend_op *opline, zend_op_array zval *member, *zv; zend_property_info *info; - if ((opline->op1_type != IS_CV && opline->op1_type != IS_UNUSED) || - opline->op2_type != IS_CONST || !ssa->ops || !ssa->var_info) { + /* TODO: supports IS_UNUSED? */ + if (opline->op1_type != IS_CV || opline->op2_type != IS_CONST || !ssa->ops || !ssa->var_info) { goto fallback; } @@ -5321,21 +5321,16 @@ static int zend_jit_fetch_obj_r(dasm_State **Dst, zend_op *opline, zend_op_array } if (opline->op1_type == IS_CV) { - | mov r0, [FP + opline->op1.var] + | lea r0, [FP + opline->op1.var] | ZVAL_DEREF r0, op1_info - if (op1_info & MAY_BE_REF) { - | IF_NOT_Z_TYPE, r0, IS_REFERENCE, >1 - | GET_Z_PTR r0, r0 - | add r0, offsetof(zend_reference, val) - |1: - } } else { - | mov r0, EX->This + | lea r0, EX->This } if (op1_info & MAY_BE_UNDEF) { | IF_Z_TYPE r0, IS_UNDEF, >8 } - | mov r0, [r0 + info->offset] + | mov r0, [r0] + | lea r0, [r0 + info->offset] | // ZVAL_COPY_UNREF if (res_info & MAY_BE_REF) { | // ZVAL_COPY_UNREF @@ -5359,6 +5354,7 @@ static int zend_jit_fetch_obj_r(dasm_State **Dst, zend_op *opline, zend_op_array } |9: // END + if (op1_info & MAY_BE_UNDEF) { |.cold_code |8: |.if X64 @@ -5374,6 +5370,7 @@ static int zend_jit_fetch_obj_r(dasm_State **Dst, zend_op *opline, zend_op_array |.endif | jmp <9 |.code + } return 1; From 3da8f923786c4a9e7473a3e583280095345955d8 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Sun, 2 Oct 2016 00:23:42 +0800 Subject: [PATCH 267/569] Fixed warning "Warning: this decimal constant is unsigned only in ISO C90" --- ext/opcache/jit/zend_jit_x86.dasc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index f025736124ac6..0b1ca5d4e0cec 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -94,7 +94,7 @@ static void* dasm_labels[zend_lb_MAX]; #define IS_32BIT(addr) (((uintptr_t)(addr)) <= 0xffffffff) -#define IS_SIGNED_32BIT(val) ((((intptr_t)(val)) <= 0x7fffffff) && (((intptr_t)(val)) >= -2147483648)) +#define IS_SIGNED_32BIT(val) ((((intptr_t)(val)) <= 0x7fffffff) && (((intptr_t)(val)) >= (-2147483647 - 1))) |.macro LOAD_ADDR, reg, addr |.if X64 From 96deeaa3ddce9f95622e833b0f05ba6634dfc596 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 3 Oct 2016 13:25:24 +0300 Subject: [PATCH 268/569] Move code into DynAsm macros --- ext/opcache/jit/zend_jit_x86.dasc | 323 ++++++++++++++---------------- 1 file changed, 145 insertions(+), 178 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 0b1ca5d4e0cec..4196f8aa416d3 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -899,25 +899,16 @@ static void* dasm_labels[zend_lb_MAX]; | EXT_CALL _zval_dtor_func, r0 |.endmacro -|.macro ZVAL_PTR_DTOR_NOGC, zv, filename, lineno -| IF_NOT_Z_REFCOUNTED zv, >1 -| GET_Z_PTR FCARG1a, zv -| GC_DELREF FCARG1a -| jnz >1 -| ZVAL_DTOR_FUNC filename, lineno -|1: -|.endmacro - -|.macro ZVAL_PTR_DTOR, zv, info, gc, cold, safe, filename, lineno -|| if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { -|| if (info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { +|.macro ZVAL_PTR_DTOR, zv, op_info, gc, cold, safe, filename, lineno +|| if (op_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { +|| if (op_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { | // if (Z_REFCOUNTED_P(cv)) { || if (cold) { | IF_Z_REFCOUNTED zv, >1 |.cold_code |1: || } else { -| IF_NOT_Z_REFCOUNTED zv, >3 +| IF_NOT_Z_REFCOUNTED zv, >2 || } || } | // if (!Z_DELREF_P(cv)) { @@ -930,48 +921,147 @@ static void* dasm_labels[zend_lb_MAX]; || } | // zval_dtor_func(r); | ZVAL_DTOR_FUNC filename, lineno -|| if (gc && (info & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT))) { -| jmp >3 +|| if (gc && (op_info & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT))) { +| jmp >2 || } |1: -|| if (gc && (info & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT))) { -|| if (info & MAY_BE_REF) { +|| if (gc && (op_info & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT))) { +|| if (op_info & MAY_BE_REF) { | lea r1, [zv] | // ZVAL_DEREF(z); -| ZVAL_DEREF r1, info +| ZVAL_DEREF r1, op_info | // if (Z_COLLECTABLE_P(z)) -| IF_NOT_Z_FLAGS r1, IS_TYPE_COLLECTABLE, >3 +| IF_NOT_Z_FLAGS r1, IS_TYPE_COLLECTABLE, >2 | // if (UNEXPECTED(!Z_GC_INFO_P(z))) | GET_Z_PTR FCARG1a, r1 -| IF_GC_INFO FCARG1a, >3 +| IF_GC_INFO FCARG1a, >2 || } else { | // if (Z_COLLECTABLE_P(z)) -| IF_NOT_Z_FLAGS zv, IS_TYPE_COLLECTABLE, >3 +| IF_NOT_Z_FLAGS zv, IS_TYPE_COLLECTABLE, >2 | // if (UNEXPECTED(!Z_GC_INFO_P(z))) | GET_Z_PTR FCARG1a, zv -| IF_GC_INFO FCARG1a, >3 +| IF_GC_INFO FCARG1a, >2 || } | // gc_possible_root(Z_COUNTED_P(z)) | EXT_CALL gc_possible_root, r0 -|| if (cold && (info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE)))) { -| jmp >3 +|| if (cold && (op_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE)))) { +| jmp >2 |.code || } -|| } else if (cold && (!gc || (info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))))) { -| jmp >3 +|| } else if (cold && (!gc || (op_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))))) { +| jmp >2 |.code || } -|3: +|2: || } |.endmacro -|.macro FREE_OP, op_type, op, op_info, op_array, lineno -||if ((op_type & (IS_VAR|IS_TMP_VAR)) && -|| (op_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { -| ZVAL_PTR_DTOR_NOGC FP + op.var, op_array->filename, lineno +|.macro FREE_OP, op_type, op, op_info, cold, op_array, lineno +||if (op_type & (IS_VAR|IS_TMP_VAR)) { +| ZVAL_PTR_DTOR FP + op.var, op_info, 0, cold, 0, op_array->filename, lineno ||} |.endmacro +|.macro SEPARATE_ZVAL_NOREF, zv, op_info, cold, filename, lineno +|| if (op_info & (MAY_BE_STRING|MAY_BE_ARRAY)) { +|| if (cold) { +| IF_Z_FLAGS zv, IS_TYPE_COPYABLE + IS_TYPE_IMMUTABLE, >1 +|.cold_code +|1: +|| } else { +| IF_Z_FLAGS zv, IS_TYPE_COPYABLE + IS_TYPE_IMMUTABLE, >2 +|| } +| GET_Z_PTR r0, zv +| cmp dword [r0], 1 // if (GC_REFCOUNTED() > 1) +| jbe >3 +| IF_Z_FLAGS zv, IS_TYPE_IMMUTABLE, >1 +| GC_DELREF r0 +|1: +| lea FCARG1a, [zv] +| ZVAL_COPY_CTOR_FUNC filename, lineno +|| if (cold) { +| jmp >2 +|.code +|| } +|2: +|| } +|.endmacro + +|.macro SEPARATE_ZVAL_NOREF_REG, op_info, cold, filename, lineno +|| if (op_info & (MAY_BE_STRING|MAY_BE_ARRAY)) { +|| if (cold) { +| IF_Z_FLAGS FCARG1a, IS_TYPE_COPYABLE + IS_TYPE_IMMUTABLE, >1 +|.cold_code +|1: +|| } else { +| IF_NOT_Z_FLAGS FCARG1a, IS_TYPE_COPYABLE + IS_TYPE_IMMUTABLE, >2 +|| } +| GET_Z_PTR r0, FCARG1a +| cmp dword [r0], 1 // if (GC_REFCOUNTED() > 1) +| jbe >2 +| IF_Z_FLAGS FCARG1a, IS_TYPE_IMMUTABLE, >1 +| GC_DELREF r0 +|1: +| mov aword [r4], FCARG1a // save +| ZVAL_COPY_CTOR_FUNC filename, lineno +| mov FCARG1a, aword [r4] // restore +|| if (cold) { +| jmp >2 +|.code +|| } +|2: +|| } +|.endmacro + +|.macro SEPARATE_ARRAY, op_type, op, op_info, cold, filename, lineno +| LONG_LOAD FCARG1a, op_type, op +| cmp dword [FCARG1a], 1 // if (GC_REFCOUNTED() > 1) +|| if (cold) { +| ja >1 +|.cold_code +|1: +|| } else { +| jbe >2 +|| } +| IF_Z_FLAGS FP + op.var, IS_TYPE_IMMUTABLE, >1 +| GC_DELREF FCARG1a +|1: +| LOAD_ZVAL_ADDR FCARG1a, op_type, op +| ZVAL_COPY_CTOR_FUNC filename, lineno +| LONG_LOAD FCARG1a, op_type, op +|| if (cold) { +| jmp >2 +|.code +|| } +|2: +|.endmacro + +|.macro SEPARATE_ARRAY_REG, op_info, cold, filename, lineno +| GET_Z_PTR r0, FCARG1a +| cmp dword [r0], 1 // if (GC_REFCOUNTED() > 1) +|| if (cold) { +| ja >1 +|.cold_code +|1: +|| } else { +| jbe >2 +|| } +| IF_Z_FLAGS FCARG1a, IS_TYPE_IMMUTABLE, >1 +| GC_DELREF r0 +|1: +| mov aword [r4], FCARG1a // save +| ZVAL_COPY_CTOR_FUNC filename, lineno +| mov FCARG1a, aword [r4] // restore +| GET_Z_PTR FCARG1a, FCARG1a +|| if (cold) { +| jmp >2 +|.code +|| } +| mov FCARG1a, r0 +|2: +|.endmacro + + |.macro EFREE_SIZE, ptr, size, op_array, opline | mov FCARG1a, ptr || if (ZEND_DEBUG) { @@ -1461,20 +1551,7 @@ static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, zend_op_arr || } || } || } else { - || if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY)) { - | // SEPARATE_ZVAL_NOREF(var_ptr); - | IF_NOT_Z_FLAGS FCARG1a, IS_TYPE_COPYABLE + IS_TYPE_IMMUTABLE, >2 - | GET_Z_PTR r0, FCARG1a - | cmp dword [r0], 1 // if (GC_REFCOUNTED() > 1) - | jbe >2 - | IF_Z_FLAGS FCARG1a, IS_TYPE_IMMUTABLE, >1 - | GC_DELREF r0 - |1: - | mov aword [r4], FCARG1a // save - | ZVAL_COPY_CTOR_FUNC op_array->filename, opline->lineno - | mov FCARG1a, aword [r4] // restore - |2: - || } + | SEPARATE_ZVAL_NOREF_REG op1_info, 0, op_array->filename, opline->lineno || } || if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { | EXT_CALL increment_function, r0 @@ -1799,11 +1876,6 @@ static int zend_jit_shift(dasm_State **Dst, const zend_op *opline, zend_op_array } | SET_Z_LVAL FP + opline->result.var, r0 | SET_Z_TYPE_INFO FP + opline->result.var, IS_LONG - if (has_slow) { - |2: - } - | FREE_OP opline->op1_type, opline->op1, op1_info, op_array, opline->lineno - //END if (has_slow) { |.cold_code @@ -1820,10 +1892,13 @@ static int zend_jit_shift(dasm_State **Dst, const zend_op *opline, zend_op_array } else { | EXT_CALL shift_left_function, r0 } - | jmp <2 + | jmp >1 |.code + |1: } + | FREE_OP opline->op1_type, opline->op1, op1_info, 1, op_array, opline->lineno + return 1; fallback: @@ -2025,8 +2100,8 @@ static int zend_jit_math_helper(dasm_State **Dst, || } else { || ZEND_ASSERT(0); || } - | FREE_OP op1_type, op1, op1_info, op_array, opline->lineno - | FREE_OP op2_type, op2, op2_info, op_array, opline->lineno + | FREE_OP op1_type, op1, op1_info, 0, op_array, opline->lineno + | FREE_OP op2_type, op2, op2_info, 0, op_array, opline->lineno || if (zend_may_throw(opline, op_array, ssa)) { || zend_jit_check_exception(Dst); || } @@ -2590,42 +2665,12 @@ static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, zend_op_ if (op1_info & MAY_BE_ARRAY) { if (op1_info & MAY_BE_REF) { | IF_NOT_Z_TYPE FCARG1a, IS_ARRAY, >7 - | // SEPARATE_ARRAY(var_ptr); - | GET_Z_PTR r0, FCARG1a - | cmp dword [r0], 1 // if (GC_REFCOUNTED() > 1) - | ja >1 - |.cold_code - |1: - | IF_Z_FLAGS FCARG1a, IS_TYPE_IMMUTABLE, >2 - | GC_DELREF r0 - |2: - | mov aword [r4], FCARG1a // save - | ZVAL_COPY_CTOR_FUNC op_array->filename, opline->lineno - | mov FCARG1a, aword [r4] // restore - | GET_Z_PTR FCARG1a, FCARG1a - | jmp >3 - |.code - | mov FCARG1a, r0 - |3: + | SEPARATE_ARRAY_REG op1_info, 1, op_array->filename, opline->lineno } else { if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { | IF_NOT_Z_TYPE FP + opline->op1.var, IS_ARRAY, >7 } - | // SEPARATE_ARRAY(var_ptr); - | LONG_LOAD FCARG1a, opline->op1_type, opline->op1 - | cmp dword [FCARG1a], 1 // if (GC_REFCOUNTED() > 1) - | ja >1 - |.cold_code - |1: - | IF_Z_FLAGS FP + opline->op1.var, IS_TYPE_IMMUTABLE, >2 - | GC_DELREF FCARG1a - |2: - | LOAD_ZVAL_ADDR FCARG1a, opline->op1_type, opline->op1 - | ZVAL_COPY_CTOR_FUNC op_array->filename, opline->lineno - | LONG_LOAD FCARG1a, opline->op1_type, opline->op1 - | jmp >3 - |.code - |3: + | SEPARATE_ARRAY opline->op1_type, opline->op1, op1_info, 1, op_array->filename, opline->lineno } } else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE)) { if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { @@ -2841,7 +2886,7 @@ static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, zend_op_ | PUSH_ZVAL_ADDR (opline+1)->op1_type, (opline+1)->op1, r0 |.endif | EXT_CALL zend_jit_assign_dim_helper, r0 - | FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, op_array, opline->lineno + | FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, 0, op_array, opline->lineno } if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { @@ -2853,7 +2898,7 @@ static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, zend_op_ } |9: - | FREE_OP opline->op2_type, opline->op2, op2_info, op_array, opline->lineno + | FREE_OP opline->op2_type, opline->op2, op2_info, 0, op_array, opline->lineno if (zend_may_throw(opline, op_array, ssa)) { zend_jit_check_exception(Dst); @@ -2888,42 +2933,12 @@ static int zend_jit_assign_dim_math(dasm_State **Dst, const zend_op *opline, zen if (op1_info & MAY_BE_ARRAY) { if (op1_info & MAY_BE_REF) { | IF_NOT_Z_TYPE FCARG1a, IS_ARRAY, >7 - | // SEPARATE_ARRAY(var_ptr); - | GET_Z_PTR r0, FCARG1a - | cmp dword [r0], 1 // if (GC_REFCOUNTED() > 1) - | ja >1 - |.cold_code - |1: - | IF_Z_FLAGS FCARG1a, IS_TYPE_IMMUTABLE, >2 - | GC_DELREF r0 - |2: - | mov aword [r4], FCARG1a // save - | ZVAL_COPY_CTOR_FUNC op_array->filename, opline->lineno - | mov FCARG1a, aword [r4] // restore - | GET_Z_PTR FCARG1a, FCARG1a - | jmp >3 - |.code - | mov FCARG1a, r0 - |3: + | SEPARATE_ARRAY_REG op1_info, 1, op_array->filename, opline->lineno } else { if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { | IF_NOT_Z_TYPE FP + opline->op1.var, IS_ARRAY, >7 } - | // SEPARATE_ARRAY(var_ptr); - | LONG_LOAD FCARG1a, opline->op1_type, opline->op1 - | cmp dword [FCARG1a], 1 // if (GC_REFCOUNTED() > 1) - | ja >1 - |.cold_code - |1: - | IF_Z_FLAGS FP + opline->op1.var, IS_TYPE_IMMUTABLE, >2 - | GC_DELREF FCARG1a - |2: - | LOAD_ZVAL_ADDR FCARG1a, opline->op1_type, opline->op1 - | ZVAL_COPY_CTOR_FUNC op_array->filename, opline->lineno - | LONG_LOAD FCARG1a, opline->op1_type, opline->op1 - | jmp >3 - |.code - |3: + | SEPARATE_ARRAY opline->op1_type, opline->op1, op1_info, 1, op_array->filename, opline->lineno } } else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE)) { if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { @@ -2987,6 +3002,8 @@ static int zend_jit_assign_dim_math(dasm_State **Dst, const zend_op *opline, zen } if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { + uint32_t var_info = zend_array_element_type(op1_info, 0, 0); + |6: if (opline->op2_type == IS_UNUSED) { | // var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval)); @@ -3034,27 +3051,10 @@ static int zend_jit_assign_dim_math(dasm_State **Dst, const zend_op *opline, zen if (op1_info & (MAY_BE_ARRAY_OF_REF)) { | ZVAL_DEREF FCARG1a, MAY_BE_REF } - if (op1_info & (MAY_BE_ARRAY_OF_STRING|MAY_BE_ARRAY_OF_ARRAY)) { - // SEPARATE_ZVAL_NOREF(var_ptr); - | IF_Z_FLAGS FCARG1a, IS_TYPE_COPYABLE + IS_TYPE_IMMUTABLE, >1 - |.cold_code - |1: - | GET_Z_PTR r0, FCARG1a - | cmp dword [r0], 1 // if (GC_REFCOUNTED() > 1) - | jbe >3 - | IF_Z_FLAGS FCARG1a, IS_TYPE_IMMUTABLE, >2 - | GC_DELREF r0 - |2: - | mov aword [r4], FCARG1a // save - | ZVAL_COPY_CTOR_FUNC op_array->filename, opline->lineno - | mov FCARG1a, aword [r4] // restore - | jmp >3 - |.code - |3: - } + | SEPARATE_ZVAL_NOREF_REG var_info, 1, op_array->filename, opline->lineno } - if (!zend_jit_math_helper(Dst, opline, op_array, ssa, IS_CV, opline->op1, FCARG1a, 0, zend_array_element_type(op1_info, 0, 0), (opline+1)->op1_type, (opline+1)->op1, FP, (opline+1)->op1.var, OP1_DATA_INFO(), FCARG1a, 0, OP1_DEF_INFO())) { + if (!zend_jit_math_helper(Dst, opline, op_array, ssa, IS_CV, opline->op1, FCARG1a, 0, var_info, (opline+1)->op1_type, (opline+1)->op1, FP, (opline+1)->op1.var, OP1_DATA_INFO(), FCARG1a, 0, OP1_DEF_INFO())) { return 0; } } @@ -3214,43 +3214,10 @@ static int zend_jit_assign_math(dasm_State **Dst, const zend_op *opline, zend_op if (op1_info & MAY_BE_REF) { | LOAD_ZVAL_ADDR FCARG1a, opline->op1_type, opline->op1 | ZVAL_DEREF FCARG1a, op1_info - if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY)) { - | // SEPARATE_ZVAL_NOREF(var); - | IF_Z_FLAGS FCARG1a, IS_TYPE_COPYABLE + IS_TYPE_IMMUTABLE, >1 - |.cold_code - |1: - | GET_Z_PTR r0, FCARG1a - | cmp dword [r0], 1 // if (GC_REFCOUNTED() > 1) - | jbe >3 - | IF_Z_FLAGS FCARG1a, IS_TYPE_IMMUTABLE, >2 - | GC_DELREF r0 - |2: - | mov aword [r4], FCARG1a // save - | ZVAL_COPY_CTOR_FUNC op_array->filename, opline->lineno - | mov FCARG1a, aword [r4] // restore - | jmp >3 - |.code - |3: - } + | SEPARATE_ZVAL_NOREF_REG op1_info, 1, op_array->filename, opline->lineno return zend_jit_math_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, FCARG1a, 0, op1_info, opline->op2_type, opline->op2, FP, opline->op2.var, op2_info, FCARG1a, 0, OP1_DEF_INFO()); } else { - if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY)) { - | // SEPARATE_ZVAL_NOREF(var); - | IF_Z_FLAGS FP + opline->op1.var, IS_TYPE_COPYABLE + IS_TYPE_IMMUTABLE, >1 - |.cold_code - |1: - | GET_Z_PTR r0, FP + opline->op1.var - | cmp dword [r0], 1 // if (GC_REFCOUNTED() > 1) - | jbe >3 - | IF_Z_FLAGS FP + opline->op1.var, IS_TYPE_IMMUTABLE, >2 - | GC_DELREF r0 - |2: - | lea FCARG1a, [FP + opline->op1.var] - | ZVAL_COPY_CTOR_FUNC op_array->filename, opline->lineno - | jmp >3 - |.code - |3: - } + | SEPARATE_ZVAL_NOREF FP + opline->op1.var, op1_info, 1, op_array->filename, opline->lineno return zend_jit_math_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, FP, opline->op1.var, op1_info, opline->op2_type, opline->op2, FP, opline->op2.var, op2_info, FP, opline->op1.var, OP1_DEF_INFO()); } @@ -3856,9 +3823,9 @@ static int zend_jit_cmp(dasm_State **Dst, const zend_op *opline, int b, int *opn } | EXT_CALL compare_function, r0 || if (opline->opcode != ZEND_CASE) { - | FREE_OP opline->op1_type, opline->op1, op1_info, op_array, opline->lineno + | FREE_OP opline->op1_type, opline->op1, op1_info, 0, op_array, opline->lineno || } - | FREE_OP opline->op2_type, opline->op2, op2_info, op_array, opline->lineno + | FREE_OP opline->op2_type, opline->op2, op2_info, 0, op_array, opline->lineno || if (zend_may_throw(opline, op_array, ssa)) { || zend_jit_check_exception(Dst); || } @@ -4456,7 +4423,7 @@ static int zend_jit_send_val(dasm_State **Dst, const zend_op *opline, zend_op_ar | EXT_CALL zend_throw_error, r0 | add r4, 16 |.endif - | FREE_OP opline->op1_type, opline->op1, op1_info, op_array, opline->lineno + | FREE_OP opline->op1_type, opline->op1, op1_info, 0, op_array, opline->lineno | jmp ->exception_handler |.code } @@ -4997,8 +4964,8 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, const zend_op *opline, zend } |9: // END - | FREE_OP opline->op2_type, opline->op2, op2_info, op_array, opline->lineno - | FREE_OP opline->op1_type, opline->op1, op1_info, op_array, opline->lineno + | FREE_OP opline->op2_type, opline->op2, op2_info, 0, op_array, opline->lineno + | FREE_OP opline->op1_type, opline->op1, op1_info, 0, op_array, opline->lineno if (zend_may_throw(opline, op_array, ssa)) { if (!zend_jit_check_exception(Dst)) { From 0040f2dae1763d5b4a32120b2b9e93065bd54c04 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 4 Oct 2016 03:08:32 +0300 Subject: [PATCH 269/569] Fixed VM interrupt handling on x86 --- ext/opcache/jit/zend_jit_x86.dasc | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 4196f8aa416d3..ee08bb58c3dcf 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -1193,10 +1193,13 @@ static int zend_jit_interrupt_handler_stub(dasm_State **Dst) | //zend_interrupt_function(execute_data); |.if X64 | mov CARG1, FP + | EXT_CALL zend_interrupt_function, r0 |.else + | sub r4, 12 | push FP - |.endif | EXT_CALL zend_interrupt_function, r0 + | add r4, 16 + |.endif | //ZEND_VM_ENTER(); | //execute_data = EG(current_execute_data); | mov FP, aword [&EG(current_execute_data)] @@ -4365,8 +4368,8 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar | // HANDLE_EXCEPTION(); | jmp ->exception_handler |.code - // TODO: Interrupt handling ??? - //return zend_jit_check_timeout(Dst, opline + 1); + // TODO: Can we avoid checking for interrupts after each call ??? + return zend_jit_check_timeout(Dst, opline + 1); } return 1; From be3ceb0ecb5f681b7d217c744341bd2a88358728 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 4 Oct 2016 17:45:31 +0300 Subject: [PATCH 270/569] Implemented JIT for ISSET_ISEMPY_DIM_OBJ (only for isset() part) --- ext/opcache/jit/zend_jit.c | 5 + ext/opcache/jit/zend_jit_disasm_x86.c | 2 + ext/opcache/jit/zend_jit_helpers.c | 99 ++++++++++++ ext/opcache/jit/zend_jit_x86.dasc | 211 +++++++++++++++++++++++++- 4 files changed, 312 insertions(+), 5 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 9a33f9ad7a71c..349d34d2d486f 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -1103,6 +1103,11 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) goto jit_failure; } break; + case ZEND_ISSET_ISEMPTY_DIM_OBJ: + if (!zend_jit_isset_isempty_dim(&dasm_state, opline, b, &i, op_array, ssa)) { + goto jit_failure; + } + break; /* case ZEND_FETCH_OBJ_R: if (!zend_jit_fetch_obj_r(&dasm_state, opline, op_array, ssa)) { diff --git a/ext/opcache/jit/zend_jit_disasm_x86.c b/ext/opcache/jit/zend_jit_disasm_x86.c index a9760f3526d8a..2209cc3dc4195 100644 --- a/ext/opcache/jit/zend_jit_disasm_x86.c +++ b/ext/opcache/jit/zend_jit_disasm_x86.c @@ -398,6 +398,7 @@ static int zend_jit_disasm_init(void) REGISTER_HELPER(zend_jit_undefined_op_helper); REGISTER_HELPER(zend_jit_fetch_dim_r_helper); REGISTER_HELPER(zend_jit_fetch_dim_is_helper); + REGISTER_HELPER(zend_jit_fetch_dim_isset_helper); REGISTER_HELPER(zend_jit_fetch_dim_str_r_helper); REGISTER_HELPER(zend_jit_fetch_dim_str_is_helper); REGISTER_HELPER(zend_jit_fetch_dim_obj_r_helper); @@ -409,6 +410,7 @@ static int zend_jit_disasm_init(void) REGISTER_HELPER(zend_jit_assign_dim_sub_helper); REGISTER_HELPER(zend_jit_assign_dim_mul_helper); REGISTER_HELPER(zend_jit_assign_dim_div_helper); + REGISTER_HELPER(zend_jit_isset_dim_helper); REGISTER_HELPER(zend_jit_free_call_frame); REGISTER_HELPER(zend_jit_zval_copy_unref_helper); REGISTER_HELPER(zend_jit_new_ref_helper); diff --git a/ext/opcache/jit/zend_jit_helpers.c b/ext/opcache/jit/zend_jit_helpers.c index ded3ab23d0dfc..585f04e45a0ba 100644 --- a/ext/opcache/jit/zend_jit_helpers.c +++ b/ext/opcache/jit/zend_jit_helpers.c @@ -323,6 +323,67 @@ static void ZEND_FASTCALL zend_jit_fetch_dim_is_helper(zend_array *ht, zval *dim ZVAL_NULL(result); } +static int ZEND_FASTCALL zend_jit_fetch_dim_isset_helper(zend_array *ht, zval *dim) +{ + zend_long hval; + zend_string *offset_key; + zval *retval; + + if (Z_TYPE_P(dim) == IS_REFERENCE) { + dim = Z_REFVAL_P(dim); + } + + switch (Z_TYPE_P(dim)) { + case IS_UNDEF: + zend_jit_undefined_op_helper(EG(current_execute_data)->opline->op2.var); + /* break missing intentionally */ + case IS_NULL: + offset_key = ZSTR_EMPTY_ALLOC(); + goto str_index; + case IS_DOUBLE: + hval = zend_dval_to_lval(Z_DVAL_P(dim)); + goto num_index; + case IS_RESOURCE: + //zend_error(E_NOTICE, "Resource ID#%d used as offset, casting to integer (%d)", Z_RES_HANDLE_P(dim), Z_RES_HANDLE_P(dim)); + hval = Z_RES_HANDLE_P(dim); + goto num_index; + case IS_FALSE: + hval = 0; + goto num_index; + case IS_TRUE: + hval = 1; + goto num_index; + default: + zend_error(E_WARNING, "Illegal offset type in isset or empty"); + return 0; + } + +str_index: + retval = zend_hash_find(ht, offset_key); + if (retval) { + /* support for $GLOBALS[...] */ + if (UNEXPECTED(Z_TYPE_P(retval) == IS_INDIRECT)) { + retval = Z_INDIRECT_P(retval); + } + if (UNEXPECTED(Z_TYPE_P(retval) == IS_REFERENCE)) { + retval = Z_REFVAL_P(retval); + } + return (Z_TYPE_P(retval) > IS_NULL); + } else { + return 0; + } + +num_index: + ZEND_HASH_INDEX_FIND(ht, hval, retval, num_undef); + if (UNEXPECTED(Z_TYPE_P(retval) == IS_REFERENCE)) { + retval = Z_REFVAL_P(retval); + } + return (Z_TYPE_P(retval) > IS_NULL); + +num_undef: + return 0; +} + static zval* ZEND_FASTCALL zend_jit_fetch_dim_rw_helper(zend_array *ht, zval *dim) { zend_long hval; @@ -1076,6 +1137,44 @@ static void ZEND_FASTCALL zend_jit_assign_dim_div_helper(zval *container, zval * } } +static int ZEND_FASTCALL zend_jit_isset_dim_helper(zval *container, zval *offset) +{ + if (UNEXPECTED(Z_TYPE_P(offset) == IS_UNDEF)) { + zend_jit_undefined_op_helper(EG(current_execute_data)->opline->op2.var); + offset = &EG(uninitialized_zval); + } + + if (EXPECTED(Z_TYPE_P(container) == IS_OBJECT)) { + if (EXPECTED(Z_OBJ_HT_P(container)->has_dimension)) { + return Z_OBJ_HT_P(container)->has_dimension(container, offset, 0); + } else { + zend_error(E_NOTICE, "Trying to check element of non-array"); + } + } else if (EXPECTED(Z_TYPE_P(container) == IS_STRING)) { /* string offsets */ + zend_long lval; + + if (EXPECTED(Z_TYPE_P(offset) == IS_LONG)) { + lval = Z_LVAL_P(offset); +isset_str_offset: + if (UNEXPECTED(lval < 0)) { /* Handle negative offset */ + lval += (zend_long)Z_STRLEN_P(container); + } + if (EXPECTED(lval >= 0) && (size_t)lval < Z_STRLEN_P(container)) { + return 1; + } + } else { + ZVAL_DEREF(offset); + if (Z_TYPE_P(offset) < IS_STRING /* simple scalar types */ + || (Z_TYPE_P(offset) == IS_STRING /* or numeric string */ + && IS_LONG == is_numeric_string(Z_STRVAL_P(offset), Z_STRLEN_P(offset), NULL, NULL, 0))) { + lval = zval_get_long(offset); + goto isset_str_offset; + } + } + } + return 0; +} + static void ZEND_FASTCALL zend_jit_free_call_frame(zend_execute_data *call) { zend_vm_stack_free_call_frame(call); diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index ee08bb58c3dcf..e6c4a856d171d 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -96,6 +96,8 @@ static void* dasm_labels[zend_lb_MAX]; #define IS_SIGNED_32BIT(val) ((((intptr_t)(val)) <= 0x7fffffff) && (((intptr_t)(val)) >= (-2147483647 - 1))) +#define BP_JIT_IS 6 + |.macro LOAD_ADDR, reg, addr |.if X64 ||if (IS_32BIT(addr)) { @@ -2175,12 +2177,25 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o | cmp r0, val |.else | cmp dword [FCARG1a + offsetof(zend_array, nNumUsed)], val - |.endif - | jle >2 // NOT_FOUND + |.endif + if (type == BP_JIT_IS) { + | jle >9 // NOT_FOUND + } else { + | jle >2 // NOT_FOUND + } | // _ret = &_ht->arData[_h].val; | mov r0, aword [FCARG1a + offsetof(zend_array, arData)] | add r0, val * sizeof(Bucket) - | IF_NOT_Z_TYPE r0, IS_UNDEF, >8 + if (type == BP_JIT_IS) { + | IF_NOT_Z_TYPE r0, IS_REFERENCE, >2 + | GET_Z_PTR r0, r0 + | add r0, 8 + |2: + | cmp byte [r0 + 8], IS_NULL + | jg >8 + } else { + | IF_NOT_Z_TYPE r0, IS_UNDEF, >8 + } } } else { | // ZEND_HASH_INDEX_FIND(ht, hval, retval, num_undef); @@ -2193,7 +2208,11 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o |.else | cmp dword [FCARG1a + offsetof(zend_array, nNumUsed)], FCARG2a |.endif - | jle >2 // NOT_FOUND + if (type == BP_JIT_IS) { + | jle >9 // NOT_FOUND + } else { + | jle >2 // NOT_FOUND + } | // _ret = &_ht->arData[_h].val; | mov r0, FCARG2a |.if X64 @@ -2202,10 +2221,42 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o | imul r0, sizeof(Bucket) |.endif | add r0, aword [FCARG1a + offsetof(zend_array, arData)] - | IF_NOT_Z_TYPE r0, IS_UNDEF, >8 + if (type == BP_JIT_IS) { + | IF_NOT_Z_TYPE r0, IS_REFERENCE, >2 + | GET_Z_PTR r0, r0 + | add r0, 8 + |2: + | cmp byte [r0 + 8], IS_NULL + | jg >8 + } else { + | IF_NOT_Z_TYPE r0, IS_UNDEF, >8 + } } } switch (type) { + case BP_JIT_IS: + if (op1_info & MAY_BE_ARRAY_KEY_LONG) { + if (opline->op2_type == IS_CONST) { + zend_long val = Z_LVAL_P(RT_CONSTANT(op_array, opline->op2)); + if (val >= 0 && val < HT_MAX_SIZE) { + | jmp >9 // NOT_FOUND + } + } else { + | jmp >9 // NOT_FOUND + } + |1: + } + | EXT_CALL zend_hash_index_find, r0 + | test r0, r0 + | jz >9 // NOT_FOUND + | IF_NOT_Z_TYPE r0, IS_REFERENCE, >1 + | GET_Z_PTR r0, r0 + | add r0, 8 + |1: + | cmp byte [r0 + 8], IS_NULL + | jg >8 + | jmp >9 // NOT_FOUND + break; case BP_VAR_R: case BP_VAR_IS: case BP_VAR_UNSET: @@ -2313,6 +2364,26 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o | LONG_LOAD FCARG2a, opline->op2_type, opline->op2 | // retval = zend_hash_find(ht, offset_key); switch (type) { + case BP_JIT_IS: + if (opline->op2_type != IS_CONST) { + | EXT_CALL zend_jit_symtable_find, r0 + } else { + | EXT_CALL zend_hash_find, r0 + } + | test r0, r0 + | jz >9 // NOT_FOUND + | // if (UNEXPECTED(Z_TYPE_P(retval) == IS_INDIRECT)) + | IF_NOT_Z_TYPE r0, IS_INDIRECT, >1 + | GET_Z_PTR r0, r0 + |1: + | IF_NOT_Z_TYPE r0, IS_REFERENCE, >1 + | GET_Z_PTR r0, r0 + | add r0, 8 + |1: + | cmp byte [r0 + 8], IS_NULL + | jg >8 + | jmp >9 + break; case BP_VAR_R: case BP_VAR_IS: case BP_VAR_UNSET: @@ -2405,6 +2476,12 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o | EXT_CALL zend_jit_fetch_dim_r_helper, r0 | jmp >9 break; + case BP_JIT_IS: + | EXT_CALL zend_jit_fetch_dim_isset_helper, r0 + | test r0, r0 + | jne >8 + | jmp >9 + break; case BP_VAR_IS: case BP_VAR_UNSET: |.if X64 @@ -4983,6 +5060,130 @@ fallback: return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); } +static int zend_jit_isset_isempty_dim(dasm_State **Dst, const zend_op *opline, int b, int *opnum, zend_op_array *op_array, zend_ssa *ssa) +{ + uint32_t op1_info, op2_info, res_info; + + if (!ssa->ops || !ssa->var_info || !(opline->extended_value & ZEND_ISSET)) { + // TODO: support for empty() ??? + goto fallback; + } + + op1_info = OP1_INFO(); + op2_info = OP2_INFO(); + res_info = RES_INFO(); + + if (op1_info & MAY_BE_REF) { + | lea FCARG1a, [FP + opline->op1.var] + | ZVAL_DEREF FCARG1a, op1_info + } + + if (op1_info & MAY_BE_ARRAY) { + if (op1_info & MAY_BE_REF) { + | IF_NOT_Z_TYPE FCARG1a, IS_ARRAY, >7 + | GET_Z_PTR FCARG1a, FCARG1a + } else { + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { + | IF_NOT_Z_TYPE FP + opline->op1.var, IS_ARRAY, >7 + } + | LONG_LOAD FCARG1a, opline->op1_type, opline->op1 + } + if (!zend_jit_fetch_dimension_address_inner(Dst, opline, op_array, BP_JIT_IS, op1_info, op2_info, 8, 9)) { + return 0; + } + } + + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY)) { + if (op1_info & MAY_BE_ARRAY) { + |.cold_code + |7: + } + + | SAVE_VALID_OPLINE opline + if (!(op1_info & MAY_BE_REF)) { + | LOAD_ZVAL_ADDR FCARG1a, opline->op1_type, opline->op1 + } + | LOAD_ZVAL_ADDR FCARG2a, opline->op2_type, opline->op2 + | EXT_CALL zend_jit_isset_dim_helper, r0 + | test r0, r0 + | jz >9 + | jmp >8 + + if (op1_info & MAY_BE_ARRAY) { + |.code + } + } + + |8: + | FREE_OP opline->op2_type, opline->op2, op2_info, 0, op_array, opline->lineno + | FREE_OP opline->op1_type, opline->op1, op1_info, 0, op_array, opline->lineno + if (zend_may_throw(opline, op_array, ssa)) { + if (!zend_jit_check_exception(Dst)) { + return 0; + } + } + if (opline->extended_value & ZEND_ISSET) { + if ((opline+1)->opcode == ZEND_JMPZ) { + unsigned int target_label = ssa->cfg.blocks[b].successors[1]; + | jmp =>target_label + } else if ((opline+1)->opcode == ZEND_JMPNZ) { + unsigned int target_label = ssa->cfg.blocks[b].successors[0]; + | jmp =>target_label + } else if ((opline+1)->opcode == ZEND_JMPZNZ) { + unsigned int target_label = ssa->cfg.blocks[b].successors[1]; + | jmp =>target_label + } else { + | SET_Z_TYPE_INFO FP + opline->result.var, IS_TRUE + | jmp >8 + } + } else { + | //???? + | int3 + } + + |9: // not found + | FREE_OP opline->op2_type, opline->op2, op2_info, 0, op_array, opline->lineno + | FREE_OP opline->op1_type, opline->op1, op1_info, 0, op_array, opline->lineno + if (zend_may_throw(opline, op_array, ssa)) { + if (!zend_jit_check_exception(Dst)) { + return 0; + } + } + if (opline->extended_value & ZEND_ISSET) { + if ((opline+1)->opcode == ZEND_JMPZ) { + unsigned int target_label = ssa->cfg.blocks[b].successors[0]; + | jmp =>target_label + } else if ((opline+1)->opcode == ZEND_JMPNZ) { + unsigned int target_label = ssa->cfg.blocks[b].successors[1]; + | jmp =>target_label + } else if ((opline+1)->opcode == ZEND_JMPZNZ) { + unsigned int target_label = ssa->cfg.blocks[b].successors[0]; + | jmp =>target_label + } else { + | SET_Z_TYPE_INFO FP + opline->result.var, IS_FALSE + } + } else { + | //???? + | int3 + } + + |8: + + if (((opline+1)->opcode == ZEND_JMPZ || + (opline+1)->opcode == ZEND_JMPNZ || + (opline+1)->opcode == ZEND_JMPZNZ) && + (opline+1)->op1_type == IS_TMP_VAR && + (opline+1)->op1.var == opline->result.var) { + (*opnum)++; + } + + return 1; + +fallback: + /* fallback to subroutine threading */ + return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); +} + static int zend_jit_bind_global(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) { uint32_t op1_info; From 1d9f1bbc7127eca16e612fb6d779825a35e0fd46 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 6 Oct 2016 11:41:21 +0300 Subject: [PATCH 271/569] Use inferred information about reference-counters to avoid dead code generation. --- ext/opcache/jit/zend_jit.c | 13 +- ext/opcache/jit/zend_jit_x86.dasc | 311 ++++++++++++++++++++---------- 2 files changed, 222 insertions(+), 102 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 349d34d2d486f..f68ad77be88cf 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -35,6 +35,17 @@ //#define CONTEXT_THREADED_JIT #define PREFER_MAP_32BIT //#define ZEND_RUNTIME_JIT +#define ZEND_JIT_USE_RC_INFERENCE + +#ifdef ZEND_JIT_USE_RC_INFERENCE +# define ZEND_SSA_RC_INFERENCE_FLAG ZEND_SSA_RC_INFERENCE +# define RC_MAY_BE_1(info) (((info) & (MAY_BE_RC1|MAY_BE_REF)) != 0) +# define RC_MAY_BE_N(info) (((info) & (MAY_BE_RCN|MAY_BE_REF)) != 0) +#else +# define ZEND_SSA_RC_INFERENCE_FLAG 0 +# define RC_MAY_BE_1(info) 1 +# define RC_MAY_BE_N(info) 1 +#endif #define JIT_PREFIX "JIT$" #define JIT_STUB_PREFIX "JIT$$" @@ -819,7 +830,7 @@ static int zend_may_overflow(const zend_op *opline, zend_op_array *op_array, zen static int zend_jit_op_array_analyze1(zend_op_array *op_array, zend_script *script, zend_ssa *ssa, uint32_t *flags) { - if (zend_build_cfg(&CG(arena), op_array, ZEND_CFG_STACKLESS | ZEND_CFG_RECV_ENTRY | ZEND_RT_CONSTANTS | ZEND_CFG_NO_ENTRY_PREDECESSORS | ZEND_SSA_RC_INFERENCE, &ssa->cfg, flags) != SUCCESS) { + if (zend_build_cfg(&CG(arena), op_array, ZEND_CFG_STACKLESS | ZEND_CFG_RECV_ENTRY | ZEND_RT_CONSTANTS | ZEND_CFG_NO_ENTRY_PREDECESSORS | ZEND_SSA_RC_INFERENCE_FLAG, &ssa->cfg, flags) != SUCCESS) { return FAILURE; } diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index e6c4a856d171d..9459dc3427ebf 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -902,8 +902,8 @@ static void* dasm_labels[zend_lb_MAX]; |.endmacro |.macro ZVAL_PTR_DTOR, zv, op_info, gc, cold, safe, filename, lineno -|| if (op_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { -|| if (op_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { +|| if ((op_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { +|| if ((op_info) & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { | // if (Z_REFCOUNTED_P(cv)) { || if (cold) { | IF_Z_REFCOUNTED zv, >1 @@ -916,22 +916,30 @@ static void* dasm_labels[zend_lb_MAX]; | // if (!Z_DELREF_P(cv)) { | GET_Z_PTR FCARG1a, zv | GC_DELREF FCARG1a -| jnz >1 -|| if (safe) { -| // ZVAL_NULL(cv); -| SET_Z_TYPE_INFO zv, IS_NULL -|| } -| // zval_dtor_func(r); -| ZVAL_DTOR_FUNC filename, lineno -|| if (gc && (op_info & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT))) { -| jmp >2 -|| } +|| if (RC_MAY_BE_1(op_info)) { +|| if (RC_MAY_BE_N(op_info)) { +|| if (gc && RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT))) { +| jnz >1 +|| } else { +| jnz >2 +|| } +|| } +|| if (safe) { +| // ZVAL_NULL(cv); +| SET_Z_TYPE_INFO zv, IS_NULL +|| } +| // zval_dtor_func(r); +| ZVAL_DTOR_FUNC filename, lineno +|| if (gc && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT))) { +| jmp >2 +|| } |1: -|| if (gc && (op_info & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT))) { -|| if (op_info & MAY_BE_REF) { +|| } +|| if (gc && RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT))) { +|| if ((op_info) & MAY_BE_REF) { | lea r1, [zv] | // ZVAL_DEREF(z); -| ZVAL_DEREF r1, op_info +| ZVAL_DEREF r1, (op_info) | // if (Z_COLLECTABLE_P(z)) | IF_NOT_Z_FLAGS r1, IS_TYPE_COLLECTABLE, >2 | // if (UNEXPECTED(!Z_GC_INFO_P(z))) @@ -946,11 +954,11 @@ static void* dasm_labels[zend_lb_MAX]; || } | // gc_possible_root(Z_COUNTED_P(z)) | EXT_CALL gc_possible_root, r0 -|| if (cold && (op_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE)))) { +|| if (cold && ((op_info) & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE)))) { | jmp >2 |.code || } -|| } else if (cold && (!gc || (op_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))))) { +|| } else if (cold && (!gc || ((op_info) & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))))) { | jmp >2 |.code || } @@ -965,7 +973,7 @@ static void* dasm_labels[zend_lb_MAX]; |.endmacro |.macro SEPARATE_ZVAL_NOREF, zv, op_info, cold, filename, lineno -|| if (op_info & (MAY_BE_STRING|MAY_BE_ARRAY)) { +|| if ((op_info & (MAY_BE_STRING|MAY_BE_ARRAY)) && RC_MAY_BE_N(op_info)) { || if (cold) { | IF_Z_FLAGS zv, IS_TYPE_COPYABLE + IS_TYPE_IMMUTABLE, >1 |.cold_code @@ -974,8 +982,10 @@ static void* dasm_labels[zend_lb_MAX]; | IF_Z_FLAGS zv, IS_TYPE_COPYABLE + IS_TYPE_IMMUTABLE, >2 || } | GET_Z_PTR r0, zv -| cmp dword [r0], 1 // if (GC_REFCOUNTED() > 1) -| jbe >3 +|| if (RC_MAY_BE_1(op_info)) { +| cmp dword [r0], 1 // if (GC_REFCOUNTED() > 1) +| jbe >3 +|| } | IF_Z_FLAGS zv, IS_TYPE_IMMUTABLE, >1 | GC_DELREF r0 |1: @@ -990,7 +1000,7 @@ static void* dasm_labels[zend_lb_MAX]; |.endmacro |.macro SEPARATE_ZVAL_NOREF_REG, op_info, cold, filename, lineno -|| if (op_info & (MAY_BE_STRING|MAY_BE_ARRAY)) { +|| if ((op_info & (MAY_BE_STRING|MAY_BE_ARRAY)) && RC_MAY_BE_N(op_info)) { || if (cold) { | IF_Z_FLAGS FCARG1a, IS_TYPE_COPYABLE + IS_TYPE_IMMUTABLE, >1 |.cold_code @@ -999,8 +1009,10 @@ static void* dasm_labels[zend_lb_MAX]; | IF_NOT_Z_FLAGS FCARG1a, IS_TYPE_COPYABLE + IS_TYPE_IMMUTABLE, >2 || } | GET_Z_PTR r0, FCARG1a -| cmp dword [r0], 1 // if (GC_REFCOUNTED() > 1) -| jbe >2 +|| if (RC_MAY_BE_1(op_info)) { +| cmp dword [r0], 1 // if (GC_REFCOUNTED() > 1) +| jbe >2 +|| } | IF_Z_FLAGS FCARG1a, IS_TYPE_IMMUTABLE, >1 | GC_DELREF r0 |1: @@ -1017,53 +1029,66 @@ static void* dasm_labels[zend_lb_MAX]; |.macro SEPARATE_ARRAY, op_type, op, op_info, cold, filename, lineno | LONG_LOAD FCARG1a, op_type, op -| cmp dword [FCARG1a], 1 // if (GC_REFCOUNTED() > 1) -|| if (cold) { -| ja >1 +|| if (RC_MAY_BE_N(op_info)) { +|| if (RC_MAY_BE_1(op_info)) { +| cmp dword [FCARG1a], 1 // if (GC_REFCOUNTED() > 1) +|| if (cold) { +| ja >1 |.cold_code |1: -|| } else { -| jbe >2 -|| } -| IF_Z_FLAGS FP + op.var, IS_TYPE_IMMUTABLE, >1 -| GC_DELREF FCARG1a +|| } else { +| jbe >2 +|| } +|| } +| IF_Z_FLAGS FP + op.var, IS_TYPE_IMMUTABLE, >1 +| GC_DELREF FCARG1a |1: -| LOAD_ZVAL_ADDR FCARG1a, op_type, op -| ZVAL_COPY_CTOR_FUNC filename, lineno -| LONG_LOAD FCARG1a, op_type, op -|| if (cold) { -| jmp >2 +| LOAD_ZVAL_ADDR FCARG1a, op_type, op +| ZVAL_COPY_CTOR_FUNC filename, lineno +| LONG_LOAD FCARG1a, op_type, op +|| if (RC_MAY_BE_1(op_info)) { +|| if (cold) { +| jmp >2 |.code -|| } +|| } +|| } |2: +|| } |.endmacro |.macro SEPARATE_ARRAY_REG, op_info, cold, filename, lineno -| GET_Z_PTR r0, FCARG1a -| cmp dword [r0], 1 // if (GC_REFCOUNTED() > 1) -|| if (cold) { -| ja >1 +|| if (RC_MAY_BE_N(op_info)) { +| GET_Z_PTR r0, FCARG1a +|| if (RC_MAY_BE_1(op_info)) { +| cmp dword [r0], 1 // if (GC_REFCOUNTED() > 1) +|| if (cold) { +| ja >1 |.cold_code |1: -|| } else { -| jbe >2 -|| } -| IF_Z_FLAGS FCARG1a, IS_TYPE_IMMUTABLE, >1 -| GC_DELREF r0 +|| } else { +| jbe >2 +|| } +|| } +| IF_Z_FLAGS FCARG1a, IS_TYPE_IMMUTABLE, >1 +| GC_DELREF r0 |1: -| mov aword [r4], FCARG1a // save -| ZVAL_COPY_CTOR_FUNC filename, lineno -| mov FCARG1a, aword [r4] // restore -| GET_Z_PTR FCARG1a, FCARG1a -|| if (cold) { -| jmp >2 +| mov aword [r4], FCARG1a // save +| ZVAL_COPY_CTOR_FUNC filename, lineno +| mov FCARG1a, aword [r4] // restore +| GET_Z_PTR FCARG1a, FCARG1a +|| if (RC_MAY_BE_1(op_info)) { +|| if (cold) { +| jmp >2 |.code -|| } -| mov FCARG1a, r0 +|| } +|| } +| mov FCARG1a, r0 |2: +|| } else { +| GET_Z_PTR FCARG1a, FCARG1a +|| } |.endmacro - |.macro EFREE_SIZE, ptr, size, op_array, opline | mov FCARG1a, ptr || if (ZEND_DEBUG) { @@ -1256,7 +1281,7 @@ static int zend_jit_leave_throw_stub(dasm_State **Dst) | mov r0, OP:IP->result.var |.endif | add r0, FP - | ZVAL_PTR_DTOR r0, MAY_BE_ANY, 1, 0, 0, NULL, 0 + | ZVAL_PTR_DTOR r0, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 1, 0, 0, NULL, 0 |5: | // opline = EG(exception_op); | LOAD_ADDR IP, &EG(exception_op) @@ -1333,8 +1358,31 @@ static int zend_jit_handler(dasm_State **Dst, const zend_op *opline, int may_thr zend_jit_check_exception(Dst); } last_valid_opline++; - if ((opline+1)->opcode == ZEND_OP_DATA) { - last_valid_opline++; + + /* Skip the following OP_DATA */ + switch (opline->opcode) { + case ZEND_ASSIGN_DIM: + case ZEND_ASSIGN_OBJ: + last_valid_opline++; + break; + case ZEND_ASSIGN_ADD: + case ZEND_ASSIGN_SUB: + case ZEND_ASSIGN_MUL: + case ZEND_ASSIGN_DIV: + case ZEND_ASSIGN_MOD: + case ZEND_ASSIGN_SL: + case ZEND_ASSIGN_SR: + case ZEND_ASSIGN_CONCAT: + case ZEND_ASSIGN_BW_OR: + case ZEND_ASSIGN_BW_AND: + case ZEND_ASSIGN_BW_XOR: + case ZEND_ASSIGN_POW: + if (opline->extended_value) { + last_valid_opline++; + } + break; + default: + break; } return 1; @@ -1468,7 +1516,7 @@ static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, zend_op_arr goto fallback; } - if (op1_info & (MAY_BE_ANY-MAY_BE_LONG)) { + if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)-MAY_BE_LONG)) { | IF_NOT_Z_TYPE FP + opline->op1.var, IS_LONG, >2 } || if ((opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) && @@ -1530,6 +1578,7 @@ static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, zend_op_arr | mov FCARG1d, opline->op1.var | EXT_CALL zend_jit_undefined_op_helper, r0 | SET_Z_TYPE_INFO FP + opline->op1.var, IS_NULL + || op1_info |= MAY_BE_NULL; || } |2: | lea FCARG1a, [FP + opline->op1.var] @@ -2679,37 +2728,45 @@ static int zend_jit_assign_to_variable(dasm_State **Dst, | // TODO: support for assignment to itself | GET_Z_PTR r0, Ra(var_reg)+var_offset | GC_DELREF r0 - | jnz >4 - | mov aword [r4], r0 // save - if (!zend_jit_simple_assign(Dst, opline, op_array, ssa, var_reg, var_offset, var_info, val_type, val, val_info, var2, in_cold)) { - return 0; - } - | mov FCARG1a, aword [r4] // restore - | ZVAL_DTOR_FUNC op_array->filename, opline->lineno - | jmp >3 - |4: - | IF_NOT_Z_FLAGS Ra(var_reg)+var_offset, IS_TYPE_COLLECTABLE, >5 - if (var_reg == FP) { - | GET_Z_PTR FCARG1a, Ra(var_reg)+var_offset - | IF_GC_INFO FCARG1a, >5 - } else if (var_reg != FCARG1a) { - | GET_Z_PTR FCARG1a, Ra(var_reg)+var_offset - | IF_GC_INFO FCARG1a, >5 - | mov [r4], Ra(var_reg) // save - } else { - | GET_Z_PTR r0, Ra(var_reg)+var_offset - | IF_GC_INFO r0, >5 - | mov [r4], Ra(var_reg) // save - | mov FCARG1a, r0 - } - | EXT_CALL gc_possible_root, r0 - if (var_reg != FP) { - | mov Ra(var_reg), [r4] // restore + if (RC_MAY_BE_1(var_info)) { + if (RC_MAY_BE_N(var_info)) { + | jnz >4 + } + | mov aword [r4], r0 // save + if (!zend_jit_simple_assign(Dst, opline, op_array, ssa, var_reg, var_offset, var_info, val_type, val, val_info, var2, in_cold)) { + return 0; + } + | mov FCARG1a, aword [r4] // restore + | ZVAL_DTOR_FUNC op_array->filename, opline->lineno + | jmp >3 + |4: } - if (in_cold) { - | jmp >5 + if (RC_MAY_BE_N(var_info)) { + | IF_NOT_Z_FLAGS Ra(var_reg)+var_offset, IS_TYPE_COLLECTABLE, >5 + if (var_reg == FP) { + | GET_Z_PTR FCARG1a, Ra(var_reg)+var_offset + | IF_GC_INFO FCARG1a, >5 + } else if (var_reg != FCARG1a) { + | GET_Z_PTR FCARG1a, Ra(var_reg)+var_offset + | IF_GC_INFO FCARG1a, >5 + | mov [r4], Ra(var_reg) // save + } else { + | GET_Z_PTR r0, Ra(var_reg)+var_offset + | IF_GC_INFO r0, >5 + | mov [r4], Ra(var_reg) // save + | mov FCARG1a, r0 + } + | EXT_CALL gc_possible_root, r0 + if (var_reg != FP) { + | mov Ra(var_reg), [r4] // restore + } + if (in_cold) { + | jmp >5 + |.code + } + } else if (in_cold) { |.code - } + } |5: } @@ -2966,6 +3023,14 @@ static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, zend_op_ | PUSH_ZVAL_ADDR (opline+1)->op1_type, (opline+1)->op1, r0 |.endif | EXT_CALL zend_jit_assign_dim_helper, r0 + +#ifdef ZEND_JIT_USE_RC_INFERENCE + if (((opline+1)->op1_type & (IS_TMP_VAR|IS_VAR)) && (val_info & MAY_BE_RC1)) { + /* ASSIGN_DIM may increase refcount of the value */ + val_info |= MAY_BE_RCN; + } +#endif + | FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, 0, op_array, opline->lineno } @@ -2977,6 +3042,13 @@ static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, zend_op_ } } +#ifdef ZEND_JIT_USE_RC_INFERENCE + if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY|MAY_BE_OBJECT))) { + /* ASSIGN_DIM may increase refcount of the key */ + op2_info |= MAY_BE_RCN; + } +#endif + |9: | FREE_OP opline->op2_type, opline->op2, op2_info, 0, op_array, opline->lineno @@ -4396,7 +4468,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar for (i = 0; i < call_info->num_args; i++ ) { uint32_t offset = (uint32_t)(uintptr_t)ZEND_CALL_VAR_NUM(NULL, i); | mov r0, aword [r4] // restore - | ZVAL_PTR_DTOR r0 + offset, -1, 0, 1, 1, op_array->filename, opline->lineno + | ZVAL_PTR_DTOR r0 + offset, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN, 0, 1, 1, op_array->filename, opline->lineno } | // zend_vm_stack_free_call_frame(call); @@ -4417,10 +4489,10 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar if (func_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { | lea r0, aword [r4 + 8] } - | add r4, 16 /* revert alloca() */ if (func_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { | ZVAL_PTR_DTOR r0, func_info, 1, 1, 0, op_array->filename, opline->lineno } + | add r4, 16 /* revert alloca() */ } | // if (UNEXPECTED(EG(exception) != NULL)) { @@ -4628,6 +4700,26 @@ static int zend_jit_free_compiled_variables(dasm_State **Dst, const zend_op *opl info = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_UNDEF; } +#ifdef ZEND_JIT_USE_RC_INFERENCE + /* Refcount may be increased by RETRUN opcode */ + if ((info & MAY_BE_RC1) && !(info & MAY_BE_RCN)) { + for (j = 0; j < ssa->cfg.blocks_count; j++) { + if ((ssa->cfg.blocks[j].flags & ZEND_BB_REACHABLE) && + ssa->cfg.blocks[j].len > 0) { + const zend_op *opline = op_array->opcodes + ssa->cfg.blocks[j].start + ssa->cfg.blocks[j].len - 1; + + if (opline->opcode == ZEND_RETURN) { + if (opline->op1_type == IS_CV && + opline->op1.var == (uint32_t)(uintptr_t)(ZEND_CALL_VAR_NUM(NULL, i))) { + info |= MAY_BE_RCN; + break; + } + } + } + } + } +#endif + if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { uint32_t offset = (uint32_t)(uintptr_t)ZEND_CALL_VAR_NUM(NULL, i); | ZVAL_PTR_DTOR FP + offset, info, 1, 1, 1, op_array->filename, opline->lineno @@ -4780,15 +4872,19 @@ static int zend_jit_return(dasm_State **Dst, const zend_op *opline, zend_op_arra || } | GET_Z_PTR FCARG1a, FP + opline->op1.var | GC_DELREF FCARG1a - || if (jit_return_label >= 0) { - | jnz =>jit_return_label - || } else { - | jnz >9 + || if (RC_MAY_BE_1(op1_info)) { + || if (RC_MAY_BE_N(op1_info)) { + || if (jit_return_label >= 0) { + | jnz =>jit_return_label + || } else { + | jnz >9 + || } + || } + | //SAVE_OPLINE() + | SAVE_VALID_OPLINE opline + | ZVAL_DTOR_FUNC op_array->filename, opline->lineno + | //????mov r1, EX->return_value // reload ??? || } - | //SAVE_OPLINE() - | SAVE_VALID_OPLINE opline - | ZVAL_DTOR_FUNC op_array->filename, opline->lineno - | mov r1, EX->return_value // reload ??? || if (jit_return_label >= 0) { | jmp =>jit_return_label || } else { @@ -5044,6 +5140,13 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, const zend_op *opline, zend } |9: // END +#ifdef ZEND_JIT_USE_RC_INFERENCE + if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & MAY_BE_OBJECT)) { + /* Magic offsetGet() may increase refcount of the key */ + op2_info |= MAY_BE_RCN; + } +#endif + | FREE_OP opline->op2_type, opline->op2, op2_info, 0, op_array, opline->lineno | FREE_OP opline->op1_type, opline->op1, op1_info, 0, op_array, opline->lineno @@ -5062,7 +5165,7 @@ fallback: static int zend_jit_isset_isempty_dim(dasm_State **Dst, const zend_op *opline, int b, int *opnum, zend_op_array *op_array, zend_ssa *ssa) { - uint32_t op1_info, op2_info, res_info; + uint32_t op1_info, op2_info; if (!ssa->ops || !ssa->var_info || !(opline->extended_value & ZEND_ISSET)) { // TODO: support for empty() ??? @@ -5071,7 +5174,6 @@ static int zend_jit_isset_isempty_dim(dasm_State **Dst, const zend_op *opline, i op1_info = OP1_INFO(); op2_info = OP2_INFO(); - res_info = RES_INFO(); if (op1_info & MAY_BE_REF) { | lea FCARG1a, [FP + opline->op1.var] @@ -5114,6 +5216,13 @@ static int zend_jit_isset_isempty_dim(dasm_State **Dst, const zend_op *opline, i } } +#ifdef ZEND_JIT_USE_RC_INFERENCE + if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & MAY_BE_OBJECT)) { + /* Magic offsetExists() may increase refcount of the key */ + op2_info |= MAY_BE_RCN; + } +#endif + |8: | FREE_OP opline->op2_type, opline->op2, op2_info, 0, op_array, opline->lineno | FREE_OP opline->op1_type, opline->op1, op1_info, 0, op_array, opline->lineno @@ -5424,7 +5533,7 @@ static int zend_jit_recv_init(dasm_State **Dst, const zend_op *opline, zend_op_a |.cold_code if (has_slow & 1) { |7: - | ZVAL_PTR_DTOR FP + opline->result.var, -1, 1, 0, 0, op_array->filename, opline->lineno + | ZVAL_PTR_DTOR FP + opline->result.var, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN, 1, 0, 0, op_array->filename, opline->lineno | SET_Z_TYPE_INFO FP + opline->result.var, IS_UNDEF | jmp <4 } From 6e63f9927b13e4bae38bd310718a790daa16ddf0 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 6 Oct 2016 16:19:29 +0300 Subject: [PATCH 272/569] Allow "run-time" JIT --- ext/opcache/jit/zend_jit.c | 10 ++++++++++ ext/opcache/jit/zend_jit_x86.dasc | 10 ++++++++++ 2 files changed, 20 insertions(+) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index f68ad77be88cf..12f7ceab7595f 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -1422,6 +1422,14 @@ ZEND_API int zend_jit_script(zend_script *script) goto jit_failure; } +#ifdef ZEND_RUNTIME_JIT + for (i = 0; i < call_graph.op_arrays_count; i++) { + ZEND_SET_FUNC_INFO(call_graph.op_arrays[i], NULL); + if (zend_jit_op_array(call_graph.op_arrays[i], script) != SUCCESS) { + goto jit_failure; + } + } +#else for (i = 0; i < call_graph.op_arrays_count; i++) { info = ZEND_FUNC_INFO(call_graph.op_arrays[i]); if (info) { @@ -1461,6 +1469,8 @@ ZEND_API int zend_jit_script(zend_script *script) for (i = 0; i < call_graph.op_arrays_count; i++) { ZEND_SET_FUNC_INFO(call_graph.op_arrays[i], NULL); } +#endif + zend_arena_release(&CG(arena), checkpoint); return SUCCESS; diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 9459dc3427ebf..75bb22e555535 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -4269,6 +4269,12 @@ static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ func = call_info->callee_func; } } + + if (!func) { + // TODO: implement JIT for INIT_FCALL for unknown functions ??? + goto fallback; + } + if (func && func->type == ZEND_INTERNAL_FUNCTION) { | LOAD_ADDR r0, func } else if (func && op_array == &func->op_array) { @@ -4312,6 +4318,10 @@ static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ | mov EX->call, FCARG1a return 1; + +fallback: + /* fallback to subroutine threading */ + return zend_jit_handler(Dst, opline, 0); } static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, int call_level) From 4fd15188d37c3b6bba69e1f508d7dbb0f1a10e72 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 6 Oct 2016 18:05:59 +0300 Subject: [PATCH 273/569] Allow switching between different JIT strategies without opcache recompilation (opcache.jit = 0 - 4) --- ext/opcache/ZendAccelerator.c | 8 +- ext/opcache/ZendAccelerator.h | 1 + ext/opcache/jit/zend_jit.c | 324 +++++++++++++------------- ext/opcache/jit/zend_jit.h | 6 +- ext/opcache/jit/zend_jit_x86.dasc | 35 ++- ext/opcache/zend_accelerator_module.c | 1 + ext/opcache/zend_persist.c | 17 +- 7 files changed, 201 insertions(+), 191 deletions(-) diff --git a/ext/opcache/ZendAccelerator.c b/ext/opcache/ZendAccelerator.c index 52834edaedfe3..914cb11e0f3d9 100644 --- a/ext/opcache/ZendAccelerator.c +++ b/ext/opcache/ZendAccelerator.c @@ -2769,8 +2769,12 @@ static int accel_startup(zend_extension *extension) zend_shared_alloc_lock(); #ifdef HAVE_JIT - if (ZCG(accel_directives).jit_buffer_size) { - zend_jit_startup(ZCG(accel_directives).jit_buffer_size); + if (ZCG(accel_directives).jit_level && + ZCG(accel_directives).jit_buffer_size) { + zend_jit_startup(ZCG(accel_directives).jit_level, ZCG(accel_directives).jit_buffer_size); + } else { + ZCG(accel_directives).jit_level = 0; + ZCG(accel_directives).jit_buffer_size = 0; } #endif zend_shared_alloc_save_state(); diff --git a/ext/opcache/ZendAccelerator.h b/ext/opcache/ZendAccelerator.h index f69af7dd68707..184cabbbd76cb 100644 --- a/ext/opcache/ZendAccelerator.h +++ b/ext/opcache/ZendAccelerator.h @@ -209,6 +209,7 @@ typedef struct _zend_accel_directives { zend_bool huge_code_pages; #endif #ifdef HAVE_JIT + zend_long jit_level; zend_long jit_buffer_size; zend_long jit_debug; #endif diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 12f7ceab7595f..d71cf9297f12a 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -62,6 +62,7 @@ typedef struct _zend_jit_stub { #define JIT_STUB(name) \ {JIT_STUB_PREFIX #name, zend_jit_ ## name ## _stub} +static zend_long zend_jit_level = 0; static void *dasm_buf = NULL; static void *dasm_end = NULL; static void **dasm_ptr = NULL; @@ -94,6 +95,7 @@ ZEND_API void zend_jit_status(zval *ret) { zval stats; array_init(&stats); + add_assoc_long(&stats, "level", zend_jit_level); if (dasm_buf) { add_assoc_long(&stats, "buffer_size", (char*)dasm_end - (char*)dasm_buf); add_assoc_long(&stats, "buffer_free", (char*)dasm_end - (char*)*dasm_ptr); @@ -848,7 +850,7 @@ static int zend_jit_op_array_analyze1(zend_op_array *op_array, zend_script *scri return FAILURE; } - if ((ZEND_JIT_LEVEL >= ZEND_JIT_LEVEL_OPT_FUNC) + if ((zend_jit_level >= ZEND_JIT_LEVEL_OPT_FUNC) && ssa->cfg.blocks && op_array->last_try_catch == 0 && !(op_array->fn_flags & ZEND_ACC_GENERATOR) @@ -877,7 +879,7 @@ static int zend_jit_op_array_analyze1(zend_op_array *op_array, zend_script *scri static int zend_jit_op_array_analyze2(zend_op_array *op_array, zend_script *script, zend_ssa *ssa, uint32_t *flags) { - if ((ZEND_JIT_LEVEL >= ZEND_JIT_LEVEL_OPT_FUNC) + if ((zend_jit_level >= ZEND_JIT_LEVEL_OPT_FUNC) && ssa->cfg.blocks && op_array->last_try_catch == 0 && !(op_array->fn_flags & ZEND_ACC_GENERATOR) @@ -937,11 +939,11 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) if (ssa->cfg.split_at_recv && opline->opcode == ZEND_RECV_INIT) { if (opline > op_array->opcodes && (opline-1)->opcode == ZEND_RECV_INIT) { -#if ZEND_JIT_LEVEL < ZEND_JIT_LEVEL_OPT_FUNC - /* repeatable opcode */ - zend_jit_label(&dasm_state, b); - continue; -#endif + if (zend_jit_level < ZEND_JIT_LEVEL_INLINE) { + /* repeatable opcode */ + zend_jit_label(&dasm_state, b); + continue; + } } else { if (opline != op_array->opcodes) { zend_jit_jmp(&dasm_state, 1); @@ -993,150 +995,156 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) case ZEND_NEW: call_level++; } - switch (opline->opcode) { -#if ZEND_JIT_LEVEL >= ZEND_JIT_LEVEL_OPT_FUNC - case ZEND_PRE_INC: - case ZEND_PRE_DEC: - case ZEND_POST_INC: - case ZEND_POST_DEC: - if (!zend_jit_inc_dec(&dasm_state, opline, op_array, ssa)) { - goto jit_failure; - } - break; - case ZEND_SR: - case ZEND_SL: - if (!zend_jit_shift(&dasm_state, opline, op_array, ssa)) { - goto jit_failure; - } - break; - case ZEND_ADD: - case ZEND_SUB: - case ZEND_MUL: -// case ZEND_DIV: // TODO: check for division by zero ??? - if (!zend_jit_math(&dasm_state, opline, &i, op_array, ssa)) { - goto jit_failure; - } - break; - case ZEND_ASSIGN_ADD: - case ZEND_ASSIGN_SUB: - case ZEND_ASSIGN_MUL: -// case ZEND_ASSIGN_DIV: // TODO: check for division by zero ??? - if (!zend_jit_assign_math(&dasm_state, opline, op_array, ssa)) { - goto jit_failure; - } - break; - case ZEND_ASSIGN_DIM: - if (!zend_jit_assign_dim(&dasm_state, opline, op_array, ssa)) { - goto jit_failure; - } - break; - case ZEND_ASSIGN: - if (!zend_jit_assign(&dasm_state, opline, op_array, ssa)) { - goto jit_failure; - } - break; - case ZEND_QM_ASSIGN: - if (!zend_jit_qm_assign(&dasm_state, opline, op_array, ssa)) { - goto jit_failure; - } - break; - case ZEND_INIT_FCALL: - if (!zend_jit_init_fcall(&dasm_state, opline, op_array, call_level)) { - goto jit_failure; - } - break; - case ZEND_SEND_VAL: - case ZEND_SEND_VAL_EX: - if (!zend_jit_send_val(&dasm_state, opline, op_array, ssa)) { - goto jit_failure; - } - break; - case ZEND_SEND_VAR: - if (!zend_jit_send_var(&dasm_state, opline, op_array, ssa)) { - goto jit_failure; - } - break; - case ZEND_DO_UCALL: - case ZEND_DO_ICALL: - if (!zend_jit_do_fcall(&dasm_state, opline, op_array, ssa, call_level)) { - goto jit_failure; - } - break; - case ZEND_IS_EQUAL: - case ZEND_IS_NOT_EQUAL: - case ZEND_IS_SMALLER: - case ZEND_IS_SMALLER_OR_EQUAL: - case ZEND_CASE: - if (!zend_jit_cmp(&dasm_state, opline, b, &i, op_array, ssa)) { - goto jit_failure; - } - break; - case ZEND_TYPE_CHECK: - if (!zend_jit_type_check(&dasm_state, opline, b, &i, op_array, ssa)) { - goto jit_failure; - } - break; - case ZEND_RETURN: - if (!zend_jit_return(&dasm_state, opline, op_array, ssa)) { - goto jit_failure; - } - break; - case ZEND_JMPZ: - case ZEND_JMPNZ: - case ZEND_JMPZNZ: - if ((opline-1)->opcode == ZEND_IS_EQUAL || - (opline-1)->opcode == ZEND_IS_NOT_EQUAL || - (opline-1)->opcode == ZEND_IS_SMALLER || - (opline-1)->opcode == ZEND_IS_SMALLER_OR_EQUAL || - (opline-1)->opcode == ZEND_CASE) { - /* skip */ - } else if ((opline->opcode != ZEND_JMPZNZ) && - ((opline-1)->opcode == ZEND_IS_IDENTICAL || - (opline-1)->opcode == ZEND_IS_NOT_IDENTICAL || - (opline-1)->opcode == ZEND_ISSET_ISEMPTY_VAR || - (opline-1)->opcode == ZEND_ISSET_ISEMPTY_STATIC_PROP || - (opline-1)->opcode == ZEND_ISSET_ISEMPTY_DIM_OBJ || - (opline-1)->opcode == ZEND_ISSET_ISEMPTY_PROP_OBJ || - (opline-1)->opcode == ZEND_INSTANCEOF || - (opline-1)->opcode == ZEND_TYPE_CHECK || - (opline-1)->opcode == ZEND_DEFINED)) { - /* smart branch */ - if (!zend_jit_cond_jmp(&dasm_state, opline + 1, ssa->cfg.blocks[b].successors[0])) { + + if (zend_jit_level >= ZEND_JIT_LEVEL_INLINE) { + switch (opline->opcode) { + case ZEND_PRE_INC: + case ZEND_PRE_DEC: + case ZEND_POST_INC: + case ZEND_POST_DEC: + if (!zend_jit_inc_dec(&dasm_state, opline, op_array, ssa)) { goto jit_failure; } - } else if (!zend_jit_jmpznz(&dasm_state, opline, b, op_array, ssa)) { - goto jit_failure; - } - break; - case ZEND_FETCH_DIM_R: - case ZEND_FETCH_DIM_IS: - if (!zend_jit_fetch_dim_read(&dasm_state, opline, op_array, ssa)) { - goto jit_failure; - } - break; - case ZEND_ISSET_ISEMPTY_DIM_OBJ: - if (!zend_jit_isset_isempty_dim(&dasm_state, opline, b, &i, op_array, ssa)) { - goto jit_failure; - } - break; - /* - case ZEND_FETCH_OBJ_R: - if (!zend_jit_fetch_obj_r(&dasm_state, opline, op_array, ssa)) { - goto jit_failure; - } - break; - */ - case ZEND_BIND_GLOBAL: - if (!zend_jit_bind_global(&dasm_state, opline, op_array, ssa)) { - goto jit_failure; - } - break; - case ZEND_RECV_INIT: - if (!zend_jit_recv_init(&dasm_state, opline, op_array, (opline + 1)->opcode != ZEND_RECV_INIT, ssa)) { - goto jit_failure; - } - break; -#else + goto done; + case ZEND_SR: + case ZEND_SL: + if (!zend_jit_shift(&dasm_state, opline, op_array, ssa)) { + goto jit_failure; + } + goto done; + case ZEND_ADD: + case ZEND_SUB: + case ZEND_MUL: +// case ZEND_DIV: // TODO: check for division by zero ??? + if (!zend_jit_math(&dasm_state, opline, &i, op_array, ssa)) { + goto jit_failure; + } + goto done; + case ZEND_ASSIGN_ADD: + case ZEND_ASSIGN_SUB: + case ZEND_ASSIGN_MUL: +// case ZEND_ASSIGN_DIV: // TODO: check for division by zero ??? + if (!zend_jit_assign_math(&dasm_state, opline, op_array, ssa)) { + goto jit_failure; + } + goto done; + case ZEND_ASSIGN_DIM: + if (!zend_jit_assign_dim(&dasm_state, opline, op_array, ssa)) { + goto jit_failure; + } + goto done; + case ZEND_ASSIGN: + if (!zend_jit_assign(&dasm_state, opline, op_array, ssa)) { + goto jit_failure; + } + goto done; + case ZEND_QM_ASSIGN: + if (!zend_jit_qm_assign(&dasm_state, opline, op_array, ssa)) { + goto jit_failure; + } + goto done; + case ZEND_INIT_FCALL: + if (!zend_jit_init_fcall(&dasm_state, opline, op_array, call_level)) { + goto jit_failure; + } + goto done; + case ZEND_SEND_VAL: + case ZEND_SEND_VAL_EX: + if (!zend_jit_send_val(&dasm_state, opline, op_array, ssa)) { + goto jit_failure; + } + goto done; + case ZEND_SEND_VAR: + if (!zend_jit_send_var(&dasm_state, opline, op_array, ssa)) { + goto jit_failure; + } + goto done; + case ZEND_DO_UCALL: + case ZEND_DO_ICALL: + if (!zend_jit_do_fcall(&dasm_state, opline, op_array, ssa, call_level)) { + goto jit_failure; + } + goto done; + case ZEND_IS_EQUAL: + case ZEND_IS_NOT_EQUAL: + case ZEND_IS_SMALLER: + case ZEND_IS_SMALLER_OR_EQUAL: + case ZEND_CASE: + if (!zend_jit_cmp(&dasm_state, opline, b, &i, op_array, ssa)) { + goto jit_failure; + } + goto done; + case ZEND_TYPE_CHECK: + if (!zend_jit_type_check(&dasm_state, opline, b, &i, op_array, ssa)) { + goto jit_failure; + } + goto done; + case ZEND_RETURN: + if (!zend_jit_return(&dasm_state, opline, op_array, ssa)) { + goto jit_failure; + } + goto done; + case ZEND_JMPZ: + case ZEND_JMPNZ: + case ZEND_JMPZNZ: + if ((opline-1)->opcode == ZEND_IS_EQUAL || + (opline-1)->opcode == ZEND_IS_NOT_EQUAL || + (opline-1)->opcode == ZEND_IS_SMALLER || + (opline-1)->opcode == ZEND_IS_SMALLER_OR_EQUAL || + (opline-1)->opcode == ZEND_CASE) { + /* skip */ + } else if ((opline->opcode != ZEND_JMPZNZ) && + ((opline-1)->opcode == ZEND_IS_IDENTICAL || + (opline-1)->opcode == ZEND_IS_NOT_IDENTICAL || + (opline-1)->opcode == ZEND_ISSET_ISEMPTY_VAR || + (opline-1)->opcode == ZEND_ISSET_ISEMPTY_STATIC_PROP || + (opline-1)->opcode == ZEND_ISSET_ISEMPTY_DIM_OBJ || + (opline-1)->opcode == ZEND_ISSET_ISEMPTY_PROP_OBJ || + (opline-1)->opcode == ZEND_INSTANCEOF || + (opline-1)->opcode == ZEND_TYPE_CHECK || + (opline-1)->opcode == ZEND_DEFINED)) { + /* smart branch */ + if (!zend_jit_cond_jmp(&dasm_state, opline + 1, ssa->cfg.blocks[b].successors[0])) { + goto jit_failure; + } + } else if (!zend_jit_jmpznz(&dasm_state, opline, b, op_array, ssa)) { + goto jit_failure; + } + goto done; + case ZEND_FETCH_DIM_R: + case ZEND_FETCH_DIM_IS: + if (!zend_jit_fetch_dim_read(&dasm_state, opline, op_array, ssa)) { + goto jit_failure; + } + goto done; + case ZEND_ISSET_ISEMPTY_DIM_OBJ: + if (!zend_jit_isset_isempty_dim(&dasm_state, opline, b, &i, op_array, ssa)) { + goto jit_failure; + } + goto done; + /* + case ZEND_FETCH_OBJ_R: + if (!zend_jit_fetch_obj_r(&dasm_state, opline, op_array, ssa)) { + goto jit_failure; + } + goto done; + */ + case ZEND_BIND_GLOBAL: + if (!zend_jit_bind_global(&dasm_state, opline, op_array, ssa)) { + goto jit_failure; + } + goto done; + case ZEND_RECV_INIT: + if (!zend_jit_recv_init(&dasm_state, opline, op_array, (opline + 1)->opcode != ZEND_RECV_INIT, ssa)) { + goto jit_failure; + } + goto done; + default: + break; + } + } + + switch (opline->opcode) { case ZEND_RECV_INIT: if (ssa->cfg.split_at_recv) { if (!zend_jit_handler(&dasm_state, opline, zend_may_throw(opline, op_array, ssa))) { @@ -1146,14 +1154,15 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) } /* break missing intentionally */ case ZEND_BIND_GLOBAL: - if (opline->opcode != op_array->opcodes[i+1].opcode) { + if (opline == op_array->opcodes || + opline->opcode != op_array->opcodes[i-1].opcode) { /* repeatable opcodes */ if (!zend_jit_handler(&dasm_state, opline, zend_may_throw(opline, op_array, ssa))) { goto jit_failure; } } + zend_jit_set_opline(&dasm_state, opline+1); break; -#endif case ZEND_NOP: case ZEND_OP_DATA: break; @@ -1168,9 +1177,7 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) case ZEND_GENERATOR_CREATE: case ZEND_GENERATOR_RETURN: case ZEND_RETURN_BY_REF: -#if ZEND_JIT_LEVEL < ZEND_JIT_LEVEL_OPT_FUNC case ZEND_RETURN: -#endif case ZEND_EXIT: /* switch through trampoline */ case ZEND_YIELD: @@ -1182,15 +1189,12 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) /* stackless execution */ case ZEND_INCLUDE_OR_EVAL: case ZEND_DO_FCALL: -#if ZEND_JIT_LEVEL < ZEND_JIT_LEVEL_OPT_FUNC case ZEND_DO_UCALL: -#endif case ZEND_DO_FCALL_BY_NAME: if (!zend_jit_call(&dasm_state, opline)) { goto jit_failure; } break; -#if ZEND_JIT_LEVEL < ZEND_JIT_LEVEL_OPT_FUNC case ZEND_JMPZNZ: if (!zend_jit_handler(&dasm_state, opline, zend_may_throw(opline, op_array, ssa)) || !zend_jit_cond_jmp(&dasm_state, OP_JMP_ADDR(opline, opline->op2), ssa->cfg.blocks[b].successors[1]) || @@ -1241,7 +1245,6 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) } } /* break missing intentionally */ -#endif case ZEND_JMPZ_EX: case ZEND_JMPNZ_EX: case ZEND_JMP_SET: @@ -1273,6 +1276,7 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) goto jit_failure; } } +done: switch (opline->opcode) { case ZEND_DO_FCALL: case ZEND_DO_ICALL: @@ -1525,12 +1529,14 @@ static int zend_jit_make_stubs(void) return 1; } -ZEND_API int zend_jit_startup(size_t size) +ZEND_API int zend_jit_startup(zend_long jit_level, size_t size) { size_t page_size = jit_page_size(); int shared = 1; int ret; + zend_jit_level = jit_level; + #ifdef HAVE_GDB zend_jit_gdb_init(); #endif @@ -1631,7 +1637,7 @@ ZEND_API void zend_jit_protect(void) { } -ZEND_API int zend_jit_startup(size_t size) +ZEND_API int zend_jit_startup(zend_long jit_level, size_t size) { return FAILURE; } diff --git a/ext/opcache/jit/zend_jit.h b/ext/opcache/jit/zend_jit.h index 188736e093558..5900491e22934 100644 --- a/ext/opcache/jit/zend_jit.h +++ b/ext/opcache/jit/zend_jit.h @@ -25,9 +25,9 @@ #define ZEND_JIT_LEVEL_MINIMAL 1 /* minimal JIT (subroutine threading) */ #define ZEND_JIT_LEVEL_INLINE 2 /* selective inline threading */ #define ZEND_JIT_LEVEL_OPT_FUNC 3 /* optimized JIT based on Type-Inference */ -#define ZEND_JIT_LEVEL_OPT_SCRIPT 4 /* optimized JIT based on Type-Inference and inner-procedute analises */ +#define ZEND_JIT_LEVEL_OPT_SCRIPT 4 /* optimized JIT based on Type-Inference and inner-procedure analises */ -#define ZEND_JIT_LEVEL ZEND_JIT_LEVEL_OPT_SCRIPT +#define ZEND_JIT_LEVEL_DEFAULT "4" #define ZEND_JIT_DEBUG_ASM (1<<0) #define ZEND_JIT_DEBUG_SSA (1<<1) @@ -41,7 +41,7 @@ ZEND_API int zend_jit_op_array(zend_op_array *op_array, zend_script *script); ZEND_API int zend_jit_script(zend_script *script); ZEND_API void zend_jit_unprotect(void); ZEND_API void zend_jit_protect(void); -ZEND_API int zend_jit_startup(size_t size); +ZEND_API int zend_jit_startup(zend_long jit_level, size_t size); ZEND_API void zend_jit_shutdown(void); ZEND_API void zend_jit_status(zval *ret); diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 75bb22e555535..29171c8d0cb3a 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -149,8 +149,6 @@ static void* dasm_labels[zend_lb_MAX]; |.endif |.endmacro -#if ZEND_JIT_LEVEL >= ZEND_JIT_LEVEL_OT_FUNC - |.macro LOAD_ZVAL_ADDR, reg, op_type, op ||if (op_type == IS_CONST) { | .if X64 @@ -1148,7 +1146,6 @@ static void* dasm_labels[zend_lb_MAX]; static const zend_op *last_valid_opline; static int jit_return_label; -#endif /* bit helpers */ @@ -1476,22 +1473,24 @@ static int zend_jit_new(dasm_State **Dst, const zend_op *opline, int *opnum, zen } if (opline->extended_value == 0 && (opline+1)->opcode == ZEND_DO_FCALL) { zend_class_entry *ce = NULL; -#if ZEND_JIT_LEVEL >= ZEND_JIT_LEVEL_OPT_FUNC - if (ssa->ops && ssa->var_info) { - zend_ssa_var_info *res_ssa = &ssa->var_info[ssa->ops[opline - op_array->opcodes].result_def]; - if (res_ssa->ce && !res_ssa->is_instanceof) { - ce = res_ssa->ce; + + if (zend_jit_level >= ZEND_JIT_LEVEL_OPT_FUNC) { + if (ssa->ops && ssa->var_info) { + zend_ssa_var_info *res_ssa = &ssa->var_info[ssa->ops[opline - op_array->opcodes].result_def]; + if (res_ssa->ce && !res_ssa->is_instanceof) { + ce = res_ssa->ce; + } } - } -#else - if (opline->op1_type == IS_CONST) { - zval *zv = RT_CONSTANT(op_array, opline->op1); - if (Z_TYPE_P(zv) == IS_STRING) { - zval *lc = zv + 1; - ce = (zend_class_entry*)zend_hash_find_ptr(EG(class_table), Z_STR_P(lc)); + } else { + if (opline->op1_type == IS_CONST) { + zval *zv = RT_CONSTANT(op_array, opline->op1); + if (Z_TYPE_P(zv) == IS_STRING) { + zval *lc = zv + 1; + ce = (zend_class_entry*)zend_hash_find_ptr(EG(class_table), Z_STR_P(lc)); + } } } -#endif + (*opnum)++; if (!ce || ce->constructor) { const zend_op *next_opline = opline + 1; @@ -1505,8 +1504,6 @@ static int zend_jit_new(dasm_State **Dst, const zend_op *opline, int *opnum, zen return 1; } -#if ZEND_JIT_LEVEL >= ZEND_JIT_LEVEL_OPT_FUNC - static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) { uint32_t op1_info, op1_def_info; @@ -5669,8 +5666,6 @@ fallback: return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); } -#endif - /* * Local variables: * tab-width: 4 diff --git a/ext/opcache/zend_accelerator_module.c b/ext/opcache/zend_accelerator_module.c index 4ba0fee87b347..d7ad54cad268d 100644 --- a/ext/opcache/zend_accelerator_module.c +++ b/ext/opcache/zend_accelerator_module.c @@ -322,6 +322,7 @@ ZEND_INI_BEGIN() STD_PHP_INI_BOOLEAN("opcache.huge_code_pages" , "0" , PHP_INI_SYSTEM, OnUpdateBool, accel_directives.huge_code_pages, zend_accel_globals, accel_globals) #endif #ifdef HAVE_JIT + STD_PHP_INI_ENTRY("opcache.jit" , ZEND_JIT_LEVEL_DEFAULT, PHP_INI_SYSTEM, OnUpdateLong, accel_directives.jit_level, zend_accel_globals, accel_globals) STD_PHP_INI_ENTRY("opcache.jit_buffer_size" , "0" , PHP_INI_SYSTEM, OnUpdateLong, accel_directives.jit_buffer_size, zend_accel_globals, accel_globals) STD_PHP_INI_ENTRY("opcache.jit_debug" , "0" , PHP_INI_SYSTEM, OnUpdateLong, accel_directives.jit_debug, zend_accel_globals, accel_globals) #endif diff --git a/ext/opcache/zend_persist.c b/ext/opcache/zend_persist.c index 20a81755431d9..e07cf86069381 100644 --- a/ext/opcache/zend_persist.c +++ b/ext/opcache/zend_persist.c @@ -570,8 +570,8 @@ static void zend_persist_op_array_ex(zend_op_array *op_array, zend_persistent_sc ZCG(mem) = (void*)((char*)ZCG(mem) + ZEND_ALIGNED_SIZE(zend_extensions_op_array_persist(op_array, ZCG(mem)))); #ifdef HAVE_JIT - if (ZEND_JIT_LEVEL <= ZEND_JIT_LEVEL_OPT_FUNC && - do_jit && ZCG(accel_directives).jit_buffer_size) { + if (ZCG(accel_directives).jit_level && + ZCG(accel_directives).jit_level <= ZEND_JIT_LEVEL_OPT_FUNC) { zend_jit_op_array(op_array, ZCG(current_persistent_script) ? &ZCG(current_persistent_script)->script : NULL); } #endif @@ -858,7 +858,9 @@ zend_persistent_script *zend_accel_script_persist(zend_persistent_script *script ZCG(mem) = (void*)((char*)ZCG(mem) + script->arena_size); #ifdef HAVE_JIT - zend_jit_unprotect(); + if (ZCG(accel_directives).jit_level) { + zend_jit_unprotect(); + } #endif ZCG(current_persistent_script) = script; @@ -868,11 +870,12 @@ zend_persistent_script *zend_accel_script_persist(zend_persistent_script *script ZCG(current_persistent_script) = NULL; #ifdef HAVE_JIT - if (ZEND_JIT_LEVEL >= ZEND_JIT_LEVEL_OPT_SCRIPT && - ZCG(accel_directives).jit_buffer_size) { - zend_jit_script(&script->script); + if (ZCG(accel_directives).jit_level) { + if (ZCG(accel_directives).jit_level >= ZEND_JIT_LEVEL_OPT_SCRIPT) { + zend_jit_script(&script->script); + } + zend_jit_protect(); } - zend_jit_protect(); #endif return script; From a5e16206c227a353f5d1897bfeb285fab48a2747 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 7 Oct 2016 01:13:37 +0300 Subject: [PATCH 274/569] Move exceptional code into stub --- ext/opcache/jit/zend_jit_x86.dasc | 55 ++++++++++++++++++++----------- 1 file changed, 35 insertions(+), 20 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 29171c8d0cb3a..8e562d8608d89 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -1288,11 +1288,44 @@ static int zend_jit_leave_throw_stub(dasm_State **Dst) return 1; } +static int zend_jit_icall_throw_stub(dasm_State **Dst) +{ + |->icall_throw_handler: + | mov IP, EX->opline + | // zend_throw_exception_internal(NULL); + |.if X64 + | xor CARG1, CARG1 + | EXT_CALL zend_throw_exception_internal, r0 + |.else + | sub r4, 12 + | push 0 + | EXT_CALL zend_throw_exception_internal, r0 + | add r4, 16 + |.endif + | // if (opline->result_type != IS_UNUSED) { + | cmp byte OP:IP->result_type, IS_UNUSED + | je >5 + | // zval_ptr_dtor(EX_VAR(opline->result.var)); + |.if X64 + | movsxd r0, dword OP:IP->result.var + |.else + | mov r0, OP:IP->result.var + |.endif + | add r0, FP + | ZVAL_PTR_DTOR r0, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 1, 0, 0, NULL, 0 + |5: + | // HANDLE_EXCEPTION() + | jmp ->exception_handler + + return 1; +} + static const zend_jit_stub zend_jit_stubs[] = { JIT_STUB(interrupt_handler), JIT_STUB(exception_handler), JIT_STUB(leave_function), JIT_STUB(leave_throw), + JIT_STUB(icall_throw), }; static int zend_jit_align_func(dasm_State **Dst) @@ -4504,26 +4537,8 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar | // if (UNEXPECTED(EG(exception) != NULL)) { | cmp aword [&EG(exception)], 0 - | jne >1 - |.cold_code - |1: - | // zend_throw_exception_internal(NULL); - |.if X64 - | xor CARG1, CARG1 - | EXT_CALL zend_throw_exception_internal, r0 - |.else - | sub r4, 12 - | push 0 - | EXT_CALL zend_throw_exception_internal, r0 - | add r4, 16 - |.endif - if (opline->result_type != IS_UNUSED) { - | // zval_ptr_dtor(EX_VAR(opline->result.var)); - | ZVAL_PTR_DTOR FP + opline->result.var, RES_INFO(), 1, 0, 0, op_array->filename, opline->lineno - } - | // HANDLE_EXCEPTION(); - | jmp ->exception_handler - |.code + | jne ->icall_throw_handler + // TODO: Can we avoid checking for interrupts after each call ??? return zend_jit_check_timeout(Dst, opline + 1); } From 647dd59293a9cbdb1cd9a95778b3554a9f2ce281 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 7 Oct 2016 01:43:59 +0300 Subject: [PATCH 275/569] Removed dead code --- ext/opcache/jit/zend_jit_x86.dasc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 8e562d8608d89..21915b838774a 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -928,7 +928,7 @@ static void* dasm_labels[zend_lb_MAX]; || } | // zval_dtor_func(r); | ZVAL_DTOR_FUNC filename, lineno -|| if (gc && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT))) { +|| if (gc && RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT))) { | jmp >2 || } |1: @@ -956,7 +956,7 @@ static void* dasm_labels[zend_lb_MAX]; | jmp >2 |.code || } -|| } else if (cold && (!gc || ((op_info) & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))))) { +|| } else if (cold && ((op_info) & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE)))) { | jmp >2 |.code || } From 471a385f58c46dcae99d99374c3eb5b7cefc2cf9 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 7 Oct 2016 10:39:26 +0300 Subject: [PATCH 276/569] Use unsigned comparison --- ext/opcache/jit/zend_jit_x86.dasc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 21915b838774a..501da2bc85362 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -2258,9 +2258,9 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o | cmp dword [FCARG1a + offsetof(zend_array, nNumUsed)], val |.endif if (type == BP_JIT_IS) { - | jle >9 // NOT_FOUND + | jbe >9 // NOT_FOUND } else { - | jle >2 // NOT_FOUND + | jbe >2 // NOT_FOUND } | // _ret = &_ht->arData[_h].val; | mov r0, aword [FCARG1a + offsetof(zend_array, arData)] @@ -2288,9 +2288,9 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o | cmp dword [FCARG1a + offsetof(zend_array, nNumUsed)], FCARG2a |.endif if (type == BP_JIT_IS) { - | jle >9 // NOT_FOUND + | jbe >9 // NOT_FOUND } else { - | jle >2 // NOT_FOUND + | jbe >2 // NOT_FOUND } | // _ret = &_ht->arData[_h].val; | mov r0, FCARG2a From 603ca4d808211e00b6f8e7f8a4fab79373c8871e Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 7 Oct 2016 11:13:53 +0300 Subject: [PATCH 277/569] Combine mov and imul into one instruction --- ext/opcache/jit/zend_jit_x86.dasc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 501da2bc85362..3c5c90ff8d2af 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -2293,11 +2293,11 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o | jbe >2 // NOT_FOUND } | // _ret = &_ht->arData[_h].val; - | mov r0, FCARG2a |.if X64 + | mov r0, FCARG2a | shl r0, 5 |.else - | imul r0, sizeof(Bucket) + | imul r0, FCARG2a, sizeof(Bucket) |.endif | add r0, aword [FCARG1a + offsetof(zend_array, arData)] if (type == BP_JIT_IS) { From 686d756df805a47433aa33df5e0041688436f1b6 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 7 Oct 2016 11:36:37 +0300 Subject: [PATCH 278/569] don't save EX(opline) before helpers that can't fail --- ext/opcache/jit/zend_jit_x86.dasc | 2 -- 1 file changed, 2 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 3c5c90ff8d2af..85a31b27b81e8 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -2420,7 +2420,6 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o if (op1_info & MAY_BE_ARRAY_KEY_LONG) { | jmp >8 |1: - | SAVE_VALID_OPLINE opline | EXT_CALL zend_jit_hash_index_lookup_w, r0 } break; @@ -2522,7 +2521,6 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o } break; case BP_VAR_W: - | SAVE_VALID_OPLINE opline if (opline->op2_type != IS_CONST) { | EXT_CALL zend_jit_symtable_lookup_w, r0 } else { From e6034d45a5034ee223ab93eb1c2a3443dbb83b54 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 7 Oct 2016 12:45:00 +0300 Subject: [PATCH 279/569] Reuse register reserved for VM IP as a general purpose register, saved across C calls (to avoid multiple spills) --- ext/opcache/jit/zend_jit_x86.dasc | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 85a31b27b81e8..ca290516fbea6 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -4487,8 +4487,12 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar | // EG(current_execute_data) = execute_data; | mov aword [&EG(current_execute_data)], FCARG1a + // Reuse register reserved for VM IP as a general purpose register, + // saved across C calls (to avoid multiple spills) + | mov IP, FCARG1a + zend_jit_reset_opline(Dst, NULL); + | // fbc->internal_function.handler(call, ret); - | mov aword [r4], FCARG1a // save |.if X64 | EXT_CALL func->internal_function.handler, r0 |.else @@ -4505,20 +4509,19 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar | // zend_vm_stack_free_args(call); for (i = 0; i < call_info->num_args; i++ ) { uint32_t offset = (uint32_t)(uintptr_t)ZEND_CALL_VAR_NUM(NULL, i); - | mov r0, aword [r4] // restore - | ZVAL_PTR_DTOR r0 + offset, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN, 0, 1, 1, op_array->filename, opline->lineno + | ZVAL_PTR_DTOR IP + offset, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN, 0, 1, 1, op_array->filename, opline->lineno } | // zend_vm_stack_free_call_frame(call); - | mov FCARG1a, aword [r4] // restore - | test byte [FCARG1a + offsetof(zend_execute_data, This.u1.type_info) + 2], ZEND_CALL_ALLOCATED + | test byte [IP + offsetof(zend_execute_data, This.u1.type_info) + 2], ZEND_CALL_ALLOCATED | jnz >1 |.cold_code |1: + | mov FCARG1a, IP | EXT_CALL zend_jit_free_call_frame, r0 | jmp >1 |.code - | mov aword [&EG(vm_stack_top)], FCARG1a + | mov aword [&EG(vm_stack_top)], IP |1: if (opline->result_type == IS_UNUSED) { From 75f86fe1c9a402765f77afe60a7886262c6c7fef Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 7 Oct 2016 14:19:10 +0300 Subject: [PATCH 280/569] Generalise VM IP register reuse to cache EX(call) (and avoid reloading in each SEND_* instruction) --- ext/opcache/jit/zend_jit_x86.dasc | 117 +++++++++++++++++++----------- 1 file changed, 75 insertions(+), 42 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index ca290516fbea6..f19ed19060a98 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -27,6 +27,8 @@ |.define FP, r14 |.define IP, r15 |.define IPl, r15d + |.define RX, IP // the same as VM IP reused as a general purpos reg + |.define RXl, IPl |.define CARG1, rdi // x64/POSIX C call arguments. |.define CARG2, rsi |.define CARG3, rdx @@ -49,6 +51,8 @@ |.define FP, esi |.define IP, edi |.define IPl, edi + |.define RX, IP // the same as VM IP reused as a general purpos reg + |.define RXl, IPl |.define FCARG1a, ecx // x86 fastcall arguments. |.define FCARG2a, edx |.define FCARG1d, ecx @@ -1144,6 +1148,7 @@ static void* dasm_labels[zend_lb_MAX]; || } |.endmacro +static zend_bool reuse_ip; static const zend_op *last_valid_opline; static int jit_return_label; @@ -1330,6 +1335,7 @@ static const zend_jit_stub zend_jit_stubs[] = { static int zend_jit_align_func(dasm_State **Dst) { + reuse_ip = 0; last_valid_opline = NULL; jit_return_label = -1; |.align 16 @@ -1356,6 +1362,7 @@ static int zend_jit_set_valid_ip(dasm_State **Dst, const zend_op *opline) | add IP, (opline - last_valid_opline) * sizeof(zend_op); } last_valid_opline = opline; + reuse_ip = 0; return 1; } @@ -1431,7 +1438,9 @@ static int zend_jit_tail_handler(dasm_State **Dst, const zend_op *opline) static int zend_jit_set_opline(dasm_State **Dst, const zend_op *target_opline) { - last_valid_opline = target_opline; + if (!reuse_ip) { + last_valid_opline = target_opline; + } return 1; } @@ -1441,6 +1450,15 @@ static int zend_jit_reset_opline(dasm_State **Dst, const zend_op *target_opline) return 1; } +static int zend_jit_start_reuse_ip(void) { + last_valid_opline = NULL; + reuse_ip = 1; +} + +static int zend_jit_stop_reuse_ip(void) { + reuse_ip = 0; +} + static int zend_jit_jmp(dasm_State **Dst, unsigned int target_label) { | jmp =>target_label @@ -4252,30 +4270,32 @@ static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, zen uint32_t used_stack = zend_vm_calc_used_stack(opline->extended_value, func); + zend_jit_start_reuse_ip(); + | // if (UNEXPECTED(used_stack > (size_t)(((char*)EG(vm_stack_end)) - (char*)call))) { - | mov FCARG1a, aword [&EG(vm_stack_top)] + | mov RX, aword [&EG(vm_stack_top)] | mov r2, aword [&EG(vm_stack_end)] - | sub r2, FCARG1a + | sub r2, RX | cmp r2, used_stack | jb >2 | // EG(vm_stack_top) = (zval*)((char*)call + used_stack); | add aword [&EG(vm_stack_top)], used_stack | // zend_vm_init_call_frame(call, call_info, func, num_args, called_scope, object); | // call->func = func; - | mov aword EX:FCARG1a->func, r0 + | mov aword EX:RX->func, r0 | // ZEND_SET_CALL_INFO(call, 0, call_info); - | mov dword EX:FCARG1a->This.u1.type_info, (IS_UNDEF | (ZEND_CALL_NESTED_FUNCTION << ZEND_CALL_INFO_SHIFT)) + | mov dword EX:RX->This.u1.type_info, (IS_UNDEF | (ZEND_CALL_NESTED_FUNCTION << ZEND_CALL_INFO_SHIFT)) |1: | // Z_CE(call->This) = called_scope; - | mov aword EX:FCARG1a->This.value.ptr, 0 + | mov aword EX:RX->This.value.ptr, 0 | // ZEND_CALL_NUM_ARGS(call) = num_args; - | mov dword EX:FCARG1a->This.u2.num_args, opline->extended_value + | mov dword EX:RX->This.u2.num_args, opline->extended_value |.cold_code |2: | mov FCARG1d, used_stack | mov FCARG2a, r0 | EXT_CALL zend_jit_extend_stack_helper, r0 - | mov FCARG1a, r0 + | mov RX, r0 | jmp <1 |.code @@ -4337,13 +4357,13 @@ static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ } | // call->prev_execute_data = EX(call); if (call_level == 1) { - | mov aword EX:FCARG1a->prev_execute_data, 0 + | mov aword EX:RX->prev_execute_data, 0 } else { | mov r0, EX->call - | mov EX:FCARG1a->prev_execute_data, r0 + | mov EX:RX->prev_execute_data, r0 } | // EX(call) = call; - | mov EX->call, FCARG1a + | mov EX->call, RX return 1; @@ -4390,26 +4410,31 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar ZEND_ASSERT(0); } - | // call = EX(call); - | mov FCARG1a, EX->call + if (!reuse_ip) { + zend_jit_start_reuse_ip(); + | // call = EX(call); + | mov RX, EX->call + } + zend_jit_stop_reuse_ip(); + | // fbc = call->func; - | // mov r2, EX:FCARG1a->func ??? + | // mov r2, EX:RX->func ??? | // SAVE_OPLINE(); | SAVE_VALID_OPLINE opline if (call_level == 1) { | mov aword EX->call, 0 } else { | //EX(call) = call->prev_execute_data; - | mov r0, EX:FCARG1a->prev_execute_data + | mov r0, EX:RX->prev_execute_data | mov EX->call, r0 } | //call->prev_execute_data = execute_data; - | mov EX:FCARG1a->prev_execute_data, EX + | mov EX:RX->prev_execute_data, EX | if (func && func->type == ZEND_USER_FUNCTION) { | // EX(call) = NULL; - | mov aword EX:FCARG1a->call, 0 + | mov aword EX:RX->call, 0 } if (func && func->type == ZEND_USER_FUNCTION) { @@ -4419,15 +4444,15 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar | SET_Z_TYPE_INFO FP + opline->result.var, IS_NULL | // EX(return_value) = EX_VAR(opline->result.var); | lea r0, aword [FP + opline->result.var] - | mov aword EX:FCARG1a->return_value, r0 + | mov aword EX:RX->return_value, r0 } else { | // EX(return_value) = 0; - | mov aword EX:FCARG1a->return_value, 0 + | mov aword EX:RX->return_value, 0 } for (i = call_info->num_args; i < func->op_array.last_var; i++) { uint32_t n = (uint32_t)(uintptr_t)ZEND_CALL_VAR_NUM(NULL, i); - | SET_Z_TYPE_INFO FCARG1a + n, IS_UNDEF + | SET_Z_TYPE_INFO RX + n, IS_UNDEF } //EX_LOAD_RUN_TIME_CACHE(op_array); @@ -4436,24 +4461,24 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar /* recursive call */ if (func->op_array.cache_size > sizeof(void*)) { | mov r0, EX->run_time_cache - | mov EX:FCARG1a->run_time_cache, r0 + | mov EX:RX->run_time_cache, r0 } } else { - | mov r2, EX:FCARG1a->func + | mov r2, EX:RX->func | mov r0, aword [r2 + offsetof(zend_op_array, run_time_cache)] - | mov EX:FCARG1a->run_time_cache, r0 + | mov EX:RX->run_time_cache, r0 } } | //EX_LOAD_LITERALS(op_array); |.if X64 | LOAD_ADDR r0, func->op_array.literals - | mov EX:FCARG1a->literals, r0 + | mov EX:RX->literals, r0 |.endif | // EG(current_execute_data) = execute_data; - | mov aword [&EG(current_execute_data)], FCARG1a - | mov FP, FCARG1a + | mov aword [&EG(current_execute_data)], RX + | mov FP, RX | // opline = op_array->opcodes; | LOAD_ADDR IP, (func->op_array.opcodes + call_info->num_args) @@ -4485,20 +4510,18 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar } | // EG(current_execute_data) = execute_data; - | mov aword [&EG(current_execute_data)], FCARG1a + | mov aword [&EG(current_execute_data)], RX - // Reuse register reserved for VM IP as a general purpose register, - // saved across C calls (to avoid multiple spills) - | mov IP, FCARG1a zend_jit_reset_opline(Dst, NULL); | // fbc->internal_function.handler(call, ret); |.if X64 + | mov FCARG1a, RX | EXT_CALL func->internal_function.handler, r0 |.else | sub r4, 8 | push FCARG2a - | push FCARG1a + | push RX | EXT_CALL func->internal_function.handler, r0 | add r4, 16 |.endif @@ -4509,19 +4532,19 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar | // zend_vm_stack_free_args(call); for (i = 0; i < call_info->num_args; i++ ) { uint32_t offset = (uint32_t)(uintptr_t)ZEND_CALL_VAR_NUM(NULL, i); - | ZVAL_PTR_DTOR IP + offset, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN, 0, 1, 1, op_array->filename, opline->lineno + | ZVAL_PTR_DTOR RX + offset, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN, 0, 1, 1, op_array->filename, opline->lineno } | // zend_vm_stack_free_call_frame(call); - | test byte [IP + offsetof(zend_execute_data, This.u1.type_info) + 2], ZEND_CALL_ALLOCATED + | test byte [RX + offsetof(zend_execute_data, This.u1.type_info) + 2], ZEND_CALL_ALLOCATED | jnz >1 |.cold_code |1: - | mov FCARG1a, IP + | mov FCARG1a, RX | EXT_CALL zend_jit_free_call_frame, r0 | jmp >1 |.code - | mov aword [&EG(vm_stack_top)], IP + | mov aword [&EG(vm_stack_top)], RX |1: if (opline->result_type == IS_UNUSED) { @@ -4568,12 +4591,16 @@ static int zend_jit_send_val(dasm_State **Dst, const zend_op *opline, zend_op_ar op1_info = OP1_INFO(); - | mov FCARG1a, EX->call + if (!reuse_ip) { + zend_jit_start_reuse_ip(); + | // call = EX(call); + | mov RX, EX->call + } if (opline->opcode == ZEND_SEND_VAL_EX) { uint32_t mask = ZEND_SEND_BY_REF << ((arg_num + 3) * 2); - | mov r0, EX:FCARG1a->func + | mov r0, EX:RX->func if (arg_num <= MAX_ARG_FLAG_NUM) { | mov eax, dword [r0 + offsetof(zend_function, quick_arg_flags)] | test eax, mask @@ -4583,7 +4610,7 @@ static int zend_jit_send_val(dasm_State **Dst, const zend_op *opline, zend_op_ar } |.cold_code |1: - | SET_Z_TYPE_INFO FCARG1a + opline->result.var, IS_UNDEF + | SET_Z_TYPE_INFO RX + opline->result.var, IS_UNDEF | SAVE_VALID_OPLINE opline |.if X64 | mov CARG1, 0 @@ -4606,12 +4633,12 @@ static int zend_jit_send_val(dasm_State **Dst, const zend_op *opline, zend_op_ar if (opline->op1_type == IS_CONST) { zval *zv = RT_CONSTANT(op_array, opline->op1); - | ZVAL_COPY_CONST FCARG1a + opline->result.var, -1, zv, r0 + | ZVAL_COPY_CONST RX + opline->result.var, -1, zv, r0 || if (Z_REFCOUNTED_P(zv)) { | ADDREF_CONST zv, r0 || } } else { - | ZVAL_COPY_VALUE FCARG1a + opline->result.var, FP + opline->op1.var, op1_info, r0, eax, r2 + | ZVAL_COPY_VALUE RX + opline->result.var, FP + opline->op1.var, op1_info, r0, eax, r2 } return 1; @@ -4635,8 +4662,14 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, zend_op_ar } op1_info = OP1_INFO(); - | mov FCARG1a, EX->call - | ZVAL_COPY_VALUE FCARG1a + opline->result.var, FP + opline->op1.var, op1_info, r0, eax, r2 + + if (!reuse_ip) { + zend_jit_start_reuse_ip(); + | // call = EX(call); + | mov RX, EX->call + } + + | ZVAL_COPY_VALUE RX + opline->result.var, FP + opline->op1.var, op1_info, r0, eax, r2 || if (opline->op1_type == IS_CV) { | TRY_ADDREF op1_info, ah, r2 || } From dc91ed5cfb973ace1307ffc793bcd0f6fd4848a1 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 10 Oct 2016 10:04:19 +0300 Subject: [PATCH 281/569] Fixed compilation warnings --- ext/opcache/jit/zend_jit_x86.dasc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index f19ed19060a98..13206270cd3ef 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -1450,12 +1450,12 @@ static int zend_jit_reset_opline(dasm_State **Dst, const zend_op *target_opline) return 1; } -static int zend_jit_start_reuse_ip(void) { +static void zend_jit_start_reuse_ip(void) { last_valid_opline = NULL; reuse_ip = 1; } -static int zend_jit_stop_reuse_ip(void) { +static void zend_jit_stop_reuse_ip(void) { reuse_ip = 0; } From 89c838a558134db7d67e0451c240f5be7ad50e5e Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 10 Oct 2016 12:31:42 +0300 Subject: [PATCH 282/569] Eliminatd dead conditional jumps --- ext/opcache/jit/zend_jit_x86.dasc | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 13206270cd3ef..f8845609ca835 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -4102,10 +4102,14 @@ static int zend_jit_jmpznz(dasm_State **Dst, const zend_op *opline, int b, zend_ | ZVAL_DEREF FCARG1a, op1_info if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE)) { - if (true_label != (uint32_t)-1) { - | IF_Z_TYPE FCARG1a, IS_TRUE, =>true_label + if (op1_info & MAY_BE_TRUE) { + if (true_label != (uint32_t)-1) { + | IF_Z_TYPE FCARG1a, IS_TRUE, =>true_label + } else { + | IF_Z_TYPE FCARG1a, IS_TRUE, >9 + } } else { - | IF_Z_TYPE FCARG1a, IS_TRUE, >9 + | cmp byte [FCARG1a + 8], IS_TRUE } if (op1_info & MAY_BE_UNDEF) { @@ -4134,7 +4138,7 @@ static int zend_jit_jmpznz(dasm_State **Dst, const zend_op *opline, int b, zend_ | jmp >9 } |.code - } else { + } else if (op1_info & (MAY_BE_NULL|MAY_BE_FALSE)) { if (false_label != (uint32_t)-1) { | jl =>false_label if (!(op1_info & MAY_BE_LONG) && From 83abaf03aeaca69f6051713389b1d180fa3cb1d5 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 10 Oct 2016 13:09:28 +0300 Subject: [PATCH 283/569] Save instruction on internal function call --- ext/opcache/jit/zend_jit_x86.dasc | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index f8845609ca835..6e867e09e88c8 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -4286,7 +4286,20 @@ static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, zen | add aword [&EG(vm_stack_top)], used_stack | // zend_vm_init_call_frame(call, call_info, func, num_args, called_scope, object); | // call->func = func; - | mov aword EX:RX->func, r0 + if (!func || func->type != ZEND_INTERNAL_FUNCTION) { + | mov aword EX:RX->func, r0 + } else { + |.if X64 + || if (!IS_32BIT(func)) { + | mov aword EX:RX->func, func + || } else { + | LOAD_ADDR r0, func + | mov aword EX:RX->func, r0 + || } + |.else + | mov aword EX:RX->func, func + |.endif + } | // ZEND_SET_CALL_INFO(call, 0, call_info); | mov dword EX:RX->This.u1.type_info, (IS_UNDEF | (ZEND_CALL_NESTED_FUNCTION << ZEND_CALL_INFO_SHIFT)) |1: @@ -4328,7 +4341,7 @@ static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ } if (func && func->type == ZEND_INTERNAL_FUNCTION) { - | LOAD_ADDR r0, func + /* load constant address later */ } else if (func && op_array == &func->op_array) { /* recursive call */ | mov r0, EX->func From 1d4575b6fd444ca883cbd000961f5f48507c4cb0 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 10 Oct 2016 18:09:49 +0300 Subject: [PATCH 284/569] Avoid unnecessary EX(call) chain construction --- ext/opcache/jit/zend_jit.c | 2 +- ext/opcache/jit/zend_jit_x86.dasc | 121 +++++++++++++++++++++++++----- 2 files changed, 104 insertions(+), 19 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index d71cf9297f12a..3220d2e86a359 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -1044,7 +1044,7 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) } goto done; case ZEND_INIT_FCALL: - if (!zend_jit_init_fcall(&dasm_state, opline, op_array, call_level)) { + if (!zend_jit_init_fcall(&dasm_state, opline, b, op_array, ssa, call_level)) { goto jit_failure; } goto done; diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 6e867e09e88c8..ed1e879692fff 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -79,9 +79,13 @@ # define r14 14 # define r15 15 # define FP r14 +# define IP r15 +# define RX IP # define FCARG1a r7 #else # define FP r6 +# define IP r7 +# define RX IP # define FCARG1a r1 #endif @@ -1149,6 +1153,8 @@ static void* dasm_labels[zend_lb_MAX]; |.endmacro static zend_bool reuse_ip; +static zend_bool delayed_call_chain; +static uint32_t delayed_call_level; static const zend_op *last_valid_opline; static int jit_return_label; @@ -1336,6 +1342,7 @@ static const zend_jit_stub zend_jit_stubs[] = { static int zend_jit_align_func(dasm_State **Dst) { reuse_ip = 0; + delayed_call_chain = 0; last_valid_opline = NULL; jit_return_label = -1; |.align 16 @@ -1354,8 +1361,30 @@ static int zend_jit_label(dasm_State **Dst, unsigned int label) return 1; } +static int zend_jit_save_call_chain(dasm_State **Dst, uint32_t call_level) +{ + | // call->prev_execute_data = EX(call); + if (call_level == 1) { + | mov aword EX:RX->prev_execute_data, 0 + } else { + | mov r0, EX->call + | mov EX:RX->prev_execute_data, r0 + } + | // EX(call) = call; + | mov EX->call, RX + + delayed_call_chain = 0; + + return 1; +} + static int zend_jit_set_valid_ip(dasm_State **Dst, const zend_op *opline) { + if (delayed_call_chain) { + if (!zend_jit_save_call_chain(Dst, delayed_call_level)) { + return 0; + } + } if (!last_valid_opline) { | LOAD_ADDR IP, opline } else if (last_valid_opline != opline) { @@ -2240,8 +2269,12 @@ static int zend_jit_math(dasm_State **Dst, const zend_op *opline, int *opnum, ze (opline+1)->op1.var == opline->result.var) { /* Eliminate the following SEND_VAL */ (*opnum)++; - | mov FCARG1a, EX->call - return zend_jit_math_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, FP, opline->op1.var, op1_info, opline->op2_type, opline->op2, FP, opline->op2.var, op2_info, FCARG1a, (opline+1)->result.var, RES_INFO()); + if (!reuse_ip) { + zend_jit_start_reuse_ip(); + | // call = EX(call); + | mov RX, EX->call + } + return zend_jit_math_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, FP, opline->op1.var, op1_info, opline->op2_type, opline->op2, FP, opline->op2.var, op2_info, RX, (opline+1)->result.var, RES_INFO()); } else { return zend_jit_math_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, FP, opline->op1.var, op1_info, opline->op2_type, opline->op2, FP, opline->op2.var, op2_info, FP, opline->result.var, RES_INFO()); } @@ -4319,14 +4352,61 @@ static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, zen return 1; } -static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, int call_level) +static int zend_jit_needs_call_chain(zend_call_info *call_info, uint32_t b, zend_op_array *op_array, zend_ssa *ssa) +{ + const zend_op *opline = call_info->caller_init_opline; + const zend_op *end = call_info->caller_call_opline; + int skip; + + if (end - op_array->opcodes >= ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len) { + /* INIT_FCALL and DO_FCALL in different BasicBlocks */ + return 1; + } + + opline++; + skip = 1; + while (opline != end) { + if (skip) { + switch (opline->opcode) { + case ZEND_SEND_VAL: + case ZEND_SEND_VAR: + case ZEND_SEND_VAL_EX: + case ZEND_SEND_VAR_EX: + case ZEND_SEND_REF: + case ZEND_SEND_VAR_NO_REF: + case ZEND_SEND_VAR_NO_REF_EX: + skip = 0; + break; + case ZEND_SEND_ARRAY: + case ZEND_SEND_USER: + case ZEND_SEND_UNPACK: + return 1; + } + } else { + if (zend_may_throw(opline, op_array, ssa)) { + return 1; + } + } + opline++; + } + + return 0; +} + +static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, uint32_t b, zend_op_array *op_array, zend_ssa *ssa, int call_level) { zend_func_info *info = ZEND_FUNC_INFO(op_array); + zend_call_info *call_info = NULL; zend_function *func = NULL; - if (info) { - zend_call_info *call_info = info->callee_info; + if (delayed_call_chain) { + if (!zend_jit_save_call_chain(Dst, delayed_call_level)) { + return 0; + } + } + if (info) { + call_info = info->callee_info; while (call_info && call_info->caller_init_opline != opline) { call_info = call_info->next_callee; } @@ -4372,15 +4452,15 @@ static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ if (!zend_jit_push_call_frame(Dst, opline, op_array, func)) { return 0; } - | // call->prev_execute_data = EX(call); - if (call_level == 1) { - | mov aword EX:RX->prev_execute_data, 0 + + if (!func || zend_jit_needs_call_chain(call_info, b, op_array, ssa)) { + if (!zend_jit_save_call_chain(Dst, call_level)) { + return 0; + } } else { - | mov r0, EX->call - | mov EX:RX->prev_execute_data, r0 + delayed_call_chain = 1; + delayed_call_level = call_level; } - | // EX(call) = call; - | mov EX->call, RX return 1; @@ -4438,13 +4518,18 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar | // mov r2, EX:RX->func ??? | // SAVE_OPLINE(); | SAVE_VALID_OPLINE opline - if (call_level == 1) { - | mov aword EX->call, 0 - } else { - | //EX(call) = call->prev_execute_data; - | mov r0, EX:RX->prev_execute_data - | mov EX->call, r0 + + if (!delayed_call_chain) { + if (call_level == 1) { + | mov aword EX->call, 0 + } else { + | //EX(call) = call->prev_execute_data; + | mov r0, EX:RX->prev_execute_data + | mov EX->call, r0 + } } + delayed_call_chain = 0; + | //call->prev_execute_data = execute_data; | mov EX:RX->prev_execute_data, EX | From 6fa74dd36c2566991679f372645274a33da6081d Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 11 Oct 2016 02:57:04 +0300 Subject: [PATCH 285/569] Implemented JIT for FAST_CONCAT, CONCAT and ASSIGN_CONCAT --- ext/opcache/jit/zend_jit.c | 9 +- ext/opcache/jit/zend_jit_disasm_x86.c | 3 + ext/opcache/jit/zend_jit_helpers.c | 102 +++++++++++ ext/opcache/jit/zend_jit_x86.dasc | 246 ++++++++++++++++++++++++-- 4 files changed, 341 insertions(+), 19 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 3220d2e86a359..b9d975d9d9286 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -1020,11 +1020,18 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) goto jit_failure; } goto done; + case ZEND_CONCAT: + case ZEND_FAST_CONCAT: + if (!zend_jit_concat(&dasm_state, opline, &i, op_array, ssa)) { + goto jit_failure; + } + goto done; case ZEND_ASSIGN_ADD: case ZEND_ASSIGN_SUB: case ZEND_ASSIGN_MUL: // case ZEND_ASSIGN_DIV: // TODO: check for division by zero ??? - if (!zend_jit_assign_math(&dasm_state, opline, op_array, ssa)) { + case ZEND_ASSIGN_CONCAT: + if (!zend_jit_assign_op(&dasm_state, opline, op_array, ssa)) { goto jit_failure; } goto done; diff --git a/ext/opcache/jit/zend_jit_disasm_x86.c b/ext/opcache/jit/zend_jit_disasm_x86.c index 2209cc3dc4195..b1df09b73e1d4 100644 --- a/ext/opcache/jit/zend_jit_disasm_x86.c +++ b/ext/opcache/jit/zend_jit_disasm_x86.c @@ -410,6 +410,9 @@ static int zend_jit_disasm_init(void) REGISTER_HELPER(zend_jit_assign_dim_sub_helper); REGISTER_HELPER(zend_jit_assign_dim_mul_helper); REGISTER_HELPER(zend_jit_assign_dim_div_helper); + REGISTER_HELPER(zend_jit_assign_dim_concat_helper); + REGISTER_HELPER(zend_jit_fast_assign_concat_helper); + REGISTER_HELPER(zend_jit_fast_concat_helper); REGISTER_HELPER(zend_jit_isset_dim_helper); REGISTER_HELPER(zend_jit_free_call_frame); REGISTER_HELPER(zend_jit_zval_copy_unref_helper); diff --git a/ext/opcache/jit/zend_jit_helpers.c b/ext/opcache/jit/zend_jit_helpers.c index 585f04e45a0ba..1d13c5e1895aa 100644 --- a/ext/opcache/jit/zend_jit_helpers.c +++ b/ext/opcache/jit/zend_jit_helpers.c @@ -1137,6 +1137,108 @@ static void ZEND_FASTCALL zend_jit_assign_dim_div_helper(zval *container, zval * } } +static void ZEND_FASTCALL zend_jit_assign_dim_concat_helper(zval *container, zval *dim, zval *value) +{ + if (EXPECTED(Z_TYPE_P(container) == IS_OBJECT)) { + zval *object = container; + zval *property = dim; + zval *z; + zval rv, res; + + if (Z_OBJ_HT_P(object)->read_dimension && + (z = Z_OBJ_HT_P(object)->read_dimension(object, property, BP_VAR_R, &rv)) != NULL) { + + if (Z_TYPE_P(z) == IS_OBJECT && Z_OBJ_HT_P(z)->get) { + zval rv2; + zval *value = Z_OBJ_HT_P(z)->get(z, &rv2); + + if (z == &rv) { + zval_ptr_dtor(&rv); + } + ZVAL_COPY_VALUE(z, value); + } + concat_function(&res, Z_ISREF_P(z) ? Z_REFVAL_P(z) : z, value); + Z_OBJ_HT_P(object)->write_dimension(object, property, &res); + if (z == &rv) { + zval_ptr_dtor(&rv); + } +//??? if (retval) { +//??? ZVAL_COPY(retval, &res); +//??? } + zval_ptr_dtor(&res); + } else { + zend_error(E_WARNING, "Attempt to assign property of non-object"); +//??? if (retval) { +//??? ZVAL_NULL(retval); +//??? } + } + } else { + if (UNEXPECTED(Z_TYPE_P(container) == IS_STRING)) { + if (!dim) { + zend_throw_error(NULL, "[] operator not supported for strings"); + } else { + zend_check_string_offset(dim, BP_VAR_RW); + zend_wrong_string_offset(); + } +//??? } else if (EXPECTED(Z_TYPE_P(container) <= IS_FALSE)) { +//??? ZEND_VM_C_GOTO(assign_dim_op_convert_to_array); + } else { +//??? if (UNEXPECTED(OP1_TYPE != IS_VAR || EXPECTED(!Z_ISERROR_P(container)))) { + zend_error(E_WARNING, "Cannot use a scalar value as an array"); +//??? } +//??? if (retval) { +//??? ZVAL_NULL(retval); +//??? } + } + } +} + +static void ZEND_FASTCALL zend_jit_fast_assign_concat_helper(zval *op1, zval *op2) +{ + size_t op1_len = Z_STRLEN_P(op1); + size_t op2_len = Z_STRLEN_P(op2); + size_t result_len = op1_len + op2_len; + zend_string *result_str; + + if (UNEXPECTED(op1_len > SIZE_MAX - op2_len)) { + zend_throw_error(NULL, "String size overflow"); + return; + } + + if (Z_REFCOUNTED_P(op1)) { + result_str = zend_string_extend(Z_STR_P(op1), result_len, 0); + } else { + result_str = zend_string_alloc(result_len, 0); + memcpy(ZSTR_VAL(result_str), Z_STRVAL_P(op1), op1_len); + } + + ZVAL_NEW_STR(op1, result_str); + + memcpy(ZSTR_VAL(result_str) + op1_len, Z_STRVAL_P(op2), op2_len); + ZSTR_VAL(result_str)[result_len] = '\0'; +} + +static void ZEND_FASTCALL zend_jit_fast_concat_helper(zval *result, zval *op1, zval *op2) +{ + size_t op1_len = Z_STRLEN_P(op1); + size_t op2_len = Z_STRLEN_P(op2); + size_t result_len = op1_len + op2_len; + zend_string *result_str; + + if (UNEXPECTED(op1_len > SIZE_MAX - op2_len)) { + zend_throw_error(NULL, "String size overflow"); + return; + } + + result_str = zend_string_alloc(result_len, 0); + memcpy(ZSTR_VAL(result_str), Z_STRVAL_P(op1), op1_len); + + ZVAL_NEW_STR(result, result_str); + + memcpy(ZSTR_VAL(result_str) + op1_len, Z_STRVAL_P(op2), op2_len); + ZSTR_VAL(result_str)[result_len] = '\0'; +} + static int ZEND_FASTCALL zend_jit_isset_dim_helper(zval *container, zval *offset) { if (UNEXPECTED(Z_TYPE_P(offset) == IS_UNDEF)) { diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index ed1e879692fff..8a0de96a3fd28 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -985,12 +985,12 @@ static void* dasm_labels[zend_lb_MAX]; |.cold_code |1: || } else { -| IF_Z_FLAGS zv, IS_TYPE_COPYABLE + IS_TYPE_IMMUTABLE, >2 +| IF_NOT_Z_FLAGS zv, IS_TYPE_COPYABLE + IS_TYPE_IMMUTABLE, >2 || } | GET_Z_PTR r0, zv || if (RC_MAY_BE_1(op_info)) { | cmp dword [r0], 1 // if (GC_REFCOUNTED() > 1) -| jbe >3 +| jbe >2 || } | IF_Z_FLAGS zv, IS_TYPE_IMMUTABLE, >1 | GC_DELREF r0 @@ -2053,7 +2053,8 @@ static int zend_jit_math_helper(dasm_State **Dst, uint32_t op2_info, uint32_t res_reg, uint32_t res_offset, - uint32_t res_info) + uint32_t res_info, + zend_bool separate_op1) /* Labels: 1,2,3,4,5 */ { zend_bool same_ops = (op1_type == op2_type) && (op1.var == op2.var); @@ -2198,6 +2199,13 @@ static int zend_jit_math_helper(dasm_State **Dst, |.cold_code |5: | SAVE_VALID_OPLINE opline + if (separate_op1) { + if (op1_reg != FCARG1a || op1_offset != 0) { + | SEPARATE_ZVAL_NOREF Ra(op1_reg)+op1_offset, op1_info, 0, op_array->filename, opline->lineno + } else { + | SEPARATE_ZVAL_NOREF_REG op1_info, 0, op_array->filename, opline->lineno + } + } if (res_reg != FCARG1a || res_offset != 0) { | lea FCARG1a, [Ra(res_reg)+res_offset] } @@ -2274,9 +2282,162 @@ static int zend_jit_math(dasm_State **Dst, const zend_op *opline, int *opnum, ze | // call = EX(call); | mov RX, EX->call } - return zend_jit_math_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, FP, opline->op1.var, op1_info, opline->op2_type, opline->op2, FP, opline->op2.var, op2_info, RX, (opline+1)->result.var, RES_INFO()); + return zend_jit_math_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, FP, opline->op1.var, op1_info, opline->op2_type, opline->op2, FP, opline->op2.var, op2_info, RX, (opline+1)->result.var, RES_INFO(), 0); } else { - return zend_jit_math_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, FP, opline->op1.var, op1_info, opline->op2_type, opline->op2, FP, opline->op2.var, op2_info, FP, opline->result.var, RES_INFO()); + return zend_jit_math_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, FP, opline->op1.var, op1_info, opline->op2_type, opline->op2, FP, opline->op2.var, op2_info, FP, opline->result.var, RES_INFO(), 0); + } + +fallback: + /* fallback to subroutine threading */ + return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); +} + +static int zend_jit_concat_helper(dasm_State **Dst, + const zend_op *opline, + zend_op_array *op_array, + zend_ssa *ssa, + zend_uchar op1_type, + znode_op op1, + uint32_t op1_reg, + uint32_t op1_offset, + uint32_t op1_info, + zend_uchar op2_type, + znode_op op2, + uint32_t op2_reg, + uint32_t op2_offset, + uint32_t op2_info, + uint32_t res_reg, + uint32_t res_offset, + uint32_t res_info) +{ +#if 1 + if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) { + if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) { + | IF_NOT_Z_TYPE Ra(op1_reg)+op1_offset, IS_STRING, >5 + } + if (op2_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) { + | IF_NOT_Z_TYPE Ra(op2_reg)+op2_offset, IS_STRING, >5 + } + if (op1_reg == res_reg && op1_offset == res_offset) { + if (res_reg != FCARG1a || res_offset != 0) { + | lea FCARG1a, [Ra(res_reg)+res_offset] + } + if (op2_type == IS_CONST) { + | LOAD_ZVAL_ADDR FCARG2a, op2_type, op2 + } else { + | lea FCARG2a, [Ra(op2_reg)+op2_offset] + } + | EXT_CALL zend_jit_fast_assign_concat_helper, r0 + } else { + if (res_reg != FCARG1a || res_offset != 0) { + | lea FCARG1a, [Ra(res_reg)+res_offset] + } + if (op1_type == IS_CONST) { + | LOAD_ZVAL_ADDR FCARG2a, op1_type, op1 + } else { + | lea FCARG2a, [Ra(op1_reg)+op1_offset] + } + if (op2_type == IS_CONST) { + |.if X64 + | LOAD_ZVAL_ADDR CARG3, op2_type, op2 + |.else + | PUSH_ZVAL_ADDR op2_type, op2, r0 + |.endif + } else { + |.if X64 + | lea CARG3, [Ra(op2_reg)+op2_offset] + |.else + | lea r0, [Ra(op2_reg)+op2_offset] + | push r0 + |.endif + } + | EXT_CALL zend_jit_fast_concat_helper, r0 + } + | FREE_OP op1_type, op1, op1_info, 0, op_array, opline->lineno + | FREE_OP op2_type, op2, op2_info, 0, op_array, opline->lineno + |4: + } + if ((op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) || + (op2_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING))) { + if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) { + |.cold_code + |5: + } +#endif + | SAVE_VALID_OPLINE opline + if (res_reg != FCARG1a || res_offset != 0) { + | lea FCARG1a, [Ra(res_reg)+res_offset] + } + if (op1_type == IS_CONST) { + | LOAD_ZVAL_ADDR FCARG2a, op1_type, op1 + } else { + | lea FCARG2a, [Ra(op1_reg)+op1_offset] + } + if (op2_type == IS_CONST) { + |.if X64 + | LOAD_ZVAL_ADDR CARG3, op2_type, op2 + |.else + | PUSH_ZVAL_ADDR op2_type, op2, r0 + |.endif + } else { + |.if X64 + | lea CARG3, [Ra(op2_reg)+op2_offset] + |.else + | lea r0, [Ra(op2_reg)+op2_offset] + | push r0 + |.endif + } + | EXT_CALL concat_function, r0 + | FREE_OP op1_type, op1, op1_info, 0, op_array, opline->lineno + | FREE_OP op2_type, op2, op2_info, 0, op_array, opline->lineno + || if (zend_may_throw(opline, op_array, ssa)) { + || zend_jit_check_exception(Dst); + || } +#if 1 + if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) { + | jmp <4 + |.code + } + } +#endif + + return 1; +} + +static int zend_jit_concat(dasm_State **Dst, const zend_op *opline, int *opnum, zend_op_array *op_array, zend_ssa *ssa) +{ + uint32_t op1_info, op2_info; + + if (!ssa->ops || !ssa->var_info) { + goto fallback; + } + + op1_info = OP1_INFO(); + op2_info = OP2_INFO(); + + if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) { + goto fallback; + } + + if (!(op1_info & MAY_BE_STRING) || + !(op2_info & MAY_BE_STRING)) { + goto fallback; + } + + if (opline->result_type == IS_TMP_VAR && + (opline+1)->opcode == ZEND_SEND_VAL && + (opline+1)->op1_type == IS_TMP_VAR && + (opline+1)->op1.var == opline->result.var) { + /* Eliminate the following SEND_VAL */ + (*opnum)++; + if (!reuse_ip) { + zend_jit_start_reuse_ip(); + | // call = EX(call); + | mov RX, EX->call + } + return zend_jit_concat_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, FP, opline->op1.var, op1_info, opline->op2_type, opline->op2, FP, opline->op2.var, op2_info, RX, (opline+1)->result.var, RES_INFO()); + } else { + return zend_jit_concat_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, FP, opline->op1.var, op1_info, opline->op2_type, opline->op2, FP, opline->op2.var, op2_info, FP, opline->result.var, RES_INFO()); } fallback: @@ -3141,7 +3302,7 @@ fallback: return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); } -static int zend_jit_assign_dim_math(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) +static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) { uint32_t op1_info, op2_info; @@ -3282,11 +3443,24 @@ static int zend_jit_assign_dim_math(dasm_State **Dst, const zend_op *opline, zen if (op1_info & (MAY_BE_ARRAY_OF_REF)) { | ZVAL_DEREF FCARG1a, MAY_BE_REF } - | SEPARATE_ZVAL_NOREF_REG var_info, 1, op_array->filename, opline->lineno } - if (!zend_jit_math_helper(Dst, opline, op_array, ssa, IS_CV, opline->op1, FCARG1a, 0, var_info, (opline+1)->op1_type, (opline+1)->op1, FP, (opline+1)->op1.var, OP1_DATA_INFO(), FCARG1a, 0, OP1_DEF_INFO())) { - return 0; + switch (opline->opcode) { + case ZEND_ASSIGN_ADD: + case ZEND_ASSIGN_SUB: + case ZEND_ASSIGN_MUL: + case ZEND_ASSIGN_DIV: + if (!zend_jit_math_helper(Dst, opline, op_array, ssa, IS_CV, opline->op1, FCARG1a, 0, var_info, (opline+1)->op1_type, (opline+1)->op1, FP, (opline+1)->op1.var, OP1_DATA_INFO(), FCARG1a, 0, OP1_DEF_INFO(), 1)) { + return 0; + } + break; + case ZEND_ASSIGN_CONCAT: + if (!zend_jit_concat_helper(Dst, opline, op_array, ssa, IS_CV, opline->op1, FCARG1a, 0, var_info, (opline+1)->op1_type, (opline+1)->op1, FP, (opline+1)->op1.var, OP1_DATA_INFO(), FCARG1a, 0, OP1_DEF_INFO())) { + return 0; + } + break; + default: + ZEND_ASSERT(0); } } @@ -3391,6 +3565,9 @@ static int zend_jit_assign_dim_math(dasm_State **Dst, const zend_op *opline, zen case ZEND_ASSIGN_DIV: | EXT_CALL zend_jit_assign_dim_div_helper, r0 break; + case ZEND_ASSIGN_CONCAT: + | EXT_CALL zend_jit_assign_dim_concat_helper, r0 + break; default: ZEND_ASSERT(0); } @@ -3412,12 +3589,12 @@ fallback: return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); } -static int zend_jit_assign_math(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) +static int zend_jit_assign_op(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) { uint32_t op1_info, op2_info; if (opline->extended_value == ZEND_ASSIGN_DIM) { - return zend_jit_assign_dim_math(Dst, opline, op_array, ssa); + return zend_jit_assign_dim_op(Dst, opline, op_array, ssa); } else if (opline->extended_value == ZEND_ASSIGN_OBJ) { goto fallback; } @@ -3437,19 +3614,52 @@ static int zend_jit_assign_math(dasm_State **Dst, const zend_op *opline, zend_op goto fallback; } - if (!(op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) || - !(op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { - goto fallback; + switch (opline->opcode) { + case ZEND_ASSIGN_ADD: + case ZEND_ASSIGN_SUB: + case ZEND_ASSIGN_MUL: + case ZEND_ASSIGN_DIV: + if (!(op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) || + !(op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { + goto fallback; + } + break; + case ZEND_ASSIGN_CONCAT: + if (!(op1_info & MAY_BE_STRING) || + !(op2_info & MAY_BE_STRING)) { + goto fallback; + } + break; + default: + ZEND_ASSERT(0); } if (op1_info & MAY_BE_REF) { | LOAD_ZVAL_ADDR FCARG1a, opline->op1_type, opline->op1 | ZVAL_DEREF FCARG1a, op1_info - | SEPARATE_ZVAL_NOREF_REG op1_info, 1, op_array->filename, opline->lineno - return zend_jit_math_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, FCARG1a, 0, op1_info, opline->op2_type, opline->op2, FP, opline->op2.var, op2_info, FCARG1a, 0, OP1_DEF_INFO()); + switch (opline->opcode) { + case ZEND_ASSIGN_ADD: + case ZEND_ASSIGN_SUB: + case ZEND_ASSIGN_MUL: + case ZEND_ASSIGN_DIV: + return zend_jit_math_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, FCARG1a, 0, op1_info, opline->op2_type, opline->op2, FP, opline->op2.var, op2_info, FCARG1a, 0, OP1_DEF_INFO(), 1); + case ZEND_ASSIGN_CONCAT: + return zend_jit_concat_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, FCARG1a, 0, op1_info, opline->op2_type, opline->op2, FP, opline->op2.var, op2_info, FCARG1a, 0, OP1_DEF_INFO()); + default: + ZEND_ASSERT(0); + } } else { - | SEPARATE_ZVAL_NOREF FP + opline->op1.var, op1_info, 1, op_array->filename, opline->lineno - return zend_jit_math_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, FP, opline->op1.var, op1_info, opline->op2_type, opline->op2, FP, opline->op2.var, op2_info, FP, opline->op1.var, OP1_DEF_INFO()); + switch (opline->opcode) { + case ZEND_ASSIGN_ADD: + case ZEND_ASSIGN_SUB: + case ZEND_ASSIGN_MUL: + case ZEND_ASSIGN_DIV: + return zend_jit_math_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, FP, opline->op1.var, op1_info, opline->op2_type, opline->op2, FP, opline->op2.var, op2_info, FP, opline->op1.var, OP1_DEF_INFO(), 1); + case ZEND_ASSIGN_CONCAT: + return zend_jit_concat_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, FP, opline->op1.var, op1_info, opline->op2_type, opline->op2, FP, opline->op2.var, op2_info, FP, opline->op1.var, OP1_DEF_INFO()); + default: + ZEND_ASSERT(0); + } } fallback: From 3646f0bc32d3451f180b0a430787f62ce66f15b8 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 11 Oct 2016 20:36:15 +0300 Subject: [PATCH 286/569] JIT for rare SEND_VAR cases --- ext/opcache/jit/zend_jit_x86.dasc | 62 ++++++++++++++++++++++++------- 1 file changed, 49 insertions(+), 13 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 8a0de96a3fd28..81dd583e1d77f 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -4963,15 +4963,6 @@ fallback: static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) { uint32_t op1_info; - if (!ssa->ops || !ssa->var_info) { - goto fallback; - } - - op1_info = OP1_INFO(); - if (op1_info & (MAY_BE_UNDEF|MAY_BE_REF)) { - // TODO: support for references ??? - goto fallback; - } op1_info = OP1_INFO(); @@ -4981,10 +4972,55 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, zend_op_ar | mov RX, EX->call } - | ZVAL_COPY_VALUE RX + opline->result.var, FP + opline->op1.var, op1_info, r0, eax, r2 - || if (opline->op1_type == IS_CV) { - | TRY_ADDREF op1_info, ah, r2 - || } + if (op1_info & MAY_BE_UNDEF) { + if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { + | IF_Z_TYPE FP + opline->op1.var, IS_UNDEF, >1 + |.cold_code + |1: + | SAVE_VALID_OPLINE opline + | mov FCARG1d, opline->op1.var + | EXT_CALL zend_jit_undefined_op_helper, r0 + | SET_Z_TYPE_INFO RX + opline->result.var, IS_NULL + | jmp >7 + |.code + } + } + + if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { + if (op1_info & MAY_BE_REF) { + if (opline->op1_type == IS_CV) { + | lea FCARG1a, [FP + opline->op1.var] + | ZVAL_DEREF FCARG1a, op1_info + | ZVAL_COPY_VALUE RX + opline->result.var, FCARG1a, op1_info, r0, eax, r2 + | TRY_ADDREF op1_info, ah, r2 + } else { + | IF_Z_TYPE FP + opline->op1.var, IS_REFERENCE, >1 + |.cold_code + |1: + | // zend_refcounted *ref = Z_COUNTED_P(retval_ptr); + | GET_Z_PTR FCARG1a, FP + opline->op1.var + | // ZVAL_COPY_VALUE(return_value, &ref->value); + | ZVAL_COPY_VALUE RX + opline->result.var, FCARG1a + 8, op1_info, r0, eax, r2 + | GC_DELREF FCARG1a + | je >1 + | IF_NOT_REFCOUNTED ah, >2 + | GC_ADDREF r2 + | jmp >2 + |1: + | EFREE_SIZE FCARG1a, sizeof(zend_reference), op_array, opline + | jmp >2 + |.code + | ZVAL_COPY_VALUE RX + opline->result.var, FP + opline->op1.var, op1_info, r0, eax, r2 + |2: + } + } else { + | ZVAL_COPY_VALUE RX + opline->result.var, FP + opline->op1.var, op1_info, r0, eax, r2 + || if (opline->op1_type == IS_CV) { + | TRY_ADDREF op1_info, ah, r2 + || } + } + } + |7: return 1; From 07670f404ddc65045a9eef76cd6e1e57de5e97b6 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 12 Oct 2016 01:59:58 +0300 Subject: [PATCH 287/569] Implemented JIT for SEND_REF and SEND_VAR_EX --- ext/opcache/jit/zend_jit.c | 6 ++ ext/opcache/jit/zend_jit_x86.dasc | 165 +++++++++++++++++++++++++++++- 2 files changed, 167 insertions(+), 4 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index b9d975d9d9286..1fbe26be7ba46 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -1061,7 +1061,13 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) goto jit_failure; } goto done; + case ZEND_SEND_REF: + if (!zend_jit_send_ref(&dasm_state, opline, op_array, ssa, 0)) { + goto jit_failure; + } + goto done; case ZEND_SEND_VAR: + case ZEND_SEND_VAR_EX: if (!zend_jit_send_var(&dasm_state, opline, op_array, ssa)) { goto jit_failure; } diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 81dd583e1d77f..6f9b0e7eb1d76 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -1113,6 +1113,24 @@ static void* dasm_labels[zend_lb_MAX]; | EXT_CALL _efree, r0 |.endmacro +|.macro EMALLOC, size, op_array, opline +| mov FCARG1a, size +|| if (ZEND_DEBUG) { +|| const char *filename = op_array->filename ? op_array->filename->val : NULL; +| LOAD_ADDR FCARG2a, filename +|.if X64 +| mov CARG3d, opline->lineno +| xor CARG4, CARG4 +| xor CARG5, CARG5 +|.else +| push 0 +| push 0 +| push opline->lineno +|.endif +|| } +| EXT_CALL _emalloc, r0 +|.endmacro + |.macro OBJ_RELEASE, reg, exit_label | GC_DELREF reg | jne >1 @@ -4960,9 +4978,121 @@ fallback: return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); } +static int zend_jit_send_ref(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, int cold) +{ + uint32_t op1_info; + + op1_info = OP1_INFO(); + + if (!reuse_ip) { + zend_jit_start_reuse_ip(); + | // call = EX(call); + | mov RX, EX->call + } + + if (opline->op1_type == IS_VAR) { + | lea r0, [FP + opline->op1.var] + | // if (EXPECTED(Z_TYPE_P(ret) == IS_INDIRECT)) { + | IF_NOT_Z_TYPE r0, IS_INDIRECT, >1 + | // ret = Z_INDIRECT_P(ret); + | GET_Z_PTR r0, r0 + |1: + if (op1_info & MAY_BE_ERROR) { + if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { + if (cold) { + | IF_NOT_Z_TYPE r0, _IS_ERROR, >1 + } else { + | IF_Z_TYPE r0, _IS_ERROR, >1 + |.cold_code + |1: + } + } + + | // ZVAL_NEW_EMPTY_REF(arg); + | EMALLOC sizeof(zend_reference), op_array, opline + | SET_Z_PTR RX + opline->result.var, r0 + | SET_Z_TYPE_INFO RX + opline->result.var, IS_REFERENCE_EX + | mov dword [r0], 1 + | mov dword [r0 + 4], IS_REFERENCE; + | // ZVAL_NULL(Z_REFVAL_P(arg)); + | SET_Z_TYPE_INFO r0 + 8, IS_NULL + + if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { + | jmp >7 + if (cold) { + |1: + } else { + |.code + } + } + } + } else if (opline->op1_type == IS_CV) { + if (op1_info & MAY_BE_UNDEF) { + if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { + | IF_NOT_Z_TYPE FP + opline->op1.var, IS_UNDEF, >1 + } + | SET_Z_TYPE_INFO FP + opline->op1.var, IS_NULL + if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { + | jmp >2 + |1: + } + } + } else { + ZEND_ASSERT(0); + } + + if (op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) { + if (op1_info & MAY_BE_REF) { + if (opline->op1_type == IS_VAR) { + | IF_NOT_Z_TYPE r0, IS_REFERENCE, >2 + | GET_Z_PTR r1, r0 + } else { + | IF_NOT_Z_TYPE FP + opline->op1.var, IS_REFERENCE, >2 + | GET_Z_PTR r1, FP + opline->op1.var + } + | GC_ADDREF r1 + | SET_Z_PTR RX + opline->result.var, r1 + | SET_Z_TYPE_INFO RX + opline->result.var, IS_REFERENCE_EX + | jmp >6 + } + |2: + | // ZVAL_NEW_REF(arg, varptr); + if (opline->op1_type == IS_VAR) { + | mov aword [r4], r0 // save + } + | EMALLOC sizeof(zend_reference), op_array, opline + | mov dword [r0], 2 + | mov dword [r0 + 4], IS_REFERENCE; + if (opline->op1_type == IS_VAR) { + | mov r1, aword [r4] // restore + | ZVAL_COPY_VALUE_clobber_src r0 + 8, r1, op1_info, r1, ecx, r2 + | mov r1, aword [r4] // restore + | SET_Z_PTR r1, r0 + | SET_Z_TYPE_INFO r1, IS_REFERENCE_EX + } else { + | ZVAL_COPY_VALUE r0 + 8, FP + opline->op1.var, op1_info, r1, ecx, r2 + | SET_Z_PTR FP + opline->op1.var, r0 + | SET_Z_TYPE_INFO FP + opline->op1.var, IS_REFERENCE_EX + } + | SET_Z_PTR RX + opline->result.var, r0 + | SET_Z_TYPE_INFO RX + opline->result.var, IS_REFERENCE_EX + } + + |6: + | FREE_OP opline->op1_type, opline->op1, op1_info, !cold, op_array, opline->lineno + |7: + + return 1; +} + static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) { uint32_t op1_info; + uint32_t arg_num = opline->op2.num; + + if (opline->opcode == ZEND_SEND_VAR_EX && arg_num > MAX_ARG_FLAG_NUM) { + goto fallback; + } op1_info = OP1_INFO(); @@ -4972,15 +5102,42 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, zend_op_ar | mov RX, EX->call } + if (opline->opcode == ZEND_SEND_VAR_EX) { + uint32_t mask = ZEND_SEND_BY_REF << ((arg_num + 3) * 2); + + | mov r0, EX:RX->func + if (arg_num <= MAX_ARG_FLAG_NUM) { + | mov eax, dword [r0 + offsetof(zend_function, quick_arg_flags)] + | test eax, mask + | jnz >1 + } else { + ZEND_ASSERT(0); + } + + |.cold_code + |1: + + if (!zend_jit_send_ref(Dst, opline, op_array, ssa, 1)) { + return 0; + } + + | jmp >7 + |.code + } + if (op1_info & MAY_BE_UNDEF) { if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { | IF_Z_TYPE FP + opline->op1.var, IS_UNDEF, >1 |.cold_code |1: - | SAVE_VALID_OPLINE opline - | mov FCARG1d, opline->op1.var - | EXT_CALL zend_jit_undefined_op_helper, r0 - | SET_Z_TYPE_INFO RX + opline->result.var, IS_NULL + } + + | SAVE_VALID_OPLINE opline + | mov FCARG1d, opline->op1.var + | EXT_CALL zend_jit_undefined_op_helper, r0 + | SET_Z_TYPE_INFO RX + opline->result.var, IS_NULL + + if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { | jmp >7 |.code } From 9afd3602ed9cb0823ac96db1e9642c51e15ba44e Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 12 Oct 2016 02:41:18 +0300 Subject: [PATCH 288/569] Prevent comparison of constants with variables --- ext/opcache/jit/zend_jit_x86.dasc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 6f9b0e7eb1d76..bdd7aaca63500 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -1820,13 +1820,13 @@ static int zend_jit_math_long_long(dasm_State **Dst, | jmp >2 |.code | SET_Z_LVAL Ra(res_reg)+res_offset, r0 - if (op1_reg != res_reg || op1_offset != res_offset) { + if (op1_type == IS_CONST || op1_reg != res_reg || op1_offset != res_offset) { | SET_Z_TYPE_INFO Ra(res_reg)+res_offset, IS_LONG } |2: } else { | SET_Z_LVAL Ra(res_reg)+res_offset, r0 - if (op1_reg != res_reg || op1_offset != res_offset) { + if (op1_type == IS_CONST || op1_reg != res_reg || op1_offset != res_offset) { | SET_Z_TYPE_INFO Ra(res_reg)+res_offset, IS_LONG } } @@ -1936,7 +1936,7 @@ static int zend_jit_math_double_long(dasm_State **Dst, || } |.endif | DOUBLE_STORE Ra(res_reg)+res_offset, xmm0 - if (op1_reg != res_reg || op1_offset != res_offset) { + if (op1_type == IS_CONST || op1_reg != res_reg || op1_offset != res_offset) { | SET_Z_TYPE_INFO Ra(res_reg)+res_offset, IS_DOUBLE } return 1; @@ -1983,7 +1983,7 @@ static int zend_jit_math_double_double(dasm_State **Dst, } |.endif | DOUBLE_STORE Ra(res_reg)+res_offset, xmm0 - if (op1_reg != res_reg || op1_offset != res_offset) { + if (op1_type == IS_CONST || op1_reg != res_reg || op1_offset != res_offset) { | SET_Z_TYPE_INFO Ra(res_reg)+res_offset, IS_DOUBLE } @@ -2336,7 +2336,7 @@ static int zend_jit_concat_helper(dasm_State **Dst, if (op2_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) { | IF_NOT_Z_TYPE Ra(op2_reg)+op2_offset, IS_STRING, >5 } - if (op1_reg == res_reg && op1_offset == res_offset) { + if (op1_type != IS_CONST && op1_reg == res_reg && op1_offset == res_offset) { if (res_reg != FCARG1a || res_offset != 0) { | lea FCARG1a, [Ra(res_reg)+res_offset] } From c6538a4194824f26b5d827da5dcd85867f37c74b Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 12 Oct 2016 15:32:28 +0300 Subject: [PATCH 289/569] Implemented JIT for TYPE_CHECK (except for is_resource) --- ext/opcache/jit/zend_jit_x86.dasc | 323 ++++++++++++++++++++++++++---- 1 file changed, 286 insertions(+), 37 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index bdd7aaca63500..096455dc04036 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -5186,56 +5186,305 @@ fallback: return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); } +static int zend_jit_smart_true(dasm_State **Dst, const zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa, int jmp) +{ + uint32_t target_label; + + if ((opline+1)->opcode == ZEND_JMPZ && + (opline+1)->op1_type == IS_TMP_VAR && + (opline+1)->op1.var == opline->result.var) { + if (jmp) { + | jmp >7 + } + } else if ((opline+1)->opcode == ZEND_JMPNZ && + (opline+1)->op1_type == IS_TMP_VAR && + (opline+1)->op1.var == opline->result.var) { + target_label = ssa->cfg.blocks[b].successors[0]; + | jmp =>target_label + } else if ((opline+1)->opcode == ZEND_JMPZNZ && + (opline+1)->op1_type == IS_TMP_VAR && + (opline+1)->op1.var == opline->result.var) { + target_label = ssa->cfg.blocks[b].successors[1]; + | jmp =>target_label + } else { + | SET_Z_TYPE_INFO FP + opline->result.var, IS_TRUE + if (jmp) { + | jmp >7 + } + } + + return 1; +} + +static int zend_jit_smart_false(dasm_State **Dst, const zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa, int jmp) +{ + uint32_t target_label; + + if ((opline+1)->opcode == ZEND_JMPZ && + (opline+1)->op1_type == IS_TMP_VAR && + (opline+1)->op1.var == opline->result.var) { + target_label = ssa->cfg.blocks[b].successors[0]; + | jmp =>target_label + } else if ((opline+1)->opcode == ZEND_JMPNZ && + (opline+1)->op1_type == IS_TMP_VAR && + (opline+1)->op1.var == opline->result.var) { + if (jmp) { + | jmp >7 + } + } else if ((opline+1)->opcode == ZEND_JMPZNZ && + (opline+1)->op1_type == IS_TMP_VAR && + (opline+1)->op1.var == opline->result.var) { + target_label = ssa->cfg.blocks[b].successors[0]; + | jmp =>target_label + } else { + | SET_Z_TYPE_INFO FP + opline->result.var, IS_FALSE + if (jmp) { + | jmp >7 + } + } + + return 1; +} + static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, int b, int *opnum, zend_op_array *op_array, zend_ssa *ssa) { uint32_t op1_info, mask; - unsigned int target_label; + uint32_t target_label; zend_uchar type; + zend_bool smart_branch = 0; - if (!ssa->ops || !ssa->var_info || opline->extended_value == IS_RESOURCE) { + if (opline->extended_value == IS_RESOURCE) { + // TODO: support for is_resource() ??? goto fallback; } + if (((opline+1)->opcode == ZEND_JMPZ || + (opline+1)->opcode == ZEND_JMPNZ || + (opline+1)->opcode == ZEND_JMPZNZ) && + (opline+1)->op1_type == IS_TMP_VAR && + (opline+1)->op1.var == opline->result.var) { + (*opnum)++; + smart_branch = 1; + } + op1_info = OP1_INFO(); - type = opline->extended_value; - if (type == IS_RESOURCE) { - goto fallback; - } else if (type == _IS_BOOL) { - mask = MAY_BE_FALSE | MAY_BE_TRUE; - } else { - mask = (1 << type); - } - if (!(op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF) - mask))) { - if ((opline+1)->opcode == ZEND_JMPZ && - (opline+1)->op1_type == IS_TMP_VAR && - (opline+1)->op1.var == opline->result.var) { - (*opnum)++; - } else if ((opline+1)->opcode == ZEND_JMPNZ && - (opline+1)->op1_type == IS_TMP_VAR && - (opline+1)->op1.var == opline->result.var) { - (*opnum)++; - target_label = ssa->cfg.blocks[b].successors[0]; - | jmp =>target_label + if (op1_info & MAY_BE_UNDEF) { + if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { + | IF_Z_TYPE FP + opline->op1.var, IS_UNDEF, >1 + |.cold_code + |1: + } + | SAVE_VALID_OPLINE opline + | mov FCARG1d, opline->op1.var + | EXT_CALL zend_jit_undefined_op_helper, r0 + if (opline->extended_value == IS_NULL) { + if (!zend_jit_smart_true(Dst, opline, b, op_array, ssa, (op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0)) { + return 0; + } } else { - | SET_Z_TYPE_INFO FP + opline->result.var, IS_TRUE + if (!zend_jit_smart_false(Dst, opline, b, op_array, ssa, (op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0)) { + return 0; + } } - } else if (!(op1_info & mask)) { - if ((opline+1)->opcode == ZEND_JMPZ && - (opline+1)->op1_type == IS_TMP_VAR && - (opline+1)->op1.var == opline->result.var) { - (*opnum)++; - target_label = ssa->cfg.blocks[b].successors[0]; - | jmp =>target_label - } else if ((opline+1)->opcode == ZEND_JMPNZ && - (opline+1)->op1_type == IS_TMP_VAR && - (opline+1)->op1.var == opline->result.var) { - (*opnum)++; + if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { + |.code + } + } + + if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { + type = opline->extended_value; + if (type == _IS_BOOL) { + mask = MAY_BE_FALSE | MAY_BE_TRUE; } else { - | SET_Z_TYPE_INFO FP + opline->result.var, IS_FALSE + mask = (1 << type); } - } else { - goto fallback; - } + + if (!(op1_info & (MAY_BE_ANY - mask))) { + | FREE_OP opline->op1_type, opline->op1, op1_info, 1, op_array, opline->lineno + if (!zend_jit_smart_true(Dst, opline, b, op_array, ssa, 0)) { + return 0; + } + } else if (!(op1_info & mask)) { + | FREE_OP opline->op1_type, opline->op1, op1_info, 1, op_array, opline->lineno + if (!zend_jit_smart_false(Dst, opline, b, op_array, ssa, 0)) { + return 0; + } + } else { + if (op1_info & MAY_BE_REF) { + | LOAD_ZVAL_ADDR r0, opline->op1_type, opline->op1 + | ZVAL_DEREF r0, op1_info + } + if (opline->extended_value == _IS_BOOL) { + if (smart_branch && + (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && + (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { + if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { + | // if (Z_REFCOUNTED_P(cv)) { + if (op1_info & MAY_BE_REF) { + | IF_Z_REFCOUNTED r0, >1 + } else { + | IF_Z_REFCOUNTED FP + opline->op1.var, >1 + } + |.cold_code + |1: + } + | // if (!Z_DELREF_P(cv)) { + if (op1_info & MAY_BE_REF) { + | GET_Z_PTR FCARG1a, r0 + } else { + | GET_Z_PTR FCARG1a, FP + opline->op1.var + } + | GC_DELREF FCARG1a + if (RC_MAY_BE_1(op1_info)) { + if (RC_MAY_BE_N(op1_info)) { + | jnz >1 + } + if (op1_info & MAY_BE_REF) { + | mov al, byte [r0 + 8] + } else { + | mov al, byte [FP + opline->op1.var + 8] + } + | mov byte [r4], al // save + | // zval_dtor_func(r); + | ZVAL_DTOR_FUNC op_array->filename, opline->lineno + | mov cl, byte [r4] // restore + |jmp >2 + } + if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { + if (!RC_MAY_BE_1(op1_info)) { + | jmp >1 + } + |.code + } + |1: + if (op1_info & MAY_BE_REF) { + | mov cl, byte [r0 + 8] + } else { + | mov cl, byte [FP + opline->op1.var + 8] + } + |2: + } else { + if (op1_info & MAY_BE_REF) { + | mov cl, byte [r0 + 8] + } else { + | mov cl, byte [FP + opline->op1.var + 8] + } + } + | mov al, 1 + | shl al, cl + | test al, 0xc + if ((opline+1)->opcode == ZEND_JMPZ && + (opline+1)->op1_type == IS_TMP_VAR && + (opline+1)->op1.var == opline->result.var) { + target_label = ssa->cfg.blocks[b].successors[0]; + | je =>target_label + } else if ((opline+1)->opcode == ZEND_JMPNZ && + (opline+1)->op1_type == IS_TMP_VAR && + (opline+1)->op1.var == opline->result.var) { + target_label = ssa->cfg.blocks[b].successors[0]; + | jne =>target_label + } else if ((opline+1)->opcode == ZEND_JMPZNZ && + (opline+1)->op1_type == IS_TMP_VAR && + (opline+1)->op1.var == opline->result.var) { + target_label = ssa->cfg.blocks[b].successors[0]; + | je =>target_label + target_label = ssa->cfg.blocks[b].successors[1]; + | jmp =>target_label + } else { + | setne al + | movzx eax, al + | add eax, 2 + | SET_Z_TYPE_INFO FP + opline->result.var, eax + | FREE_OP opline->op1_type, opline->op1, op1_info, 1, op_array, opline->lineno + } + } else if (opline->extended_value == IS_RESOURCE) { + ZEND_ASSERT(0); + } else { + if (smart_branch && + (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && + (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { + if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { + | // if (Z_REFCOUNTED_P(cv)) { + if (op1_info & MAY_BE_REF) { + | IF_Z_REFCOUNTED r0, >1 + } else { + | IF_Z_REFCOUNTED FP + opline->op1.var, >1 + } + |.cold_code + |1: + } + | // if (!Z_DELREF_P(cv)) { + if (op1_info & MAY_BE_REF) { + | GET_Z_PTR FCARG1a, r0 + } else { + | GET_Z_PTR FCARG1a, FP + opline->op1.var + } + | GC_DELREF FCARG1a + if (RC_MAY_BE_1(op1_info)) { + if (RC_MAY_BE_N(op1_info)) { + | jnz >1 + } + if (op1_info & MAY_BE_REF) { + | mov al, byte [r0 + 8] + } else { + | mov al, byte [FP + opline->op1.var + 8] + } + | mov byte [r4], al // save + | // zval_dtor_func(r); + | ZVAL_DTOR_FUNC op_array->filename, opline->lineno + | mov cl, byte [r4] // restore + |jmp >2 + } + if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { + if (!RC_MAY_BE_1(op1_info)) { + | jmp >1 + } + |.code + } + |1: + if (op1_info & MAY_BE_REF) { + | mov cl, byte [r0 + 8] + } else { + | mov cl, byte [FP + opline->op1.var + 8] + } + |2: + | cmp cl, opline->extended_value + } else { + if (op1_info & MAY_BE_REF) { + | cmp byte [r0 + 8], opline->extended_value + } else { + | cmp byte [FP + opline->op1.var + 8], opline->extended_value + } + } + if ((opline+1)->opcode == ZEND_JMPZ && + (opline+1)->op1_type == IS_TMP_VAR && + (opline+1)->op1.var == opline->result.var) { + target_label = ssa->cfg.blocks[b].successors[0]; + | jne =>target_label + } else if ((opline+1)->opcode == ZEND_JMPNZ && + (opline+1)->op1_type == IS_TMP_VAR && + (opline+1)->op1.var == opline->result.var) { + target_label = ssa->cfg.blocks[b].successors[0]; + | je =>target_label + } else if ((opline+1)->opcode == ZEND_JMPZNZ && + (opline+1)->op1_type == IS_TMP_VAR && + (opline+1)->op1.var == opline->result.var) { + target_label = ssa->cfg.blocks[b].successors[0]; + | jne =>target_label + target_label = ssa->cfg.blocks[b].successors[1]; + | jmp =>target_label + } else { + | sete al + | movzx eax, al + | add eax, 2 + | SET_Z_TYPE_INFO FP + opline->result.var, eax + | FREE_OP opline->op1_type, opline->op1, op1_info, 1, op_array, opline->lineno + } + } + } + } + + |7: return 1; From cfa96589057db551a416fb3ec0f33203c1b11e90 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 12 Oct 2016 20:38:09 +0300 Subject: [PATCH 290/569] Jump optimization and duplicate code removing --- ext/opcache/jit/zend_jit_x86.dasc | 78 +++++++++++-------------------- 1 file changed, 27 insertions(+), 51 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 096455dc04036..92f1aec00153b 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -2464,7 +2464,7 @@ fallback: } static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, uint32_t type, uint32_t op1_info, uint32_t op2_info, uint32_t found, uint32_t not_found) -/* Labels: 1,2,3 */ +/* Labels: 1,2,3,4,5 */ { if (op2_info & MAY_BE_LONG) { if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_LONG)) { @@ -2479,7 +2479,7 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o if (val >= 0 && val < HT_MAX_SIZE) { | // ZEND_HASH_INDEX_FIND(ht, hval, retval, num_undef); | test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED - | jz >1 // HASH_FIND + | jz >4 // HASH_FIND | // if (EXPECTED((zend_ulong)(_h) < (zend_ulong)(_ht)->nNumUsed)) |.if X64 | movsxd r0, dword [FCARG1a + offsetof(zend_array, nNumUsed)] @@ -2496,12 +2496,7 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o | mov r0, aword [FCARG1a + offsetof(zend_array, arData)] | add r0, val * sizeof(Bucket) if (type == BP_JIT_IS) { - | IF_NOT_Z_TYPE r0, IS_REFERENCE, >2 - | GET_Z_PTR r0, r0 - | add r0, 8 - |2: - | cmp byte [r0 + 8], IS_NULL - | jg >8 + | jmp >5 } else { | IF_NOT_Z_TYPE r0, IS_UNDEF, >8 } @@ -2509,7 +2504,7 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o } else { | // ZEND_HASH_INDEX_FIND(ht, hval, retval, num_undef); | test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED - | jz >1 // HASH_FIND + | jz >4 // HASH_FIND | // if (EXPECTED((zend_ulong)(_h) < (zend_ulong)(_ht)->nNumUsed)) |.if X64 | movsxd r0, dword [FCARG1a + offsetof(zend_array, nNumUsed)] @@ -2531,12 +2526,7 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o |.endif | add r0, aword [FCARG1a + offsetof(zend_array, arData)] if (type == BP_JIT_IS) { - | IF_NOT_Z_TYPE r0, IS_REFERENCE, >2 - | GET_Z_PTR r0, r0 - | add r0, 8 - |2: - | cmp byte [r0 + 8], IS_NULL - | jg >8 + | jmp >5 } else { | IF_NOT_Z_TYPE r0, IS_UNDEF, >8 } @@ -2545,26 +2535,14 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o switch (type) { case BP_JIT_IS: if (op1_info & MAY_BE_ARRAY_KEY_LONG) { - if (opline->op2_type == IS_CONST) { - zend_long val = Z_LVAL_P(RT_CONSTANT(op_array, opline->op2)); - if (val >= 0 && val < HT_MAX_SIZE) { - | jmp >9 // NOT_FOUND - } - } else { - | jmp >9 // NOT_FOUND - } - |1: + |4: } | EXT_CALL zend_hash_index_find, r0 | test r0, r0 | jz >9 // NOT_FOUND - | IF_NOT_Z_TYPE r0, IS_REFERENCE, >1 - | GET_Z_PTR r0, r0 - | add r0, 8 - |1: - | cmp byte [r0 + 8], IS_NULL - | jg >8 - | jmp >9 // NOT_FOUND + if (op2_info & MAY_BE_STRING) { + | jmp >5 + } break; case BP_VAR_R: case BP_VAR_IS: @@ -2578,7 +2556,7 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o } else { | jmp >2 // NOT_FOUND } - |1: + |4: } | EXT_CALL zend_hash_index_find, r0 | test r0, r0 @@ -2623,7 +2601,7 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o | EXT_CALL zend_jit_fetch_dimension_rw_long_helper, r0 if (op1_info & MAY_BE_ARRAY_KEY_LONG) { | jmp >8 - |1: + |4: | SAVE_VALID_OPLINE opline | EXT_CALL zend_jit_hash_index_lookup_rw, r0 } @@ -2649,7 +2627,7 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o | EXT_CALL _zend_hash_index_add_new, r0 if (op1_info & MAY_BE_ARRAY_KEY_LONG) { | jmp >8 - |1: + |4: | EXT_CALL zend_jit_hash_index_lookup_w, r0 } break; @@ -2657,7 +2635,7 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o ZEND_ASSERT(0); } - if (op2_info & MAY_BE_STRING) { + if (type != BP_JIT_IS && (op2_info & MAY_BE_STRING)) { | jmp >8 } } @@ -2684,13 +2662,6 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o | IF_NOT_Z_TYPE r0, IS_INDIRECT, >1 | GET_Z_PTR r0, r0 |1: - | IF_NOT_Z_TYPE r0, IS_REFERENCE, >1 - | GET_Z_PTR r0, r0 - | add r0, 8 - |1: - | cmp byte [r0 + 8], IS_NULL - | jg >8 - | jmp >9 break; case BP_VAR_R: case BP_VAR_IS: @@ -2760,9 +2731,15 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o default: ZEND_ASSERT(0); } - if (op2_info & (MAY_BE_ANY - (MAY_BE_LONG|MAY_BE_STRING))) { - | jmp >8 + } + + if (type == BP_JIT_IS && (op2_info & (MAY_BE_LONG|MAY_BE_STRING))) { + |5: + if (op1_info & MAY_BE_ARRAY_OF_REF) { + | ZVAL_DEREF r0, MAY_BE_REF } + | cmp byte [r0 + 8], IS_NULL + | jle >9 // NOT FOUND } if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) { @@ -4366,8 +4343,11 @@ static int zend_jit_jmpznz(dasm_State **Dst, const zend_op *opline, int b, zend_ if (op1_info & MAY_BE_TRUE) { if (true_label != (uint32_t)-1) { | IF_Z_TYPE FCARG1a, IS_TRUE, =>true_label - } else { + } else if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) { | IF_Z_TYPE FCARG1a, IS_TRUE, >9 + } else { + | //IF_Z_TYPE FCARG1a, IS_TRUE, >9 + | cmp byte [FCARG1a + 8], IS_TRUE } } else { | cmp byte [FCARG1a + 8], IS_TRUE @@ -4932,8 +4912,7 @@ static int zend_jit_send_val(dasm_State **Dst, const zend_op *opline, zend_op_ar | mov r0, EX:RX->func if (arg_num <= MAX_ARG_FLAG_NUM) { - | mov eax, dword [r0 + offsetof(zend_function, quick_arg_flags)] - | test eax, mask + | test dword [r0 + offsetof(zend_function, quick_arg_flags)], mask | jnz >1 } else { ZEND_ASSERT(0); @@ -5107,8 +5086,7 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, zend_op_ar | mov r0, EX:RX->func if (arg_num <= MAX_ARG_FLAG_NUM) { - | mov eax, dword [r0 + offsetof(zend_function, quick_arg_flags)] - | test eax, mask + | test dword [r0 + offsetof(zend_function, quick_arg_flags)], mask | jnz >1 } else { ZEND_ASSERT(0); @@ -6073,8 +6051,6 @@ static int zend_jit_isset_isempty_dim(dasm_State **Dst, const zend_op *opline, i unsigned int target_label = ssa->cfg.blocks[b].successors[0]; | jmp =>target_label } else if ((opline+1)->opcode == ZEND_JMPNZ) { - unsigned int target_label = ssa->cfg.blocks[b].successors[1]; - | jmp =>target_label } else if ((opline+1)->opcode == ZEND_JMPZNZ) { unsigned int target_label = ssa->cfg.blocks[b].successors[0]; | jmp =>target_label From f7520ad39e5c026506c4118889c97dec12601873 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 12 Oct 2016 21:04:19 +0300 Subject: [PATCH 291/569] Proper initialization --- ext/opcache/jit/zend_jit_x86.dasc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 92f1aec00153b..870413fc1335a 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -4551,7 +4551,11 @@ static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, zen |.cold_code |2: | mov FCARG1d, used_stack - | mov FCARG2a, r0 + if (func && func->type == ZEND_INTERNAL_FUNCTION) { + | LOAD_ADDR FCARG2a, func + } else { + | mov FCARG2a, r0 + } | EXT_CALL zend_jit_extend_stack_helper, r0 | mov RX, r0 | jmp <1 From 41cbc14f881261b6ca88f5df35d1fa1cd8f9c41d Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 12 Oct 2016 22:03:12 +0300 Subject: [PATCH 292/569] Reduce code size --- ext/opcache/jit/zend_jit_disasm_x86.c | 1 + ext/opcache/jit/zend_jit_helpers.c | 7 +++++++ ext/opcache/jit/zend_jit_x86.dasc | 21 +++++++++++++-------- 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/ext/opcache/jit/zend_jit_disasm_x86.c b/ext/opcache/jit/zend_jit_disasm_x86.c index b1df09b73e1d4..a38b5d818131f 100644 --- a/ext/opcache/jit/zend_jit_disasm_x86.c +++ b/ext/opcache/jit/zend_jit_disasm_x86.c @@ -385,6 +385,7 @@ static int zend_jit_disasm_init(void) REGISTER_HELPER(memcmp); REGISTER_HELPER(zend_jit_find_func_helper); REGISTER_HELPER(zend_jit_extend_stack_helper); + REGISTER_HELPER(zend_jit_int_extend_stack_helper); REGISTER_HELPER(zend_jit_leave_nested_func_helper); REGISTER_HELPER(zend_jit_leave_top_func_helper); REGISTER_HELPER(zend_jit_symtable_find); diff --git a/ext/opcache/jit/zend_jit_helpers.c b/ext/opcache/jit/zend_jit_helpers.c index 1d13c5e1895aa..d06b727db4594 100644 --- a/ext/opcache/jit/zend_jit_helpers.c +++ b/ext/opcache/jit/zend_jit_helpers.c @@ -43,6 +43,13 @@ static zend_execute_data* ZEND_FASTCALL zend_jit_extend_stack_helper(uint32_t us return call; } +static zend_execute_data* ZEND_FASTCALL zend_jit_int_extend_stack_helper(uint32_t used_stack) +{ + zend_execute_data *call = (zend_execute_data*)zend_vm_stack_extend(used_stack); + ZEND_SET_CALL_INFO(call, 0, ZEND_CALL_NESTED_FUNCTION|ZEND_CALL_ALLOCATED); + return call; +} + static zval* ZEND_FASTCALL zend_jit_symtable_find(HashTable *ht, zend_string *str) { zend_ulong idx; diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 870413fc1335a..cc9e73c317ba4 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -4526,10 +4526,11 @@ static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, zen | // EG(vm_stack_top) = (zval*)((char*)call + used_stack); | add aword [&EG(vm_stack_top)], used_stack | // zend_vm_init_call_frame(call, call_info, func, num_args, called_scope, object); + | // ZEND_SET_CALL_INFO(call, 0, call_info); + | mov dword EX:RX->This.u1.type_info, (IS_UNDEF | (ZEND_CALL_NESTED_FUNCTION << ZEND_CALL_INFO_SHIFT)) | // call->func = func; - if (!func || func->type != ZEND_INTERNAL_FUNCTION) { - | mov aword EX:RX->func, r0 - } else { + if (func && func->type == ZEND_INTERNAL_FUNCTION) { + |1: |.if X64 || if (!IS_32BIT(func)) { | mov aword EX:RX->func, func @@ -4540,10 +4541,10 @@ static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, zen |.else | mov aword EX:RX->func, func |.endif + } else { + | mov aword EX:RX->func, r0 + |1: } - | // ZEND_SET_CALL_INFO(call, 0, call_info); - | mov dword EX:RX->This.u1.type_info, (IS_UNDEF | (ZEND_CALL_NESTED_FUNCTION << ZEND_CALL_INFO_SHIFT)) - |1: | // Z_CE(call->This) = called_scope; | mov aword EX:RX->This.value.ptr, 0 | // ZEND_CALL_NUM_ARGS(call) = num_args; @@ -4552,11 +4553,11 @@ static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, zen |2: | mov FCARG1d, used_stack if (func && func->type == ZEND_INTERNAL_FUNCTION) { - | LOAD_ADDR FCARG2a, func + | EXT_CALL zend_jit_int_extend_stack_helper, r0 } else { | mov FCARG2a, r0 + | EXT_CALL zend_jit_extend_stack_helper, r0 } - | EXT_CALL zend_jit_extend_stack_helper, r0 | mov RX, r0 | jmp <1 |.code @@ -6115,8 +6116,12 @@ static int zend_jit_bind_global(dasm_State **Dst, const zend_op *opline, zend_op |.endif | IF_Z_TYPE r0, IS_UNDEF, >9 // (EXPECTED(p->key == Z_STR_P(varname)) + |.if X64 | LOAD_ADDR r1, Z_PTR_P(varname) | cmp [r0 + offsetof(Bucket, key)], r1 + |.else + | cmp aword [r0 + offsetof(Bucket, key)], Z_PTR_P(varname) + |.endif | jne >1 |.cold_code |1: From f5a2a0d7abfa20affeb1b826ca97cb982e6dd0aa Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 12 Oct 2016 23:37:25 +0300 Subject: [PATCH 293/569] Move exceptional code into stub --- ext/opcache/jit/zend_jit_x86.dasc | 58 ++++++++++++++++++++++--------- 1 file changed, 42 insertions(+), 16 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index cc9e73c317ba4..537c7ccd234c8 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -1349,12 +1349,53 @@ static int zend_jit_icall_throw_stub(dasm_State **Dst) return 1; } +static int zend_jit_throw_cannot_pass_by_ref_stub(dasm_State **Dst) +{ + |->throw_cannot_pass_by_ref: + | mov r0, EX->opline + |.if X64 + | movsxd r1, dword OP:r0->result.var + |.else + | mov r1, OP:r0->result.var + |.endif + | SET_Z_TYPE_INFO RX+r1, IS_UNDEF + | mov RX, r0 + |.if X64 + | mov CARG1, 0 + | LOAD_ADDR CARG2, "Cannot pass parameter %d by reference" + | mov CARG3d, dword OP:r0->op2.num + | EXT_CALL zend_throw_error, r0 + |.else + | mov r1, dword OP:r0->op2.num + | sub r4, 4 + | push r1 + | push "Cannot pass parameter %d by reference" + | push 0 + | EXT_CALL zend_throw_error, r0 + | add r4, 16 + |.endif + | cmp byte OP:RX->op1_type, IS_TMP_VAR + | jne >9 + |.if X64 + | movsxd r0, dword OP:RX->op1.var + |.else + | mov r0, OP:RX->op1.var + |.endif + | add r0, FP + | ZVAL_PTR_DTOR r0, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, 0, 0, NULL, 0 + |9: + | jmp ->exception_handler + + return 1; +} + static const zend_jit_stub zend_jit_stubs[] = { JIT_STUB(interrupt_handler), JIT_STUB(exception_handler), JIT_STUB(leave_function), JIT_STUB(leave_throw), JIT_STUB(icall_throw), + JIT_STUB(throw_cannot_pass_by_ref), }; static int zend_jit_align_func(dasm_State **Dst) @@ -4924,23 +4965,8 @@ static int zend_jit_send_val(dasm_State **Dst, const zend_op *opline, zend_op_ar } |.cold_code |1: - | SET_Z_TYPE_INFO RX + opline->result.var, IS_UNDEF | SAVE_VALID_OPLINE opline - |.if X64 - | mov CARG1, 0 - | LOAD_ADDR CARG2, "Cannot pass parameter %d by reference" - | mov CARG3d, arg_num - | EXT_CALL zend_throw_error, r0 - |.else - | sub r4, 4 - | push arg_num - | push "Cannot pass parameter %d by reference" - | push 0 - | EXT_CALL zend_throw_error, r0 - | add r4, 16 - |.endif - | FREE_OP opline->op1_type, opline->op1, op1_info, 0, op_array, opline->lineno - | jmp ->exception_handler + | jmp ->throw_cannot_pass_by_ref |.code } From 5dcd9bdb640e12948f79edc38ea555ba71b123fb Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 13 Oct 2016 00:26:27 +0300 Subject: [PATCH 294/569] Improved efree() calls --- ext/opcache/jit/zend_jit_x86.dasc | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 537c7ccd234c8..ecaf80ea86f3a 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -1095,9 +1095,8 @@ static void* dasm_labels[zend_lb_MAX]; || } |.endmacro -|.macro EFREE_SIZE, ptr, size, op_array, opline -| mov FCARG1a, ptr -|| if (ZEND_DEBUG) { +|.macro EFREE_REG_24, op_array, opline +||#if ZEND_DEBUG || const char *filename = op_array->filename ? op_array->filename->val : NULL; | LOAD_ADDR FCARG2a, filename |.if X64 @@ -1109,8 +1108,15 @@ static void* dasm_labels[zend_lb_MAX]; | push 0 | push opline->lineno |.endif -|| } -| EXT_CALL _efree, r0 +| EXT_CALL _efree, r0 +||#else +| EXT_CALL _efree_24, r0 +||#endif +|.endmacro + +|.macro EFREE_24, ptr, op_array, opline +| mov FCARG1a, ptr +| EFREE_REG_24 op_array, opline |.endmacro |.macro EMALLOC, size, op_array, opline @@ -2941,7 +2947,7 @@ static int zend_jit_simple_assign(dasm_State **Dst, | GC_ADDREF r0 |2: } - | EFREE_SIZE aword [FP + val.var], sizeof(zend_reference), op_array, opline + | EFREE_24 aword [FP + val.var], op_array, opline | jmp >3 if (in_cold) { |1: @@ -5173,7 +5179,7 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, zend_op_ar | GC_ADDREF r2 | jmp >2 |1: - | EFREE_SIZE FCARG1a, sizeof(zend_reference), op_array, opline + | EFREE_REG_24 op_array, opline | jmp >2 |.code | ZVAL_COPY_VALUE RX + opline->result.var, FP + opline->op1.var, op1_info, r0, eax, r2 @@ -5794,7 +5800,7 @@ static int zend_jit_return(dasm_State **Dst, const zend_op *opline, zend_op_arra | jmp >9 || } |2: - | EFREE_SIZE r0, sizeof(zend_reference), op_array, opline + | EFREE_24 r0, op_array, opline || if (jit_return_label >= 0) { | jmp =>jit_return_label || } else { From 73c27a72d68d7783a3909dfb9c6dc30969f12526 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 13 Oct 2016 00:59:09 +0300 Subject: [PATCH 295/569] Optimise emalloc() calls --- ext/opcache/jit/zend_jit_x86.dasc | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index ecaf80ea86f3a..a6bd00e4eea43 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -1120,9 +1120,9 @@ static void* dasm_labels[zend_lb_MAX]; |.endmacro |.macro EMALLOC, size, op_array, opline -| mov FCARG1a, size -|| if (ZEND_DEBUG) { +||#if ZEND_DEBUG || const char *filename = op_array->filename ? op_array->filename->val : NULL; +| mov FCARG1a, size | LOAD_ADDR FCARG2a, filename |.if X64 | mov CARG3d, opline->lineno @@ -1133,8 +1133,15 @@ static void* dasm_labels[zend_lb_MAX]; | push 0 | push opline->lineno |.endif +| EXT_CALL _emalloc, r0 +||#else +|| if (size == 24) { +| EXT_CALL _emalloc_24, r0 +|| } else { +| mov FCARG1a, size +| EXT_CALL _emalloc, r0 || } -| EXT_CALL _emalloc, r0 +||#endif |.endmacro |.macro OBJ_RELEASE, reg, exit_label From 17b7e77adbeb83e6dacb3c3fc26c3094bb451119 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 13 Oct 2016 02:28:37 +0300 Subject: [PATCH 296/569] Moved exceptional code into stubs --- ext/opcache/jit/zend_jit_x86.dasc | 209 ++++++++++++++++++++++-------- 1 file changed, 152 insertions(+), 57 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index a6bd00e4eea43..0d1e2588dd301 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -1402,6 +1402,146 @@ static int zend_jit_throw_cannot_pass_by_ref_stub(dasm_State **Dst) return 1; } +static int zend_jit_undefined_offset_stub(dasm_State **Dst) +{ + |->undefined_offset: + |.if X64 + | sub r4, 8 + |.else + | sub r4, 12 + |.endif + | mov r0, EX->opline + |.if X64 + | movsxd r1, dword OP:r0->result.var + |.else + | mov r1, OP:r0->result.var + |.endif + | cmp byte OP:r0->op2_type, IS_CONST + | SET_Z_TYPE_INFO FP + r1, IS_NULL + | jne >2 + |.if X64 + | movsxd r0, dword OP:r0->op2.constant + | add r0, aword EX->literals + |.else + | mov r0, aword OP:r0->op2.zv + |.endif + | jmp >3 + |2: + |.if X64 + | movsxd r0, dword OP:r0->op2.var + |.else + | mov r0, OP:r0->op2.var + |.endif + | add r0, FP + |3: + |.if X64 + | mov CARG1, E_NOTICE + | LOAD_ADDR CARG2, "Undefined offset: " ZEND_LONG_FMT + | mov CARG3, aword [r0] + | EXT_CALL zend_error, r0 + | add r4, 8 // stack alignment + |.else + | sub r4, 4 + | push aword [r0] + | push "Undefined offset: " ZEND_LONG_FMT + | push E_NOTICE + | EXT_CALL zend_error, r0 + | add r4, 28 + |.endif + | ret + + return 1; +} + +static int zend_jit_undefined_index_stub(dasm_State **Dst) +{ + |->undefined_index: + |.if X64 + | sub r4, 8 + |.else + | sub r4, 12 + |.endif + | mov r0, EX->opline + |.if X64 + | movsxd r1, dword OP:r0->result.var + |.else + | mov r1, OP:r0->result.var + |.endif + | cmp byte OP:r0->op2_type, IS_CONST + | SET_Z_TYPE_INFO FP + r1, IS_NULL + | jne >2 + |.if X64 + | movsxd r0, dword OP:r0->op2.constant + | add r0, aword EX->literals + |.else + | mov r0, aword OP:r0->op2.zv + |.endif + | jmp >3 + |2: + |.if X64 + | movsxd r0, dword OP:r0->op2.var + |.else + | mov r0, OP:r0->op2.var + |.endif + | add r0, FP + |3: + |.if X64 + | mov CARG1, E_NOTICE + | LOAD_ADDR CARG2, "Undefined index: %s" + | mov CARG3, aword [r0] + | add CARG3, offsetof(zend_string, val) + | EXT_CALL zend_error, r0 + | add r4, 8 + |.else + | sub r4, 4 + | mov r0, aword [r0] + | add r0, offsetof(zend_string, val) + | push r0 + | push "Undefined index: %s" + | push E_NOTICE + | EXT_CALL zend_error, r0 + | add r4, 28 + |.endif + | ret + + return 1; +} + +static int zend_jit_cannot_add_element_stub(dasm_State **Dst) +{ + |->cannot_add_element: + |.if X64 + | sub r4, 8 + |.else + | sub r4, 12 + |.endif + | mov r0, EX->opline + | cmp byte OP:r0->result_type, IS_UNUSED + | jz >1 + |.if X64 + | movsxd r0, dword OP:r0->result.var + |.else + | mov r0, OP:r0->result.var + |.endif + | SET_Z_TYPE_INFO FP + r0, IS_NULL + |1: + |.if X64 + | mov CARG1, E_WARNING + | LOAD_ADDR CARG2, "Cannot add element to the array as the next element is already occupied" + | EXT_CALL zend_error, r0 + | add r4, 8 + |.else + | sub r4, 8 + | push "Cannot add element to the array as the next element is already occupied" + | push E_WARNING + | EXT_CALL zend_error, r0 + | add r4, 28 + |.endif + | ret + + return 1; +} + static const zend_jit_stub zend_jit_stubs[] = { JIT_STUB(interrupt_handler), JIT_STUB(exception_handler), @@ -1409,6 +1549,9 @@ static const zend_jit_stub zend_jit_stubs[] = { JIT_STUB(leave_throw), JIT_STUB(icall_throw), JIT_STUB(throw_cannot_pass_by_ref), + JIT_STUB(undefined_offset), + JIT_STUB(undefined_index), + JIT_STUB(cannot_add_element), }; static int zend_jit_align_func(dasm_State **Dst) @@ -2621,21 +2764,10 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o case BP_VAR_R: | SAVE_VALID_OPLINE opline | // zend_error(E_NOTICE,"Undefined offset: " ZEND_LONG_FMT, hval); - |.if X64 - | mov CARG1, E_NOTICE - | LOAD_ADDR CARG2, "Undefined offset: " ZEND_LONG_FMT - | LONG_LOAD CARG3, opline->op2_type, opline->op2 - | EXT_CALL zend_error, r0 - |.else - | sub r4, 4 - | LONG_LOAD r0, opline->op2_type, opline->op2 - | push r0 - | push "Undefined offset: " ZEND_LONG_FMT - | push E_NOTICE - | EXT_CALL zend_error, r0 - | add r4, 16 - |.endif - /* break missing intentionally */ + | // retval = &EG(uninitialized_zval); + | call ->undefined_offset + | jmp >9 + break; case BP_VAR_IS: case BP_VAR_UNSET: | // retval = &EG(uninitialized_zval); @@ -2739,23 +2871,9 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o case BP_VAR_R: // zend_error(E_NOTICE, "Undefined index: %s", ZSTR_VAL(offset_key)); | SAVE_VALID_OPLINE opline - |.if X64 - | mov CARG1, E_NOTICE - | LOAD_ADDR CARG2, "Undefined index: %s" - | LONG_LOAD CARG3, opline->op2_type, opline->op2 - | add CARG3, offsetof(zend_string, val) - | EXT_CALL zend_error, r0 - |.else - | sub r4, 4 - | LONG_LOAD r0, opline->op2_type, opline->op2 - | add r0, offsetof(zend_string, val) - | push r0 - | push "Undefined index: %s" - | push E_NOTICE - | EXT_CALL zend_error, r0 - | add r4, 16 - |.endif - /* break missing intentionally */ + | call ->undefined_index + | jmp >9 + break; case BP_VAR_IS: case BP_VAR_UNSET: | // retval = &EG(uninitialized_zval); @@ -3173,20 +3291,7 @@ static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, zend_op_ |1: | // zend_error(E_WARNING, "Cannot add element to the array as the next element is already occupied"); | SAVE_VALID_OPLINE opline - |.if X64 - | mov CARG1, E_WARNING - | LOAD_ADDR CARG2, "Cannot add element to the array as the next element is already occupied" - | EXT_CALL zend_error, r0 - |.else - | sub r4, 8 - | push "Cannot add element to the array as the next element is already occupied" - | push E_WARNING - | EXT_CALL zend_error, r0 - | add r4, 16 - |.endif - if (opline->result_type != IS_UNUSED) { - | SET_Z_TYPE_INFO FP + opline->result.var, IS_NULL - } + | call ->cannot_add_element | //ZEND_VM_C_GOTO(assign_dim_op_ret_null); | jmp >9 |.code @@ -3467,17 +3572,7 @@ static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, zend_ |1: | // zend_error(E_WARNING, "Cannot add element to the array as the next element is already occupied"); | SAVE_VALID_OPLINE opline - |.if X64 - | mov CARG1, E_WARNING - | LOAD_ADDR CARG2, "Cannot add element to the array as the next element is already occupied" - | EXT_CALL zend_error, r0 - |.else - | sub r4, 8 - | push "Cannot add element to the array as the next element is already occupied" - | push E_WARNING - | EXT_CALL zend_error, r0 - | add r4, 16 - |.endif + | call ->cannot_add_element | //ZEND_VM_C_GOTO(assign_dim_op_ret_null); | jmp >9 |.code From 3e191622c059403496660abd65d4902abbb6ee35 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 13 Oct 2016 02:51:24 +0300 Subject: [PATCH 297/569] Reduce amount of generated "cold" code --- ext/opcache/jit/zend_jit_x86.dasc | 69 +++++++++++++++++++++++++++---- 1 file changed, 61 insertions(+), 8 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 0d1e2588dd301..8436fa27087c3 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -1183,6 +1183,33 @@ static void* dasm_labels[zend_lb_MAX]; || } |.endmacro +|.macro UNDEFINED_OFFSET, opline +|| if (opline == last_valid_opline) { +| call ->undefined_offset_ex +|| } else { +| SAVE_VALID_OPLINE, opline +| call ->undefined_offset +|| } +|.endmacro + +|.macro UNDEFINED_INDEX, opline +|| if (opline == last_valid_opline) { +| call ->undefined_index_ex +|| } else { +| SAVE_VALID_OPLINE, opline +| call ->undefined_index +|| } +|.endmacro + +|.macro CANNOT_ADD_ELEMENT, opline +|| if (opline == last_valid_opline) { +| call ->cannot_add_element_ex +|| } else { +| SAVE_VALID_OPLINE, opline +| call ->cannot_add_element +|| } +|.endmacro + static zend_bool reuse_ip; static zend_bool delayed_call_chain; static uint32_t delayed_call_level; @@ -1402,6 +1429,15 @@ static int zend_jit_throw_cannot_pass_by_ref_stub(dasm_State **Dst) return 1; } +static int zend_jit_undefined_offset_ex_stub(dasm_State **Dst) +{ + |->undefined_offset_ex: + | mov aword EX->opline, IP + | jmp ->undefined_offset + + return 1; +} + static int zend_jit_undefined_offset_stub(dasm_State **Dst) { |->undefined_offset: @@ -1453,6 +1489,15 @@ static int zend_jit_undefined_offset_stub(dasm_State **Dst) return 1; } +static int zend_jit_undefined_index_ex_stub(dasm_State **Dst) +{ + |->undefined_index_ex: + | mov aword EX->opline, IP + | jmp ->undefined_index + + return 1; +} + static int zend_jit_undefined_index_stub(dasm_State **Dst) { |->undefined_index: @@ -1507,6 +1552,15 @@ static int zend_jit_undefined_index_stub(dasm_State **Dst) return 1; } +static int zend_jit_cannot_add_element_ex_stub(dasm_State **Dst) +{ + |->cannot_add_element_ex: + | mov aword EX->opline, IP + | jmp ->cannot_add_element + + return 1; +} + static int zend_jit_cannot_add_element_stub(dasm_State **Dst) { |->cannot_add_element: @@ -1552,6 +1606,9 @@ static const zend_jit_stub zend_jit_stubs[] = { JIT_STUB(undefined_offset), JIT_STUB(undefined_index), JIT_STUB(cannot_add_element), + JIT_STUB(undefined_offset_ex), + JIT_STUB(undefined_index_ex), + JIT_STUB(cannot_add_element_ex), }; static int zend_jit_align_func(dasm_State **Dst) @@ -2762,10 +2819,9 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o |2: switch (type) { case BP_VAR_R: - | SAVE_VALID_OPLINE opline | // zend_error(E_NOTICE,"Undefined offset: " ZEND_LONG_FMT, hval); | // retval = &EG(uninitialized_zval); - | call ->undefined_offset + | UNDEFINED_OFFSET opline | jmp >9 break; case BP_VAR_IS: @@ -2870,8 +2926,7 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o switch (type) { case BP_VAR_R: // zend_error(E_NOTICE, "Undefined index: %s", ZSTR_VAL(offset_key)); - | SAVE_VALID_OPLINE opline - | call ->undefined_index + | UNDEFINED_INDEX opline | jmp >9 break; case BP_VAR_IS: @@ -3290,8 +3345,7 @@ static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, zend_op_ |.cold_code |1: | // zend_error(E_WARNING, "Cannot add element to the array as the next element is already occupied"); - | SAVE_VALID_OPLINE opline - | call ->cannot_add_element + | CANNOT_ADD_ELEMENT opline | //ZEND_VM_C_GOTO(assign_dim_op_ret_null); | jmp >9 |.code @@ -3571,8 +3625,7 @@ static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, zend_ |.cold_code |1: | // zend_error(E_WARNING, "Cannot add element to the array as the next element is already occupied"); - | SAVE_VALID_OPLINE opline - | call ->cannot_add_element + | CANNOT_ADD_ELEMENT opline | //ZEND_VM_C_GOTO(assign_dim_op_ret_null); | jmp >9 |.code From 44c43094d1bbd4d4773086414db958a820ae9d20 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 13 Oct 2016 12:20:57 +0300 Subject: [PATCH 298/569] Dirty PoC that allows identifying and JIT-ing only the hotest functions --- ext/opcache/jit/zend_jit.c | 39 ++++++++++++++++++++++++++++ ext/opcache/jit/zend_jit_vm_helper.c | 29 +++++++++++++++++++++ ext/opcache/jit/zend_jit_vm_helper.h | 2 ++ ext/opcache/jit/zend_jit_x86.dasc | 15 +++++++++++ 4 files changed, 85 insertions(+) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 1fbe26be7ba46..e72f95b7ed210 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -35,6 +35,8 @@ //#define CONTEXT_THREADED_JIT #define PREFER_MAP_32BIT //#define ZEND_RUNTIME_JIT +//#define ZEND_JIT_RECORD +//#define ZEND_JIT_FILTER #define ZEND_JIT_USE_RC_INFERENCE #ifdef ZEND_JIT_USE_RC_INFERENCE @@ -901,6 +903,35 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) void *handler; int call_level = 0; +#ifdef ZEND_JIT_FILTER + const char *names[] = { +#include "zend_jit_filter.c" + }; + + for (i = 0; i < sizeof(names)/sizeof(names[0]); i++) { + const char *name = names[i]; + const char *sep = strstr(name, "::"); + + if (sep) { + if (op_array->scope && + op_array->scope->name && + op_array->function_name && + strncmp(ZSTR_VAL(op_array->scope->name), name, sep-name) == 0 && + strcmp(ZSTR_VAL(op_array->function_name), sep+2) == 0) { + goto pass; + } + } else { + if (op_array->scope == NULL && + op_array->function_name && + strcmp(ZSTR_VAL(op_array->function_name), name) == 0) { + goto pass; + } + } + } + return SUCCESS; +pass: +#endif + /* mark hidden branch targets */ for (b = 0; b < ssa->cfg.blocks_count; b++) { if (ssa->cfg.blocks[b].flags & ZEND_BB_REACHABLE && @@ -961,6 +992,11 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) zend_jit_label(&dasm_state, ssa->cfg.blocks_count + b); zend_jit_prologue(&dasm_state); } +#ifdef ZEND_JIT_RECORD + if (opline->opcode != ZEND_RECV && opline->opcode != ZEND_RECV_INIT) { + zend_jit_func_header(&dasm_state, op_array); + } +#endif } zend_jit_label(&dasm_state, b); @@ -974,6 +1010,9 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) } } if (ssa->cfg.blocks[b].flags & ZEND_BB_LOOP_HEADER) { +#ifdef ZEND_JIT_RECORD + zend_jit_loop_header(&dasm_state, op_array, op_array->opcodes + ssa->cfg.blocks[b].start); +#endif if (!zend_jit_check_timeout(&dasm_state, op_array->opcodes + ssa->cfg.blocks[b].start)) { goto jit_failure; } diff --git a/ext/opcache/jit/zend_jit_vm_helper.c b/ext/opcache/jit/zend_jit_vm_helper.c index bd1411c8e2eb9..ae650124fa1ac 100644 --- a/ext/opcache/jit/zend_jit_vm_helper.c +++ b/ext/opcache/jit/zend_jit_vm_helper.c @@ -83,6 +83,35 @@ void ZEND_FASTCALL zend_jit_leave_top_func_helper(uint32_t call_info) opline = NULL; } +/* The recorded log may be postprocessed to identify the hot functions and + * loops. + * + * To get the top functions: + * sed 's/^\(.*\), (.:\(.*\):.*$/\1,\2/' | sort | uniq -c | sort -nr | \ + * head -n25 | sed 's/^\s*[0-9]*\s*\(.*\),.*$/"\1",/' > zend_jit_filter.c + * + */ + +void ZEND_FASTCALL zend_jit_func_header_helper(void) +{ + fprintf(stderr, "%s%s%s, (F:%s:%d)\n", + EX(func)->op_array.scope ? ZSTR_VAL(EX(func)->op_array.scope->name) : "", + EX(func)->op_array.scope ? "::" : "", + EX(func)->op_array.function_name ? ZSTR_VAL(EX(func)->op_array.function_name) : "", + ZSTR_VAL(EX(func)->op_array.filename), + EX(func)->op_array.line_start); +} + +void ZEND_FASTCALL zend_jit_loop_header_helper(void) +{ + fprintf(stderr, "%s%s%s, (L:%s:%d)\n", + EX(func)->op_array.scope ? ZSTR_VAL(EX(func)->op_array.scope->name) : "", + EX(func)->op_array.scope ? "::" : "", + EX(func)->op_array.function_name ? ZSTR_VAL(EX(func)->op_array.function_name) : "", + ZSTR_VAL(EX(func)->op_array.filename), + opline->lineno); +} + /* * Local variables: * tab-width: 4 diff --git a/ext/opcache/jit/zend_jit_vm_helper.h b/ext/opcache/jit/zend_jit_vm_helper.h index 5eedf423b0650..81554008cf04c 100644 --- a/ext/opcache/jit/zend_jit_vm_helper.h +++ b/ext/opcache/jit/zend_jit_vm_helper.h @@ -22,6 +22,8 @@ void ZEND_FASTCALL zend_jit_leave_nested_func_helper(uint32_t call_info); void ZEND_FASTCALL zend_jit_leave_top_func_helper(uint32_t call_info); +void ZEND_FASTCALL zend_jit_func_header_helper(void); +void ZEND_FASTCALL zend_jit_loop_header_helper(void); #endif diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 8436fa27087c3..433c51158971f 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -6641,6 +6641,21 @@ fallback: return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); } +static int zend_jit_func_header(dasm_State **Dst, zend_op_array *op_array) +{ + | EXT_CALL zend_jit_func_header_helper, r0 + return 1; +} + +static int zend_jit_loop_header(dasm_State **Dst, zend_op_array *op_array, zend_op *opline) +{ + if (!zend_jit_set_valid_ip(Dst, opline)) { + return 0; + } + | EXT_CALL zend_jit_loop_header_helper, r0 + return 1; +} + /* * Local variables: * tab-width: 4 From 795e895b6b3f2631073380ce5922fbd56b3f2492 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 13 Oct 2016 16:02:45 +0300 Subject: [PATCH 299/569] Implemented JIT for INIT_FCALL_BY_NAME --- ext/opcache/jit/zend_jit.c | 1 + ext/opcache/jit/zend_jit_disasm_x86.c | 1 + ext/opcache/jit/zend_jit_helpers.c | 17 ++++++ ext/opcache/jit/zend_jit_x86.dasc | 88 +++++++++++++++++++-------- 4 files changed, 82 insertions(+), 25 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index e72f95b7ed210..45def2fcf99cf 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -1090,6 +1090,7 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) } goto done; case ZEND_INIT_FCALL: + case ZEND_INIT_FCALL_BY_NAME: if (!zend_jit_init_fcall(&dasm_state, opline, b, op_array, ssa, call_level)) { goto jit_failure; } diff --git a/ext/opcache/jit/zend_jit_disasm_x86.c b/ext/opcache/jit/zend_jit_disasm_x86.c index a38b5d818131f..3fe47c694cdfa 100644 --- a/ext/opcache/jit/zend_jit_disasm_x86.c +++ b/ext/opcache/jit/zend_jit_disasm_x86.c @@ -384,6 +384,7 @@ static int zend_jit_disasm_init(void) (uint64_t)(uintptr_t)n, sizeof(void*)); REGISTER_HELPER(memcmp); REGISTER_HELPER(zend_jit_find_func_helper); + REGISTER_HELPER(zend_jit_find_func_by_name_helper); REGISTER_HELPER(zend_jit_extend_stack_helper); REGISTER_HELPER(zend_jit_int_extend_stack_helper); REGISTER_HELPER(zend_jit_leave_nested_func_helper); diff --git a/ext/opcache/jit/zend_jit_helpers.c b/ext/opcache/jit/zend_jit_helpers.c index d06b727db4594..8f27db9506cd3 100644 --- a/ext/opcache/jit/zend_jit_helpers.c +++ b/ext/opcache/jit/zend_jit_helpers.c @@ -35,6 +35,23 @@ static zend_function* ZEND_FASTCALL zend_jit_find_func_helper(zend_string *name) return fbc; } +static zend_function* ZEND_FASTCALL zend_jit_find_func_by_name_helper(zend_string *name, zend_string *key) +{ + zval *func = zend_hash_find(EG(function_table), key); + zend_function *fbc; + + if (UNEXPECTED(func == NULL)) { + zend_throw_error(NULL, "Call to undefined function %s()", ZSTR_VAL(name)); + return NULL; + } + fbc = Z_FUNC_P(func); + if (EXPECTED(fbc->type == ZEND_USER_FUNCTION) && UNEXPECTED(!fbc->op_array.run_time_cache)) { + fbc->op_array.run_time_cache = zend_arena_alloc(&CG(arena), fbc->op_array.cache_size); + memset(fbc->op_array.run_time_cache, 0, fbc->op_array.cache_size); + } + return fbc; +} + static zend_execute_data* ZEND_FASTCALL zend_jit_extend_stack_helper(uint32_t used_stack, zend_function *fbc) { zend_execute_data *call = (zend_execute_data*)zend_vm_stack_extend(used_stack); diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 433c51158971f..86c6228dbf358 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -4711,11 +4711,30 @@ fallback: static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_function *func) { - if (!func) { - assert(0); - } + uint32_t used_stack; - uint32_t used_stack = zend_vm_calc_used_stack(opline->extended_value, func); + if (func) { + used_stack = zend_vm_calc_used_stack(opline->extended_value, func); + } else { + used_stack = (ZEND_CALL_FRAME_SLOT + opline->extended_value) * sizeof(zval); + + | // if (EXPECTED(ZEND_USER_CODE(func->type))) { + | test byte [r0 + offsetof(zend_function, type)], 1 + | mov FCARG1a, used_stack + | jnz >1 + | // used_stack += (func->op_array.last_var + func->op_array.T - MIN(func->op_array.num_args, num_args)) * sizeof(zval); + | mov edx, opline->extended_value + | cmp edx, dword [r0 + offsetof(zend_function, op_array.num_args)] + | cmova edx, dword [r0 + offsetof(zend_function, op_array.num_args)] + | sub edx, dword [r0 + offsetof(zend_function, op_array.last_var)] + | sub edx, dword [r0 + offsetof(zend_function, op_array.T)] + | shl edx, 5 + |.if X64 + | movsxd r2, edx + |.endif + | sub FCARG1a, r2 + |1: + } zend_jit_start_reuse_ip(); @@ -4723,10 +4742,18 @@ static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, zen | mov RX, aword [&EG(vm_stack_top)] | mov r2, aword [&EG(vm_stack_end)] | sub r2, RX - | cmp r2, used_stack + if (func) { + | cmp r2, used_stack + } else { + | cmp r2, FCARG1a + } | jb >2 | // EG(vm_stack_top) = (zval*)((char*)call + used_stack); - | add aword [&EG(vm_stack_top)], used_stack + if (func) { + | add aword [&EG(vm_stack_top)], used_stack + } else { + | add aword [&EG(vm_stack_top)], FCARG1a + } | // zend_vm_init_call_frame(call, call_info, func, num_args, called_scope, object); | // ZEND_SET_CALL_INFO(call, 0, call_info); | mov dword EX:RX->This.u1.type_info, (IS_UNDEF | (ZEND_CALL_NESTED_FUNCTION << ZEND_CALL_INFO_SHIFT)) @@ -4753,7 +4780,9 @@ static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, zen | mov dword EX:RX->This.u2.num_args, opline->extended_value |.cold_code |2: - | mov FCARG1d, used_stack + if (func) { + | mov FCARG1d, used_stack + } if (func && func->type == ZEND_INTERNAL_FUNCTION) { | EXT_CALL zend_jit_int_extend_stack_helper, r0 } else { @@ -4769,10 +4798,16 @@ static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, zen static int zend_jit_needs_call_chain(zend_call_info *call_info, uint32_t b, zend_op_array *op_array, zend_ssa *ssa) { - const zend_op *opline = call_info->caller_init_opline; - const zend_op *end = call_info->caller_call_opline; + const zend_op *opline, *end; int skip; + if (!call_info) { + return 1; + } + + opline = call_info->caller_init_opline; + end = call_info->caller_call_opline; + if (end - op_array->opcodes >= ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len) { /* INIT_FCALL and DO_FCALL in different BasicBlocks */ return 1; @@ -4811,7 +4846,7 @@ static int zend_jit_needs_call_chain(zend_call_info *call_info, uint32_t b, zend static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, uint32_t b, zend_op_array *op_array, zend_ssa *ssa, int call_level) { zend_func_info *info = ZEND_FUNC_INFO(op_array); - zend_call_info *call_info = NULL; + zend_call_info *call_info = NULL; zend_function *func = NULL; if (delayed_call_chain) { @@ -4830,11 +4865,6 @@ static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, uint32_t } } - if (!func) { - // TODO: implement JIT for INIT_FCALL for unknown functions ??? - goto fallback; - } - if (func && func->type == ZEND_INTERNAL_FUNCTION) { /* load constant address later */ } else if (func && op_array == &func->op_array) { @@ -4852,11 +4882,20 @@ static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, uint32_t |1: | // SAVE_OPLINE(); | SAVE_VALID_OPLINE opline - | LOAD_ADDR FCARG1a, Z_STR_P(zv); - | EXT_CALL zend_jit_find_func_helper, r0 - | // This opcode must not throw exception ??? - | // test r0, r0 - | // jz ->exception_handler */ + if (opline->opcode == ZEND_INIT_FCALL) { + | LOAD_ADDR FCARG1a, Z_STR_P(zv); + | EXT_CALL zend_jit_find_func_helper, r0 + } else if (opline->opcode == ZEND_INIT_FCALL_BY_NAME) { + | LOAD_ADDR FCARG1a, Z_STR_P(zv); + | LOAD_ADDR FCARG2a, Z_STR_P(zv + 1); + | EXT_CALL zend_jit_find_func_by_name_helper, r0 + } else { + ZEND_ASSERT(0); + } + if (opline->opcode != ZEND_INIT_FCALL) { + | test r0, r0 + | jz ->exception_handler + } | // CACHE_PTR(Z_CACHE_SLOT_P(fname), fbc); | mov r1, EX->run_time_cache | mov aword [r1 + Z_CACHE_SLOT_P(zv)], r0 @@ -4864,11 +4903,12 @@ static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, uint32_t |.code |3: } + if (!zend_jit_push_call_frame(Dst, opline, op_array, func)) { return 0; } - if (!func || zend_jit_needs_call_chain(call_info, b, op_array, ssa)) { + if (zend_jit_needs_call_chain(call_info, b, op_array, ssa)) { if (!zend_jit_save_call_chain(Dst, call_level)) { return 0; } @@ -4878,10 +4918,6 @@ static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, uint32_t } return 1; - -fallback: - /* fallback to subroutine threading */ - return zend_jit_handler(Dst, opline, 0); } static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, int call_level) @@ -6641,6 +6677,7 @@ fallback: return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); } +#ifdef ZEND_JIT_RECORD static int zend_jit_func_header(dasm_State **Dst, zend_op_array *op_array) { | EXT_CALL zend_jit_func_header_helper, r0 @@ -6655,6 +6692,7 @@ static int zend_jit_loop_header(dasm_State **Dst, zend_op_array *op_array, zend_ | EXT_CALL zend_jit_loop_header_helper, r0 return 1; } +#endif /* * Local variables: From f48fae17cf7d0955a05d0fab9eea1a9eb5dbf6bb Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 18 Oct 2016 09:45:58 +0300 Subject: [PATCH 300/569] Saved instruction --- ext/opcache/jit/zend_jit_x86.dasc | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 86c6228dbf358..c0e00232f7e24 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -5104,10 +5104,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar uint32_t func_info = zend_get_func_info(call_info, ssa); if (func_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { - | lea r0, aword [r4 + 8] - } - if (func_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { - | ZVAL_PTR_DTOR r0, func_info, 1, 1, 0, op_array->filename, opline->lineno + | ZVAL_PTR_DTOR r4 + 8, func_info, 1, 1, 0, op_array->filename, opline->lineno } | add r4, 16 /* revert alloca() */ } From b3882e416de1d73f679fbf268000c1c76fc76ab1 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 18 Oct 2016 12:35:57 +0300 Subject: [PATCH 301/569] END_SILENCE doesn't destroy operand --- ext/opcache/jit/zend_jit.c | 1 + 1 file changed, 1 insertion(+) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 45def2fcf99cf..e0639720693d8 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -386,6 +386,7 @@ static int zend_may_throw(const zend_op *opline, zend_op_array *op_array, zend_s case ZEND_SEND_VAR_NO_REF_EX: case ZEND_SEND_REF: case ZEND_SEPARATE: + case ZEND_END_SILENCE: break; default: /* destructor may be called */ From e2fd107570dc56d66a709fad7137075ca62926d0 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 18 Oct 2016 12:37:11 +0300 Subject: [PATCH 302/569] Prevent double copying --- ext/opcache/jit/zend_jit_x86.dasc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index c0e00232f7e24..afc7a06c52ed3 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -5234,12 +5234,12 @@ static int zend_jit_send_ref(dasm_State **Dst, const zend_op *opline, zend_op_ar if (op1_info & MAY_BE_UNDEF) { if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { | IF_NOT_Z_TYPE FP + opline->op1.var, IS_UNDEF, >1 - } - | SET_Z_TYPE_INFO FP + opline->op1.var, IS_NULL - if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { + | SET_Z_TYPE_INFO FP + opline->op1.var, IS_NULL | jmp >2 |1: } + op1_info &= ~MAY_BE_UNDEF; + op1_info |= MAY_BE_NULL; } } else { ZEND_ASSERT(0); From fd7f61aa1b6da76ab76381faec370e76776920c7 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 18 Oct 2016 18:00:17 +0300 Subject: [PATCH 303/569] cleanup --- ext/opcache/jit/zend_jit_x86.dasc | 37 +++++++------------------------ 1 file changed, 8 insertions(+), 29 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index afc7a06c52ed3..bfc27d65d8439 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -5922,36 +5922,15 @@ static int zend_jit_return(dasm_State **Dst, const zend_op *opline, zend_op_arra | ZVAL_COPY_VALUE r1, FP + opline->op1.var, op1_info, r0, eax, r2 } else if (opline->op1_type == IS_CV) { if (op1_info & MAY_BE_REF) { - | IF_Z_TYPE FP + opline->op1.var, IS_REFERENCE, >1 - |.cold_code - |1: - | // retval_ptr = Z_REFVAL_P(retval_ptr); - | GET_Z_PTR r0, FP + opline->op1.var - | // ZVAL_COPY(return_value, retval_ptr); - | GET_Z_TYPE_INFO edx, r0 + offsetof(zend_reference, val) - | SET_Z_TYPE_INFO r1, edx - | IF_REFCOUNTED dh, >2 - |.if X64 - | GET_Z_PTR r2, r0 + offsetof(zend_reference, val) - | SET_Z_PTR r1, r2 - |.else - | GET_Z_PTR r2, r0 + offsetof(zend_reference, val) - | SET_Z_PTR r1, r2 - | GET_Z_W2 r2, r0 + offsetof(zend_reference, val) - | SET_Z_W2 r1, r2 - |.endif - | jmp >3 - |2: - | GET_Z_PTR r2, r0 + offsetof(zend_reference, val) - | SET_Z_PTR r1, r2 - | GC_ADDREF r2 - | jmp >3 - |.code + | lea r0, [FP + opline->op1.var] + | ZVAL_DEREF r0, op1_info + | ZVAL_COPY_VALUE_clobber_src r1, r0, op1_info, r0, eax, r2 + | TRY_ADDREF op1_info, ah, r2 + } else { + | ZVAL_COPY_VALUE r1, FP + opline->op1.var, op1_info, r0, eax, r2 + | // TODO: JIT: if (EXPECTED(!(EX_CALL_INFO() & ZEND_CALL_CODE))) ZVAL_NULL(retval_ptr); ??? + | TRY_ADDREF op1_info, ah, r2 } - | ZVAL_COPY_VALUE r1, FP + opline->op1.var, op1_info, r0, eax, r2 - | // TODO: JIT: if (EXPECTED(!(EX_CALL_INFO() & ZEND_CALL_CODE))) ZVAL_NULL(retval_ptr); ??? - | TRY_ADDREF op1_info, ah, r2 - |3: } else { if (op1_info & MAY_BE_REF) { | IF_Z_TYPE FP + opline->op1.var, IS_REFERENCE, >1 From 6a9a26ef1c389261af23ab1bb9a7fd00b1dc36fd Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 20 Oct 2016 10:52:00 +0300 Subject: [PATCH 304/569] Keep stack overflow detection and stack extension code together --- ext/opcache/jit/zend_jit_x86.dasc | 33 ++++++++++++++++--------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index bfc27d65d8439..21ad9e232a078 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -4740,6 +4740,7 @@ static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, zen | // if (UNEXPECTED(used_stack > (size_t)(((char*)EG(vm_stack_end)) - (char*)call))) { | mov RX, aword [&EG(vm_stack_top)] + | // Check Stack Overflow | mov r2, aword [&EG(vm_stack_end)] | sub r2, RX if (func) { @@ -4747,8 +4748,23 @@ static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, zen } else { | cmp r2, FCARG1a } - | jb >2 + | jb >1 | // EG(vm_stack_top) = (zval*)((char*)call + used_stack); + |.cold_code + |1: + if (func) { + | mov FCARG1d, used_stack + } + if (func && func->type == ZEND_INTERNAL_FUNCTION) { + | EXT_CALL zend_jit_int_extend_stack_helper, r0 + } else { + | mov FCARG2a, r0 + | EXT_CALL zend_jit_extend_stack_helper, r0 + } + | mov RX, r0 + | jmp >1 + |.code + if (func) { | add aword [&EG(vm_stack_top)], used_stack } else { @@ -4778,21 +4794,6 @@ static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, zen | mov aword EX:RX->This.value.ptr, 0 | // ZEND_CALL_NUM_ARGS(call) = num_args; | mov dword EX:RX->This.u2.num_args, opline->extended_value - |.cold_code - |2: - if (func) { - | mov FCARG1d, used_stack - } - if (func && func->type == ZEND_INTERNAL_FUNCTION) { - | EXT_CALL zend_jit_int_extend_stack_helper, r0 - } else { - | mov FCARG2a, r0 - | EXT_CALL zend_jit_extend_stack_helper, r0 - } - | mov RX, r0 - | jmp <1 - |.code - return 1; } From 0edfe19bbc09d936eab70cbfd609abc76d2d7fc2 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 20 Oct 2016 20:31:39 +0300 Subject: [PATCH 305/569] Show EG(symbol_table) in disassemble --- ext/opcache/jit/zend_jit_disasm_x86.c | 1 + 1 file changed, 1 insertion(+) diff --git a/ext/opcache/jit/zend_jit_disasm_x86.c b/ext/opcache/jit/zend_jit_disasm_x86.c index 3fe47c694cdfa..c08349dd57b7f 100644 --- a/ext/opcache/jit/zend_jit_disasm_x86.c +++ b/ext/opcache/jit/zend_jit_disasm_x86.c @@ -370,6 +370,7 @@ static int zend_jit_disasm_init(void) REGISTER_EG(current_execute_data); REGISTER_EG(vm_stack_top); REGISTER_EG(vm_stack_end); + REGISTER_EG(symbol_table); #undef REGISTER_EG #define REGISTER_CG(n) \ zend_jit_disasm_add_symbol("CG("#n")", \ From 031a16947a15c72ada5536633014cbfaafa38d63 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 21 Oct 2016 17:51:53 +0300 Subject: [PATCH 306/569] Support for GC_COLLECTABE --- ext/opcache/jit/zend_jit_x86.dasc | 53 ++++++++++++------------------- 1 file changed, 21 insertions(+), 32 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 21ad9e232a078..5c7bf8d144d6a 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -791,16 +791,13 @@ static void* dasm_labels[zend_lb_MAX]; | dec dword [zv] |.endmacro -|.macro IF_GC_INFO, ptr, label -| cmp word [ptr + 6], 0 +|.macro IF_GC_MAY_NOT_LEAK, ptr, tmp_reg, label +| mov tmp_reg, dword [ptr + 4] +| and tmp_reg, (GC_INFO_MASK | (GC_COLLECTABLE << GC_FLAGS_SHIFT)) +| cmp tmp_reg, (GC_COLLECTABLE << GC_FLAGS_SHIFT) | jne label |.endmacro -|.macro IF_NOT_GC_INFO, ptr, label -| cmp word [ptr + 6], 0 -| je label -|.endmacro - |.macro ADDREF_CONST, zv, tmp_reg |.if X64 || if (!IS_SIGNED_32BIT(Z_LVAL_P(zv))) { @@ -943,21 +940,15 @@ static void* dasm_labels[zend_lb_MAX]; || } || if (gc && RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT))) { || if ((op_info) & MAY_BE_REF) { -| lea r1, [zv] -| // ZVAL_DEREF(z); -| ZVAL_DEREF r1, (op_info) -| // if (Z_COLLECTABLE_P(z)) -| IF_NOT_Z_FLAGS r1, IS_TYPE_COLLECTABLE, >2 -| // if (UNEXPECTED(!Z_GC_INFO_P(z))) -| GET_Z_PTR FCARG1a, r1 -| IF_GC_INFO FCARG1a, >2 -|| } else { -| // if (Z_COLLECTABLE_P(z)) -| IF_NOT_Z_FLAGS zv, IS_TYPE_COLLECTABLE, >2 -| // if (UNEXPECTED(!Z_GC_INFO_P(z))) | GET_Z_PTR FCARG1a, zv -| IF_GC_INFO FCARG1a, >2 +| cmp byte [FCARG1a + 4], IS_REFERENCE +| jnz >1 +| GET_Z_PTR FCARG1a, FCARG1a + offsetof(zend_reference, val) +| IF_NOT_Z_REFCOUNTED zv, >2 +| GET_Z_PTR FCARG1a, zv +|1: || } +| IF_GC_MAY_NOT_LEAK FCARG1a, eax, >2 | // gc_possible_root(Z_COUNTED_P(z)) | EXT_CALL gc_possible_root, r0 || if (cold && ((op_info) & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE)))) { @@ -1144,7 +1135,7 @@ static void* dasm_labels[zend_lb_MAX]; ||#endif |.endmacro -|.macro OBJ_RELEASE, reg, exit_label +|.macro OBJ_RELEASE, reg, tmp_reg, exit_label | GC_DELREF reg | jne >1 | // zend_objects_store_del(obj); @@ -1159,11 +1150,11 @@ static void* dasm_labels[zend_lb_MAX]; |.endif | jmp exit_label |1: -| // if (UNEXPECTED(!GC_INFO(obj))) { -| IF_GC_INFO reg, exit_label +| IF_GC_MAY_NOT_LEAK reg, tmp_reg, >1 | // gc_possible_root(obj) | mov FCARG1a, reg | EXT_CALL gc_possible_root, r0 +|1: |.endmacro |.macro SAVE_VALID_OPLINE, opline @@ -3204,19 +3195,18 @@ static int zend_jit_assign_to_variable(dasm_State **Dst, |4: } if (RC_MAY_BE_N(var_info)) { - | IF_NOT_Z_FLAGS Ra(var_reg)+var_offset, IS_TYPE_COLLECTABLE, >5 if (var_reg == FP) { | GET_Z_PTR FCARG1a, Ra(var_reg)+var_offset - | IF_GC_INFO FCARG1a, >5 + | IF_GC_MAY_NOT_LEAK FCARG1a, eax, >5 } else if (var_reg != FCARG1a) { | GET_Z_PTR FCARG1a, Ra(var_reg)+var_offset - | IF_GC_INFO FCARG1a, >5 + | IF_GC_MAY_NOT_LEAK FCARG1a, eax, >5 | mov [r4], Ra(var_reg) // save } else { | GET_Z_PTR r0, Ra(var_reg)+var_offset - | IF_GC_INFO r0, >5 + | IF_GC_MAY_NOT_LEAK r0, r0, >5 | mov [r4], Ra(var_reg) // save - | mov FCARG1a, r0 + | GET_Z_PTR FCARG1a, Ra(var_reg)+var_offset } | EXT_CALL gc_possible_root, r0 if (var_reg != FP) { @@ -5807,7 +5797,7 @@ static int zend_jit_leave_func(dasm_State **Dst, const zend_op *opline, zend_op_ |5: } | // OBJ_RELEASE(object); - | OBJ_RELEASE r0, >4 + | OBJ_RELEASE r0, ecx, >4 | jmp >4 if (op_array->fn_flags & ZEND_ACC_STATIC) { |.code @@ -5823,7 +5813,7 @@ static int zend_jit_leave_func(dasm_State **Dst, const zend_op *opline, zend_op_ | // OBJ_RELEASE((zend_object*)execute_data->func->op_array.prototype); | mov r0, EX->func | mov r0, aword [r0 + offsetof(zend_op_array, prototype)] - | OBJ_RELEASE r0, >4 + | OBJ_RELEASE r0, ecx, >4 | jmp >4 |.code } @@ -6410,8 +6400,7 @@ static int zend_jit_bind_global(dasm_State **Dst, const zend_op *opline, zend_op if (op1_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) { |3: // GC_ZVAL_CHECK_POSSIBLE_ROOT(variable_ptr) - | IF_NOT_Z_REFCOUNTED FP + opline->op1.var, >5 - | IF_GC_INFO FCARG1a, >5 + | IF_GC_MAY_NOT_LEAK FCARG1a, eax, >5 | mov aword [r4], r0 //save | EXT_CALL gc_possible_root, r1 | mov r0, aword [r4] // restore From c7ee5347c9cbc67c5c1ab1c203f04d3ab3eef613 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Sat, 22 Oct 2016 22:39:01 +0800 Subject: [PATCH 307/569] Fixed build --- ext/opcache/jit/zend_jit_x86.dasc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 5c7bf8d144d6a..b94c09e5c94e5 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -3204,7 +3204,7 @@ static int zend_jit_assign_to_variable(dasm_State **Dst, | mov [r4], Ra(var_reg) // save } else { | GET_Z_PTR r0, Ra(var_reg)+var_offset - | IF_GC_MAY_NOT_LEAK r0, r0, >5 + | IF_GC_MAY_NOT_LEAK r0, eax, >5 | mov [r4], Ra(var_reg) // save | GET_Z_PTR FCARG1a, Ra(var_reg)+var_offset } From 1652605c88ad71d9d8d6b29cbfc0cac75f76c4aa Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Sat, 22 Oct 2016 23:04:47 +0800 Subject: [PATCH 308/569] Save checks for echo --- ext/opcache/jit/zend_jit.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index e0639720693d8..3322da2ce84a2 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -641,6 +641,8 @@ static int zend_may_throw(const zend_op *opline, zend_op_array *op_array, zend_s default: return 1; } + case ZEND_ECHO: + return (t1 & (MAY_BE_OBJECT|MAY_BE_ARRAY)); default: return 1; } From 230bcbe421ac71095d0cc203b605054d89c18b7f Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Sat, 22 Oct 2016 23:21:02 +0800 Subject: [PATCH 309/569] Remove useless seting to null codes is generated --- ext/opcache/jit/zend_jit_x86.dasc | 41 +++++++++++++++---------------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index b94c09e5c94e5..58921ee6bc7ce 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -6072,29 +6072,28 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, const zend_op *opline, zend |6: } - if ((opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) || - (op2_info & MAY_BE_UNDEF)) { + if ((opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) || (op2_info & MAY_BE_UNDEF)) { | SAVE_VALID_OPLINE opline + if (opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) { + if (op1_info & MAY_BE_REF) { + | IF_NOT_Z_TYPE FCARG1a, IS_UNDEF, >1 + } else { + | IF_NOT_Z_TYPE FP + opline->op1.var, IS_UNDEF, >1 + } + | // zend_error(E_NOTICE, "Undefined variable: %s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); + | mov FCARG1d, opline->op1.var + | EXT_CALL zend_jit_undefined_op_helper, r0 + |1: + } + if (op2_info & MAY_BE_UNDEF) { + | IF_NOT_Z_TYPE FP + opline->op2.var, IS_UNDEF, >1 + | mov FCARG1d, opline->op2.var + | EXT_CALL zend_jit_undefined_op_helper, r0 + |1: + } + | SET_Z_TYPE_INFO FP + opline->result.var, IS_NULL + | jmp >9 // END } - if (opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) { - if (op1_info & MAY_BE_REF) { - | IF_NOT_Z_TYPE FCARG1a, IS_UNDEF, >1 - } else { - | IF_NOT_Z_TYPE FP + opline->op1.var, IS_UNDEF, >1 - } - | // zend_error(E_NOTICE, "Undefined variable: %s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); - | mov FCARG1d, opline->op1.var - | EXT_CALL zend_jit_undefined_op_helper, r0 - |1: - } - if (op2_info & MAY_BE_UNDEF) { - | IF_NOT_Z_TYPE FP + opline->op2.var, IS_UNDEF, >1 - | mov FCARG1d, opline->op2.var - | EXT_CALL zend_jit_undefined_op_helper, r0 - |1: - } - | SET_Z_TYPE_INFO FP + opline->result.var, IS_NULL - | jmp >9 // END if (op1_info & MAY_BE_ARRAY) { |.code } From cefd589045a561e2c84ca39b4254451ef33f3d0f Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Sun, 23 Oct 2016 00:04:49 +0800 Subject: [PATCH 310/569] Removed unecessary jmps --- ext/opcache/jit/zend_jit_x86.dasc | 90 ++++++++++++++++++------------- 1 file changed, 53 insertions(+), 37 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 58921ee6bc7ce..0737db6b51949 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -6038,7 +6038,11 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, const zend_op *opline, zend } else { ZEND_ASSERT(0); } + if ((op1_info & MAY_BE_ARRAY) || + ((op1_info & MAY_BE_OBJECT) || + (opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) || (op2_info & MAY_BE_UNDEF))) { | jmp >9 // END + } |6: } @@ -6068,57 +6072,68 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, const zend_op *opline, zend } else { ZEND_ASSERT(0); } + if ((op1_info & MAY_BE_ARRAY) || + ((opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) || (op2_info & MAY_BE_UNDEF))) { | jmp >9 // END + } |6: } if ((opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) || (op2_info & MAY_BE_UNDEF)) { | SAVE_VALID_OPLINE opline - if (opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) { - if (op1_info & MAY_BE_REF) { - | IF_NOT_Z_TYPE FCARG1a, IS_UNDEF, >1 - } else { - | IF_NOT_Z_TYPE FP + opline->op1.var, IS_UNDEF, >1 - } - | // zend_error(E_NOTICE, "Undefined variable: %s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); - | mov FCARG1d, opline->op1.var - | EXT_CALL zend_jit_undefined_op_helper, r0 - |1: - } - if (op2_info & MAY_BE_UNDEF) { - | IF_NOT_Z_TYPE FP + opline->op2.var, IS_UNDEF, >1 - | mov FCARG1d, opline->op2.var - | EXT_CALL zend_jit_undefined_op_helper, r0 - |1: + } + + if (opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) { + if (op1_info & MAY_BE_REF) { + | IF_NOT_Z_TYPE FCARG1a, IS_UNDEF, >1 + } else { + | IF_NOT_Z_TYPE FP + opline->op1.var, IS_UNDEF, >1 } + | // zend_error(E_NOTICE, "Undefined variable: %s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); + | mov FCARG1d, opline->op1.var + | EXT_CALL zend_jit_undefined_op_helper, r0 + |1: + } + + if (op2_info & MAY_BE_UNDEF) { + | IF_NOT_Z_TYPE FP + opline->op2.var, IS_UNDEF, >1 + | mov FCARG1d, opline->op2.var + | EXT_CALL zend_jit_undefined_op_helper, r0 + |1: + } + + if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_STRING|MAY_BE_OBJECT)))) { | SET_Z_TYPE_INFO FP + opline->result.var, IS_NULL - | jmp >9 // END } + if (op1_info & MAY_BE_ARRAY) { + | jmp >9 // END |.code } } - |8: - if (res_info & MAY_BE_REF) { - | // ZVAL_COPY_UNREF - | IF_NOT_Z_REFCOUNTED r0, >2 - | GET_Z_PTR r1, r0 - | IF_NOT_Z_TYPE r0, IS_REFERENCE, >1 - | cmp dword [r1], 1 - | jne >1 - | lea FCARG1a, [FP + opline->result.var] - | mov FCARG2a, r0 - | EXT_CALL zend_jit_zval_copy_unref_helper, r0 - | jmp >9 - |1: - | GC_ADDREF r1 - |2: - | ZVAL_COPY_VALUE FP + opline->result.var, r0, MAY_BE_ANY, r1, ecx, r2 - } else { - | // ZVAL_COPY - | ZVAL_COPY_VALUE FP + opline->result.var, r0, MAY_BE_ANY, r1, ecx, r2 - | TRY_ADDREF res_info, ch, r2 + if (op1_info & MAY_BE_ARRAY) { + |8: + if (res_info & MAY_BE_REF) { + | // ZVAL_COPY_UNREF + | IF_NOT_Z_REFCOUNTED r0, >2 + | GET_Z_PTR r1, r0 + | IF_NOT_Z_TYPE r0, IS_REFERENCE, >1 + | cmp dword [r1], 1 + | jne >1 + | lea FCARG1a, [FP + opline->result.var] + | mov FCARG2a, r0 + | EXT_CALL zend_jit_zval_copy_unref_helper, r0 + | jmp >9 + |1: + | GC_ADDREF r1 + |2: + | ZVAL_COPY_VALUE FP + opline->result.var, r0, MAY_BE_ANY, r1, ecx, r2 + } else { + | // ZVAL_COPY + | ZVAL_COPY_VALUE FP + opline->result.var, r0, MAY_BE_ANY, r1, ecx, r2 + | TRY_ADDREF res_info, ch, r2 + } } |9: // END @@ -6558,6 +6573,7 @@ static int zend_jit_fetch_obj_r(dasm_State **Dst, zend_op *opline, zend_op_array goto fallback; } + res_info = RES_INFO(); op1_info = OP1_INFO(); if ((op1_info & MAY_BE_ANY) != MAY_BE_OBJECT) { goto fallback; From 1a72bd609acb14fa56be162fd40947de8131c628 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Sun, 23 Oct 2016 00:20:30 +0800 Subject: [PATCH 311/569] Fixed IS_NULL handling --- ext/opcache/jit/zend_jit_x86.dasc | 44 +++++++++++++++---------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 0737db6b51949..dbdd149e4300b 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -6039,8 +6039,7 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, const zend_op *opline, zend ZEND_ASSERT(0); } if ((op1_info & MAY_BE_ARRAY) || - ((op1_info & MAY_BE_OBJECT) || - (opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) || (op2_info & MAY_BE_UNDEF))) { + (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_STRING)))) { | jmp >9 // END } |6: @@ -6073,7 +6072,7 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, const zend_op *opline, zend ZEND_ASSERT(0); } if ((op1_info & MAY_BE_ARRAY) || - ((opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) || (op2_info & MAY_BE_UNDEF))) { + (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_STRING|MAY_BE_OBJECT)))) { | jmp >9 // END } |6: @@ -6081,33 +6080,34 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, const zend_op *opline, zend if ((opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) || (op2_info & MAY_BE_UNDEF)) { | SAVE_VALID_OPLINE opline - } - - if (opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) { - if (op1_info & MAY_BE_REF) { - | IF_NOT_Z_TYPE FCARG1a, IS_UNDEF, >1 - } else { - | IF_NOT_Z_TYPE FP + opline->op1.var, IS_UNDEF, >1 + if (opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) { + if (op1_info & MAY_BE_REF) { + | IF_NOT_Z_TYPE FCARG1a, IS_UNDEF, >1 + } else { + | IF_NOT_Z_TYPE FP + opline->op1.var, IS_UNDEF, >1 + } + | // zend_error(E_NOTICE, "Undefined variable: %s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); + | mov FCARG1d, opline->op1.var + | EXT_CALL zend_jit_undefined_op_helper, r0 + |1: } - | // zend_error(E_NOTICE, "Undefined variable: %s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); - | mov FCARG1d, opline->op1.var - | EXT_CALL zend_jit_undefined_op_helper, r0 - |1: - } - if (op2_info & MAY_BE_UNDEF) { - | IF_NOT_Z_TYPE FP + opline->op2.var, IS_UNDEF, >1 - | mov FCARG1d, opline->op2.var - | EXT_CALL zend_jit_undefined_op_helper, r0 - |1: + if (op2_info & MAY_BE_UNDEF) { + | IF_NOT_Z_TYPE FP + opline->op2.var, IS_UNDEF, >1 + | mov FCARG1d, opline->op2.var + | EXT_CALL zend_jit_undefined_op_helper, r0 + |1: + } } - if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_STRING|MAY_BE_OBJECT)))) { + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_STRING|MAY_BE_OBJECT))) { | SET_Z_TYPE_INFO FP + opline->result.var, IS_NULL + if (op1_info & MAY_BE_ARRAY) { + | jmp >9 // END + } } if (op1_info & MAY_BE_ARRAY) { - | jmp >9 // END |.code } } From b6bdcddbf5bec3a1853dc14862871a4e8cafbb4c Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Sun, 23 Oct 2016 09:04:52 +0800 Subject: [PATCH 312/569] Revert "Save checks for echo" This reverts commit 1652605c88ad71d9d8d6b29cbfc0cac75f76c4aa. --- ext/opcache/jit/zend_jit.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 3322da2ce84a2..e0639720693d8 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -641,8 +641,6 @@ static int zend_may_throw(const zend_op *opline, zend_op_array *op_array, zend_s default: return 1; } - case ZEND_ECHO: - return (t1 & (MAY_BE_OBJECT|MAY_BE_ARRAY)); default: return 1; } From 0d5fb28e4f0e3958d82bebddac026f283e3a11e2 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Mon, 24 Oct 2016 12:42:17 +0800 Subject: [PATCH 313/569] Improved ZVAL_DTOR_FUNC --- ext/opcache/jit/zend_jit_x86.dasc | 140 ++++++++++++++++++------------ 1 file changed, 84 insertions(+), 56 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index dbdd149e4300b..5c4e1447eb5b1 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -888,7 +888,34 @@ static void* dasm_labels[zend_lb_MAX]; |.endmacro // zval should be in FCARG1a -|.macro ZVAL_DTOR_FUNC, filename, lineno // arg1 must be in FCARG1a +|.macro ZVAL_DTOR_FUNC, var_info, filename, lineno // arg1 must be in FCARG1a +||do { +|| if (has_concrete_type(var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { +|| zend_uchar type = concrete_type(var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)); +|| if (type == IS_STRING && !ZEND_DEBUG) { +| test byte [FCARG1a + 5], IS_STR_PERSISTENT +| jnz >1 +| EXT_CALL _efree, r0 +| jmp >2 +|1: +| EXT_CALL free, r0 +|2: +|| break; +|| } else if (type == IS_ARRAY) { +| EXT_CALL zend_array_destroy, r0 +|| break; +|| } else if (type == IS_OBJECT) { +|.if X64 +| EXT_CALL zend_objects_store_del, r0 +|.else +| sub r4, 12 +| push FCARG1a +| EXT_CALL zend_objects_store_del, r0 +| add r4, 16 +|.endif +|| break; +|| } +|| } || if (ZEND_DEBUG) { || if (filename) { | LOAD_ADDR FCARG2a, ZSTR_VAL((zend_string*)filename) @@ -901,7 +928,8 @@ static void* dasm_labels[zend_lb_MAX]; | push lineno |.endif || } -| EXT_CALL _zval_dtor_func, r0 +| EXT_CALL _zval_dtor_func, r0 +||} while(0); |.endmacro |.macro ZVAL_PTR_DTOR, zv, op_info, gc, cold, safe, filename, lineno @@ -913,7 +941,7 @@ static void* dasm_labels[zend_lb_MAX]; |.cold_code |1: || } else { -| IF_NOT_Z_REFCOUNTED zv, >2 +| IF_NOT_Z_REFCOUNTED zv, >4 || } || } | // if (!Z_DELREF_P(cv)) { @@ -922,9 +950,9 @@ static void* dasm_labels[zend_lb_MAX]; || if (RC_MAY_BE_1(op_info)) { || if (RC_MAY_BE_N(op_info)) { || if (gc && RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT))) { -| jnz >1 +| jnz >3 || } else { -| jnz >2 +| jnz >4 || } || } || if (safe) { @@ -932,11 +960,11 @@ static void* dasm_labels[zend_lb_MAX]; | SET_Z_TYPE_INFO zv, IS_NULL || } | // zval_dtor_func(r); -| ZVAL_DTOR_FUNC filename, lineno +| ZVAL_DTOR_FUNC op_info, filename, lineno || if (gc && RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT))) { -| jmp >2 +| jmp >4 || } -|1: +|3: || } || if (gc && RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT))) { || if ((op_info) & MAY_BE_REF) { @@ -944,22 +972,22 @@ static void* dasm_labels[zend_lb_MAX]; | cmp byte [FCARG1a + 4], IS_REFERENCE | jnz >1 | GET_Z_PTR FCARG1a, FCARG1a + offsetof(zend_reference, val) -| IF_NOT_Z_REFCOUNTED zv, >2 +| IF_NOT_Z_REFCOUNTED zv, >4 | GET_Z_PTR FCARG1a, zv |1: || } -| IF_GC_MAY_NOT_LEAK FCARG1a, eax, >2 +| IF_GC_MAY_NOT_LEAK FCARG1a, eax, >4 | // gc_possible_root(Z_COUNTED_P(z)) | EXT_CALL gc_possible_root, r0 || if (cold && ((op_info) & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE)))) { -| jmp >2 +| jmp >4 |.code || } || } else if (cold && ((op_info) & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE)))) { -| jmp >2 +| jmp >4 |.code || } -|2: +|4: || } |.endmacro @@ -2318,7 +2346,7 @@ static int zend_jit_math_helper(dasm_State **Dst, uint32_t res_offset, uint32_t res_info, zend_bool separate_op1) -/* Labels: 1,2,3,4,5 */ +/* Labels: 1,2,3,4,5,6 */ { zend_bool same_ops = (op1_type == op2_type) && (op1.var == op2.var); zend_bool has_slow = @@ -2332,7 +2360,7 @@ static int zend_jit_math_helper(dasm_State **Dst, if (op1_info & MAY_BE_DOUBLE) { | IF_NOT_Z_TYPE Ra(op1_reg)+op1_offset, IS_LONG, >3 } else { - | IF_NOT_Z_TYPE Ra(op1_reg)+op1_offset, IS_LONG, >5 + | IF_NOT_Z_TYPE Ra(op1_reg)+op1_offset, IS_LONG, >6 } } if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_LONG))) { @@ -2341,15 +2369,15 @@ static int zend_jit_math_helper(dasm_State **Dst, |.cold_code |1: if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { - | IF_NOT_Z_TYPE Ra(op2_reg)+op2_offset, IS_DOUBLE, >5 + | IF_NOT_Z_TYPE Ra(op2_reg)+op2_offset, IS_DOUBLE, >6 } if (!zend_jit_math_long_double(Dst, opline, op1_type, op1, op1_reg, op1_offset, op2_type, op2, op2_reg, op2_offset, res_reg, res_offset)) { return 0; } - | jmp >4 + | jmp >5 |.code } else { - | IF_NOT_Z_TYPE Ra(op2_reg)+op2_offset, IS_LONG, >5 + | IF_NOT_Z_TYPE Ra(op2_reg)+op2_offset, IS_LONG, >6 } } if (!zend_jit_math_long_long(Dst, opline, op_array, ssa, op1_type, op1, op1_reg, op1_offset, op2_type, op2, op2_reg, op2_offset, res_reg, res_offset, res_info)) { @@ -2359,30 +2387,30 @@ static int zend_jit_math_helper(dasm_State **Dst, |.cold_code |3: if (op1_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { - | IF_NOT_Z_TYPE Ra(op1_reg)+op1_offset, IS_DOUBLE, >5 + | IF_NOT_Z_TYPE Ra(op1_reg)+op1_offset, IS_DOUBLE, >6 } if (op2_info & MAY_BE_DOUBLE) { if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { if (!same_ops) { | IF_NOT_Z_TYPE, Ra(op2_reg)+op2_offset, IS_DOUBLE, >1 } else { - | IF_NOT_Z_TYPE, Ra(op2_reg)+op2_offset, IS_DOUBLE, >5 + | IF_NOT_Z_TYPE, Ra(op2_reg)+op2_offset, IS_DOUBLE, >6 } } if (!zend_jit_math_double_double(Dst, opline, op1_type, op1, op1_reg, op1_offset, op2_type, op2, op2_reg, op2_offset, res_reg, res_offset)) { return 0; } - | jmp >4 + | jmp >5 } if (!same_ops) { |1: if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { - | IF_NOT_Z_TYPE Ra(op2_reg)+op2_offset, IS_LONG, >5 + | IF_NOT_Z_TYPE Ra(op2_reg)+op2_offset, IS_LONG, >6 } if (!zend_jit_math_double_long(Dst, opline, op1_type, op1, op1_reg, op1_offset, op2_type, op2, op2_reg, op2_offset, res_reg, res_offset)) { return 0; } - | jmp >4 + | jmp >5 } |.code } @@ -2390,14 +2418,14 @@ static int zend_jit_math_helper(dasm_State **Dst, !(op1_info & MAY_BE_LONG) && (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { if (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) { - | IF_NOT_Z_TYPE Ra(op1_reg)+op1_offset, IS_DOUBLE, >5 + | IF_NOT_Z_TYPE Ra(op1_reg)+op1_offset, IS_DOUBLE, >6 } if (op2_info & MAY_BE_DOUBLE) { if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { if (!same_ops && (op2_info & MAY_BE_LONG)) { | IF_NOT_Z_TYPE Ra(op2_reg)+op2_offset, IS_DOUBLE, >1 } else { - | IF_NOT_Z_TYPE Ra(op2_reg)+op2_offset, IS_DOUBLE, >5 + | IF_NOT_Z_TYPE Ra(op2_reg)+op2_offset, IS_DOUBLE, >6 } } if (!zend_jit_math_double_double(Dst, opline, op1_type, op1, op1_reg, op1_offset, op2_type, op2, op2_reg, op2_offset, res_reg, res_offset)) { @@ -2416,7 +2444,7 @@ static int zend_jit_math_helper(dasm_State **Dst, return 0; } if (op2_info & MAY_BE_DOUBLE) { - | jmp >4 + | jmp >5 |.code } } @@ -2424,14 +2452,14 @@ static int zend_jit_math_helper(dasm_State **Dst, !(op2_info & MAY_BE_LONG) && (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { if (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) { - | IF_NOT_Z_TYPE Ra(op2_reg)+op2_offset, IS_DOUBLE, >5 + | IF_NOT_Z_TYPE Ra(op2_reg)+op2_offset, IS_DOUBLE, >6 } if (op1_info & MAY_BE_DOUBLE) { if (!same_ops && (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { if (!same_ops && (op1_info & MAY_BE_LONG)) { | IF_NOT_Z_TYPE Ra(op1_reg)+op1_offset, IS_DOUBLE, >1 } else { - | IF_NOT_Z_TYPE Ra(op1_reg)+op1_offset, IS_DOUBLE, >5 + | IF_NOT_Z_TYPE Ra(op1_reg)+op1_offset, IS_DOUBLE, >6 } } if (!zend_jit_math_double_double(Dst, opline, op1_type, op1, op1_reg, op1_offset, op2_type, op2, op2_reg, op2_offset, res_reg, res_offset)) { @@ -2444,23 +2472,23 @@ static int zend_jit_math_helper(dasm_State **Dst, } |1: if (op1_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) { - | IF_NOT_Z_TYPE Ra(op1_reg)+op1_offset, IS_LONG, >5 + | IF_NOT_Z_TYPE Ra(op1_reg)+op1_offset, IS_LONG, >6 } if (!zend_jit_math_long_double(Dst, opline, op1_type, op1, op1_reg, op1_offset, op2_type, op2, op2_reg, op2_offset, res_reg, res_offset)) { return 0; } if (op1_info & MAY_BE_DOUBLE) { - | jmp >4 + | jmp >5 |.code } } } - |4: + |5: if (has_slow) { |.cold_code - |5: + |6: | SAVE_VALID_OPLINE opline if (separate_op1) { if (op1_reg != FCARG1a || op1_offset != 0) { @@ -2507,7 +2535,7 @@ static int zend_jit_math_helper(dasm_State **Dst, || if (zend_may_throw(opline, op_array, ssa)) { || zend_jit_check_exception(Dst); || } - | jmp <4 + | jmp <5 |.code } @@ -2576,10 +2604,10 @@ static int zend_jit_concat_helper(dasm_State **Dst, #if 1 if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) { if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) { - | IF_NOT_Z_TYPE Ra(op1_reg)+op1_offset, IS_STRING, >5 + | IF_NOT_Z_TYPE Ra(op1_reg)+op1_offset, IS_STRING, >6 } if (op2_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) { - | IF_NOT_Z_TYPE Ra(op2_reg)+op2_offset, IS_STRING, >5 + | IF_NOT_Z_TYPE Ra(op2_reg)+op2_offset, IS_STRING, >6 } if (op1_type != IS_CONST && op1_reg == res_reg && op1_offset == res_offset) { if (res_reg != FCARG1a || res_offset != 0) { @@ -2618,13 +2646,13 @@ static int zend_jit_concat_helper(dasm_State **Dst, } | FREE_OP op1_type, op1, op1_info, 0, op_array, opline->lineno | FREE_OP op2_type, op2, op2_info, 0, op_array, opline->lineno - |4: + |5: } if ((op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) || (op2_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING))) { if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) { |.cold_code - |5: + |6: } #endif | SAVE_VALID_OPLINE opline @@ -2658,7 +2686,7 @@ static int zend_jit_concat_helper(dasm_State **Dst, || } #if 1 if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) { - | jmp <4 + | jmp <5 |.code } } @@ -3190,7 +3218,7 @@ static int zend_jit_assign_to_variable(dasm_State **Dst, return 0; } | mov FCARG1a, aword [r4] // restore - | ZVAL_DTOR_FUNC op_array->filename, opline->lineno + | ZVAL_DTOR_FUNC var_info, op_array->filename, opline->lineno | jmp >3 |4: } @@ -4615,14 +4643,14 @@ static int zend_jit_jmpznz(dasm_State **Dst, const zend_op *opline, int b, zend_ if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { | test byte [FP + opline->op1.var + 9], IS_TYPE_REFCOUNTED - | jz >1 + | jz >3 | mov FCARG1a, aword [FP + opline->op1.var] | dec dword [FCARG1a] - | jnz >1 + | jnz >3 | mov aword [r4], r0 // save - | ZVAL_DTOR_FUNC op_array->filename, opline->lineno + | ZVAL_DTOR_FUNC op1_info, op_array->filename, opline->lineno | mov r0, aword [r4] // restore - |1: + |3: } if (zend_may_throw(opline, op_array, ssa)) { if (!zend_jit_check_exception(Dst)) { @@ -5531,7 +5559,7 @@ static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, int b, i | GC_DELREF FCARG1a if (RC_MAY_BE_1(op1_info)) { if (RC_MAY_BE_N(op1_info)) { - | jnz >1 + | jnz >3 } if (op1_info & MAY_BE_REF) { | mov al, byte [r0 + 8] @@ -5540,17 +5568,17 @@ static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, int b, i } | mov byte [r4], al // save | // zval_dtor_func(r); - | ZVAL_DTOR_FUNC op_array->filename, opline->lineno + | ZVAL_DTOR_FUNC op1_info, op_array->filename, opline->lineno | mov cl, byte [r4] // restore |jmp >2 } if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { if (!RC_MAY_BE_1(op1_info)) { - | jmp >1 + | jmp >3 } |.code } - |1: + |3: if (op1_info & MAY_BE_REF) { | mov cl, byte [r0 + 8] } else { @@ -5616,7 +5644,7 @@ static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, int b, i | GC_DELREF FCARG1a if (RC_MAY_BE_1(op1_info)) { if (RC_MAY_BE_N(op1_info)) { - | jnz >1 + | jnz >3 } if (op1_info & MAY_BE_REF) { | mov al, byte [r0 + 8] @@ -5625,17 +5653,17 @@ static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, int b, i } | mov byte [r4], al // save | // zval_dtor_func(r); - | ZVAL_DTOR_FUNC op_array->filename, opline->lineno + | ZVAL_DTOR_FUNC op1_info, op_array->filename, opline->lineno | mov cl, byte [r4] // restore |jmp >2 } if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { if (!RC_MAY_BE_1(op1_info)) { - | jmp >1 + | jmp >3 } |.code } - |1: + |3: if (op1_info & MAY_BE_REF) { | mov cl, byte [r0 + 8] } else { @@ -5886,7 +5914,7 @@ static int zend_jit_return(dasm_State **Dst, const zend_op *opline, zend_op_arra || } | //SAVE_OPLINE() | SAVE_VALID_OPLINE opline - | ZVAL_DTOR_FUNC op_array->filename, opline->lineno + | ZVAL_DTOR_FUNC op1_info, op_array->filename, opline->lineno | //????mov r1, EX->return_value // reload ??? || } || if (jit_return_label >= 0) { @@ -6408,7 +6436,7 @@ static int zend_jit_bind_global(dasm_State **Dst, const zend_op *opline, zend_op | jnz >3 } | mov aword [r4], r0 // save - | ZVAL_DTOR_FUNC op_array->filename, opline->lineno + | ZVAL_DTOR_FUNC op1_info, op_array->filename, opline->lineno | mov r0, aword [r4] // restore | jmp >5 if (op1_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) { @@ -6455,7 +6483,7 @@ static int zend_jit_recv_init(dasm_State **Dst, const zend_op *opline, zend_op_a zval *zv = RT_CONSTANT(op_array, opline->op2); | cmp dword EX->This.u2.num_args, arg_num - | jae >4 + | jae >5 | ZVAL_COPY_CONST FP + opline->result.var, -1, zv, r0 if (Z_REFCOUNTED_P(zv)) { | ADDREF_CONST zv, r0 @@ -6480,7 +6508,7 @@ static int zend_jit_recv_init(dasm_State **Dst, const zend_op *opline, zend_op_a | test al, al | jnz >7 } - |4: + |5: if (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) { do { if (arg_num <= op_array->num_args) { @@ -6533,7 +6561,7 @@ static int zend_jit_recv_init(dasm_State **Dst, const zend_op *opline, zend_op_a |7: | ZVAL_PTR_DTOR FP + opline->result.var, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN, 1, 0, 0, op_array->filename, opline->lineno | SET_Z_TYPE_INFO FP + opline->result.var, IS_UNDEF - | jmp <4 + | jmp <5 } if (has_slow & 2) { |8: From f8423a7524bfce7e46a5428c8a889458a5d58333 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 24 Oct 2016 15:34:56 +0300 Subject: [PATCH 314/569] Added support for Intel VTune --- ext/opcache/jit/vtune/ittnotify_config.h | 596 +++++++++++++++++++ ext/opcache/jit/vtune/ittnotify_types.h | 115 ++++ ext/opcache/jit/vtune/jitprofiling.c | 315 ++++++++++ ext/opcache/jit/vtune/jitprofiling.h | 694 +++++++++++++++++++++++ ext/opcache/jit/zend_jit.c | 20 +- ext/opcache/jit/zend_jit.h | 1 + ext/opcache/jit/zend_jit_vtune.c | 49 ++ 7 files changed, 1786 insertions(+), 4 deletions(-) create mode 100644 ext/opcache/jit/vtune/ittnotify_config.h create mode 100644 ext/opcache/jit/vtune/ittnotify_types.h create mode 100644 ext/opcache/jit/vtune/jitprofiling.c create mode 100644 ext/opcache/jit/vtune/jitprofiling.h create mode 100644 ext/opcache/jit/zend_jit_vtune.c diff --git a/ext/opcache/jit/vtune/ittnotify_config.h b/ext/opcache/jit/vtune/ittnotify_config.h new file mode 100644 index 0000000000000..fc3a476cdd1c9 --- /dev/null +++ b/ext/opcache/jit/vtune/ittnotify_config.h @@ -0,0 +1,596 @@ +/* + This file is provided under a dual BSD/GPLv2 license. When using or + redistributing this file, you may do so under either license. + + GPL LICENSE SUMMARY + + Copyright (c) 2005-2014 Intel Corporation. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. + The full GNU General Public License is included in this distribution + in the file called LICENSE.GPL. + + Contact Information: + http://software.intel.com/en-us/articles/intel-vtune-amplifier-xe/ + + BSD LICENSE + + Copyright (c) 2005-2014 Intel Corporation. All rights reserved. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef _ITTNOTIFY_CONFIG_H_ +#define _ITTNOTIFY_CONFIG_H_ + +/** @cond exclude_from_documentation */ +#ifndef ITT_OS_WIN +# define ITT_OS_WIN 1 +#endif /* ITT_OS_WIN */ + +#ifndef ITT_OS_LINUX +# define ITT_OS_LINUX 2 +#endif /* ITT_OS_LINUX */ + +#ifndef ITT_OS_MAC +# define ITT_OS_MAC 3 +#endif /* ITT_OS_MAC */ + +#ifndef ITT_OS_FREEBSD +# define ITT_OS_FREEBSD 4 +#endif /* ITT_OS_FREEBSD */ + +#ifndef ITT_OS +# if defined WIN32 || defined _WIN32 +# define ITT_OS ITT_OS_WIN +# elif defined( __APPLE__ ) && defined( __MACH__ ) +# define ITT_OS ITT_OS_MAC +# elif defined( __FreeBSD__ ) +# define ITT_OS ITT_OS_FREEBSD +# else +# define ITT_OS ITT_OS_LINUX +# endif +#endif /* ITT_OS */ + +#ifndef ITT_PLATFORM_WIN +# define ITT_PLATFORM_WIN 1 +#endif /* ITT_PLATFORM_WIN */ + +#ifndef ITT_PLATFORM_POSIX +# define ITT_PLATFORM_POSIX 2 +#endif /* ITT_PLATFORM_POSIX */ + +#ifndef ITT_PLATFORM_MAC +# define ITT_PLATFORM_MAC 3 +#endif /* ITT_PLATFORM_MAC */ + +#ifndef ITT_PLATFORM_FREEBSD +# define ITT_PLATFORM_FREEBSD 4 +#endif /* ITT_PLATFORM_FREEBSD */ + +#ifndef ITT_PLATFORM +# if ITT_OS==ITT_OS_WIN +# define ITT_PLATFORM ITT_PLATFORM_WIN +# elif ITT_OS==ITT_OS_MAC +# define ITT_PLATFORM ITT_PLATFORM_MAC +# elif ITT_OS==ITT_OS_FREEBSD +# define ITT_PLATFORM ITT_PLATFORM_FREEBSD +# else +# define ITT_PLATFORM ITT_PLATFORM_POSIX +# endif +#endif /* ITT_PLATFORM */ + +#if defined(_UNICODE) && !defined(UNICODE) +#define UNICODE +#endif + +#include +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#include +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#include +#if defined(UNICODE) || defined(_UNICODE) +#include +#endif /* UNICODE || _UNICODE */ +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ + +#ifndef ITTAPI_CDECL +# if ITT_PLATFORM==ITT_PLATFORM_WIN +# define ITTAPI_CDECL __cdecl +# else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +# if defined _M_IX86 || defined __i386__ +# define ITTAPI_CDECL __attribute__ ((cdecl)) +# else /* _M_IX86 || __i386__ */ +# define ITTAPI_CDECL /* actual only on x86 platform */ +# endif /* _M_IX86 || __i386__ */ +# endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* ITTAPI_CDECL */ + +#ifndef STDCALL +# if ITT_PLATFORM==ITT_PLATFORM_WIN +# define STDCALL __stdcall +# else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +# if defined _M_IX86 || defined __i386__ +# define STDCALL __attribute__ ((stdcall)) +# else /* _M_IX86 || __i386__ */ +# define STDCALL /* supported only on x86 platform */ +# endif /* _M_IX86 || __i386__ */ +# endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* STDCALL */ + +#define ITTAPI ITTAPI_CDECL +#define LIBITTAPI ITTAPI_CDECL + +/* TODO: Temporary for compatibility! */ +#define ITTAPI_CALL ITTAPI_CDECL +#define LIBITTAPI_CALL ITTAPI_CDECL + +#if ITT_PLATFORM==ITT_PLATFORM_WIN +/* use __forceinline (VC++ specific) */ +#define ITT_INLINE __forceinline +#define ITT_INLINE_ATTRIBUTE /* nothing */ +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +/* + * Generally, functions are not inlined unless optimization is specified. + * For functions declared inline, this attribute inlines the function even + * if no optimization level was specified. + */ +#ifdef __STRICT_ANSI__ +#define ITT_INLINE static +#define ITT_INLINE_ATTRIBUTE __attribute__((unused)) +#else /* __STRICT_ANSI__ */ +#define ITT_INLINE static inline +#define ITT_INLINE_ATTRIBUTE __attribute__((always_inline, unused)) +#endif /* __STRICT_ANSI__ */ +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +/** @endcond */ + +#ifndef ITT_ARCH_IA32 +# define ITT_ARCH_IA32 1 +#endif /* ITT_ARCH_IA32 */ + +#ifndef ITT_ARCH_IA32E +# define ITT_ARCH_IA32E 2 +#endif /* ITT_ARCH_IA32E */ + +#ifndef ITT_ARCH_ARM +# define ITT_ARCH_ARM 4 +#endif /* ITT_ARCH_ARM */ + +#ifndef ITT_ARCH_PPC64 +# define ITT_ARCH_PPC64 5 +#endif /* ITT_ARCH_PPC64 */ + +#ifndef ITT_ARCH +# if defined _M_IX86 || defined __i386__ +# define ITT_ARCH ITT_ARCH_IA32 +# elif defined _M_X64 || defined _M_AMD64 || defined __x86_64__ +# define ITT_ARCH ITT_ARCH_IA32E +# elif defined _M_IA64 || defined __ia64__ +# define ITT_ARCH ITT_ARCH_IA64 +# elif defined _M_ARM || defined __arm__ +# define ITT_ARCH ITT_ARCH_ARM +# elif defined __powerpc64__ +# define ITT_ARCH ITT_ARCH_PPC64 +# endif +#endif + +#ifdef __cplusplus +# define ITT_EXTERN_C extern "C" +# define ITT_EXTERN_C_BEGIN extern "C" { +# define ITT_EXTERN_C_END } +#else +# define ITT_EXTERN_C /* nothing */ +# define ITT_EXTERN_C_BEGIN /* nothing */ +# define ITT_EXTERN_C_END /* nothing */ +#endif /* __cplusplus */ + +#define ITT_TO_STR_AUX(x) #x +#define ITT_TO_STR(x) ITT_TO_STR_AUX(x) + +#define __ITT_BUILD_ASSERT(expr, suffix) do { \ + static char __itt_build_check_##suffix[(expr) ? 1 : -1]; \ + __itt_build_check_##suffix[0] = 0; \ +} while(0) +#define _ITT_BUILD_ASSERT(expr, suffix) __ITT_BUILD_ASSERT((expr), suffix) +#define ITT_BUILD_ASSERT(expr) _ITT_BUILD_ASSERT((expr), __LINE__) + +#define ITT_MAGIC { 0xED, 0xAB, 0xAB, 0xEC, 0x0D, 0xEE, 0xDA, 0x30 } + +/* Replace with snapshot date YYYYMMDD for promotion build. */ +#define API_VERSION_BUILD 20151119 + +#ifndef API_VERSION_NUM +#define API_VERSION_NUM 0.0.0 +#endif /* API_VERSION_NUM */ + +#define API_VERSION "ITT-API-Version " ITT_TO_STR(API_VERSION_NUM) \ + " (" ITT_TO_STR(API_VERSION_BUILD) ")" + +/* OS communication functions */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#include +typedef HMODULE lib_t; +typedef DWORD TIDT; +typedef CRITICAL_SECTION mutex_t; +#define MUTEX_INITIALIZER { 0 } +#define strong_alias(name, aliasname) /* empty for Windows */ +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#include +#if defined(UNICODE) || defined(_UNICODE) +#include +#endif /* UNICODE */ +#ifndef _GNU_SOURCE +#define _GNU_SOURCE 1 /* need for PTHREAD_MUTEX_RECURSIVE */ +#endif /* _GNU_SOURCE */ +#ifndef __USE_UNIX98 +#define __USE_UNIX98 1 /* need for PTHREAD_MUTEX_RECURSIVE, on SLES11.1 with gcc 4.3.4 wherein pthread.h missing dependency on __USE_XOPEN2K8 */ +#endif /*__USE_UNIX98*/ +#include +typedef void* lib_t; +typedef pthread_t TIDT; +typedef pthread_mutex_t mutex_t; +#define MUTEX_INITIALIZER PTHREAD_MUTEX_INITIALIZER +#define _strong_alias(name, aliasname) \ + extern __typeof (name) aliasname __attribute__ ((alias (#name))); +#define strong_alias(name, aliasname) _strong_alias(name, aliasname) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ + +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_get_proc(lib, name) GetProcAddress(lib, name) +#define __itt_mutex_init(mutex) InitializeCriticalSection(mutex) +#define __itt_mutex_lock(mutex) EnterCriticalSection(mutex) +#define __itt_mutex_unlock(mutex) LeaveCriticalSection(mutex) +#define __itt_load_lib(name) LoadLibraryA(name) +#define __itt_unload_lib(handle) FreeLibrary(handle) +#define __itt_system_error() (int)GetLastError() +#define __itt_fstrcmp(s1, s2) lstrcmpA(s1, s2) +#define __itt_fstrnlen(s, l) strnlen_s(s, l) +#define __itt_fstrcpyn(s1, b, s2, l) strncpy_s(s1, b, s2, l) +#define __itt_fstrdup(s) _strdup(s) +#define __itt_thread_id() GetCurrentThreadId() +#define __itt_thread_yield() SwitchToThread() +#ifndef ITT_SIMPLE_INIT +ITT_INLINE long +__itt_interlocked_increment(volatile long* ptr) ITT_INLINE_ATTRIBUTE; +ITT_INLINE long __itt_interlocked_increment(volatile long* ptr) +{ + return InterlockedIncrement(ptr); +} +#endif /* ITT_SIMPLE_INIT */ +#else /* ITT_PLATFORM!=ITT_PLATFORM_WIN */ +#define __itt_get_proc(lib, name) dlsym(lib, name) +#define __itt_mutex_init(mutex) {\ + pthread_mutexattr_t mutex_attr; \ + int error_code = pthread_mutexattr_init(&mutex_attr); \ + if (error_code) \ + __itt_report_error(__itt_error_system, "pthread_mutexattr_init", \ + error_code); \ + error_code = pthread_mutexattr_settype(&mutex_attr, \ + PTHREAD_MUTEX_RECURSIVE); \ + if (error_code) \ + __itt_report_error(__itt_error_system, "pthread_mutexattr_settype", \ + error_code); \ + error_code = pthread_mutex_init(mutex, &mutex_attr); \ + if (error_code) \ + __itt_report_error(__itt_error_system, "pthread_mutex_init", \ + error_code); \ + error_code = pthread_mutexattr_destroy(&mutex_attr); \ + if (error_code) \ + __itt_report_error(__itt_error_system, "pthread_mutexattr_destroy", \ + error_code); \ +} +#define __itt_mutex_lock(mutex) pthread_mutex_lock(mutex) +#define __itt_mutex_unlock(mutex) pthread_mutex_unlock(mutex) +#define __itt_load_lib(name) dlopen(name, RTLD_LAZY) +#define __itt_unload_lib(handle) dlclose(handle) +#define __itt_system_error() errno +#define __itt_fstrcmp(s1, s2) strcmp(s1, s2) + +/* makes customer code define safe APIs for SDL_STRNLEN_S and SDL_STRNCPY_S */ +#ifdef SDL_STRNLEN_S +#define __itt_fstrnlen(s, l) SDL_STRNLEN_S(s, l) +#else +#define __itt_fstrnlen(s, l) strlen(s) +#endif /* SDL_STRNLEN_S */ +#ifdef SDL_STRNCPY_S +#define __itt_fstrcpyn(s1, b, s2, l) SDL_STRNCPY_S(s1, b, s2, l) +#else +#define __itt_fstrcpyn(s1, b, s2, l) strncpy(s1, s2, l) +#endif /* SDL_STRNCPY_S */ + +#define __itt_fstrdup(s) strdup(s) +#define __itt_thread_id() pthread_self() +#define __itt_thread_yield() sched_yield() +#if ITT_ARCH==ITT_ARCH_IA64 +#ifdef __INTEL_COMPILER +#define __TBB_machine_fetchadd4(addr, val) __fetchadd4_acq((void *)addr, val) +#else /* __INTEL_COMPILER */ +/* TODO: Add Support for not Intel compilers for IA-64 architecture */ +#endif /* __INTEL_COMPILER */ +#elif ITT_ARCH==ITT_ARCH_IA32 || ITT_ARCH==ITT_ARCH_IA32E /* ITT_ARCH!=ITT_ARCH_IA64 */ +ITT_INLINE long +__TBB_machine_fetchadd4(volatile void* ptr, long addend) ITT_INLINE_ATTRIBUTE; +ITT_INLINE long __TBB_machine_fetchadd4(volatile void* ptr, long addend) +{ + long result; + __asm__ __volatile__("lock\nxadd %0,%1" + : "=r"(result),"=m"(*(int*)ptr) + : "0"(addend), "m"(*(int*)ptr) + : "memory"); + return result; +} +#elif ITT_ARCH==ITT_ARCH_ARM || ITT_ARCH==ITT_ARCH_PPC64 +#define __TBB_machine_fetchadd4(addr, val) __sync_fetch_and_add(addr, val) +#endif /* ITT_ARCH==ITT_ARCH_IA64 */ +#ifndef ITT_SIMPLE_INIT +ITT_INLINE long +__itt_interlocked_increment(volatile long* ptr) ITT_INLINE_ATTRIBUTE; +ITT_INLINE long __itt_interlocked_increment(volatile long* ptr) +{ + return __TBB_machine_fetchadd4(ptr, 1) + 1L; +} +#endif /* ITT_SIMPLE_INIT */ +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ + +typedef enum { + __itt_collection_normal = 0, + __itt_collection_paused = 1 +} __itt_collection_state; + +typedef enum { + __itt_thread_normal = 0, + __itt_thread_ignored = 1 +} __itt_thread_state; + +#pragma pack(push, 8) + +typedef struct ___itt_thread_info +{ + const char* nameA; /*!< Copy of original name in ASCII. */ +#if defined(UNICODE) || defined(_UNICODE) + const wchar_t* nameW; /*!< Copy of original name in UNICODE. */ +#else /* UNICODE || _UNICODE */ + void* nameW; +#endif /* UNICODE || _UNICODE */ + TIDT tid; + __itt_thread_state state; /*!< Thread state (paused or normal) */ + int extra1; /*!< Reserved to the runtime */ + void* extra2; /*!< Reserved to the runtime */ + struct ___itt_thread_info* next; +} __itt_thread_info; + +#include "ittnotify_types.h" /* For __itt_group_id definition */ + +typedef struct ___itt_api_info_20101001 +{ + const char* name; + void** func_ptr; + void* init_func; + __itt_group_id group; +} __itt_api_info_20101001; + +typedef struct ___itt_api_info +{ + const char* name; + void** func_ptr; + void* init_func; + void* null_func; + __itt_group_id group; +} __itt_api_info; + +typedef struct __itt_counter_info +{ + const char* nameA; /*!< Copy of original name in ASCII. */ +#if defined(UNICODE) || defined(_UNICODE) + const wchar_t* nameW; /*!< Copy of original name in UNICODE. */ +#else /* UNICODE || _UNICODE */ + void* nameW; +#endif /* UNICODE || _UNICODE */ + const char* domainA; /*!< Copy of original name in ASCII. */ +#if defined(UNICODE) || defined(_UNICODE) + const wchar_t* domainW; /*!< Copy of original name in UNICODE. */ +#else /* UNICODE || _UNICODE */ + void* domainW; +#endif /* UNICODE || _UNICODE */ + int type; + long index; + int extra1; /*!< Reserved to the runtime */ + void* extra2; /*!< Reserved to the runtime */ + struct __itt_counter_info* next; +} __itt_counter_info_t; + +struct ___itt_domain; +struct ___itt_string_handle; + +typedef struct ___itt_global +{ + unsigned char magic[8]; + unsigned long version_major; + unsigned long version_minor; + unsigned long version_build; + volatile long api_initialized; + volatile long mutex_initialized; + volatile long atomic_counter; + mutex_t mutex; + lib_t lib; + void* error_handler; + const char** dll_path_ptr; + __itt_api_info* api_list_ptr; + struct ___itt_global* next; + /* Joinable structures below */ + __itt_thread_info* thread_list; + struct ___itt_domain* domain_list; + struct ___itt_string_handle* string_list; + __itt_collection_state state; + __itt_counter_info_t* counter_list; +} __itt_global; + +#pragma pack(pop) + +#define NEW_THREAD_INFO_W(gptr,h,h_tail,t,s,n) { \ + h = (__itt_thread_info*)malloc(sizeof(__itt_thread_info)); \ + if (h != NULL) { \ + h->tid = t; \ + h->nameA = NULL; \ + h->nameW = n ? _wcsdup(n) : NULL; \ + h->state = s; \ + h->extra1 = 0; /* reserved */ \ + h->extra2 = NULL; /* reserved */ \ + h->next = NULL; \ + if (h_tail == NULL) \ + (gptr)->thread_list = h; \ + else \ + h_tail->next = h; \ + } \ +} + +#define NEW_THREAD_INFO_A(gptr,h,h_tail,t,s,n) { \ + h = (__itt_thread_info*)malloc(sizeof(__itt_thread_info)); \ + if (h != NULL) { \ + h->tid = t; \ + h->nameA = n ? __itt_fstrdup(n) : NULL; \ + h->nameW = NULL; \ + h->state = s; \ + h->extra1 = 0; /* reserved */ \ + h->extra2 = NULL; /* reserved */ \ + h->next = NULL; \ + if (h_tail == NULL) \ + (gptr)->thread_list = h; \ + else \ + h_tail->next = h; \ + } \ +} + +#define NEW_DOMAIN_W(gptr,h,h_tail,name) { \ + h = (__itt_domain*)malloc(sizeof(__itt_domain)); \ + if (h != NULL) { \ + h->flags = 1; /* domain is enabled by default */ \ + h->nameA = NULL; \ + h->nameW = name ? _wcsdup(name) : NULL; \ + h->extra1 = 0; /* reserved */ \ + h->extra2 = NULL; /* reserved */ \ + h->next = NULL; \ + if (h_tail == NULL) \ + (gptr)->domain_list = h; \ + else \ + h_tail->next = h; \ + } \ +} + +#define NEW_DOMAIN_A(gptr,h,h_tail,name) { \ + h = (__itt_domain*)malloc(sizeof(__itt_domain)); \ + if (h != NULL) { \ + h->flags = 1; /* domain is enabled by default */ \ + h->nameA = name ? __itt_fstrdup(name) : NULL; \ + h->nameW = NULL; \ + h->extra1 = 0; /* reserved */ \ + h->extra2 = NULL; /* reserved */ \ + h->next = NULL; \ + if (h_tail == NULL) \ + (gptr)->domain_list = h; \ + else \ + h_tail->next = h; \ + } \ +} + +#define NEW_STRING_HANDLE_W(gptr,h,h_tail,name) { \ + h = (__itt_string_handle*)malloc(sizeof(__itt_string_handle)); \ + if (h != NULL) { \ + h->strA = NULL; \ + h->strW = name ? _wcsdup(name) : NULL; \ + h->extra1 = 0; /* reserved */ \ + h->extra2 = NULL; /* reserved */ \ + h->next = NULL; \ + if (h_tail == NULL) \ + (gptr)->string_list = h; \ + else \ + h_tail->next = h; \ + } \ +} + +#define NEW_STRING_HANDLE_A(gptr,h,h_tail,name) { \ + h = (__itt_string_handle*)malloc(sizeof(__itt_string_handle)); \ + if (h != NULL) { \ + h->strA = name ? __itt_fstrdup(name) : NULL; \ + h->strW = NULL; \ + h->extra1 = 0; /* reserved */ \ + h->extra2 = NULL; /* reserved */ \ + h->next = NULL; \ + if (h_tail == NULL) \ + (gptr)->string_list = h; \ + else \ + h_tail->next = h; \ + } \ +} + +#define NEW_COUNTER_W(gptr,h,h_tail,name,domain,type) { \ + h = (__itt_counter_info_t*)malloc(sizeof(__itt_counter_info_t)); \ + if (h != NULL) { \ + h->nameA = NULL; \ + h->nameW = name ? _wcsdup(name) : NULL; \ + h->domainA = NULL; \ + h->domainW = name ? _wcsdup(domain) : NULL; \ + h->type = type; \ + h->index = 0; \ + h->next = NULL; \ + if (h_tail == NULL) \ + (gptr)->counter_list = h; \ + else \ + h_tail->next = h; \ + } \ +} + +#define NEW_COUNTER_A(gptr,h,h_tail,name,domain,type) { \ + h = (__itt_counter_info_t*)malloc(sizeof(__itt_counter_info_t)); \ + if (h != NULL) { \ + h->nameA = name ? __itt_fstrdup(name) : NULL; \ + h->nameW = NULL; \ + h->domainA = domain ? __itt_fstrdup(domain) : NULL; \ + h->domainW = NULL; \ + h->type = type; \ + h->index = 0; \ + h->next = NULL; \ + if (h_tail == NULL) \ + (gptr)->counter_list = h; \ + else \ + h_tail->next = h; \ + } \ +} + +#endif /* _ITTNOTIFY_CONFIG_H_ */ diff --git a/ext/opcache/jit/vtune/ittnotify_types.h b/ext/opcache/jit/vtune/ittnotify_types.h new file mode 100644 index 0000000000000..e10250bd63d4f --- /dev/null +++ b/ext/opcache/jit/vtune/ittnotify_types.h @@ -0,0 +1,115 @@ +/* + This file is provided under a dual BSD/GPLv2 license. When using or + redistributing this file, you may do so under either license. + + GPL LICENSE SUMMARY + + Copyright (c) 2005-2014 Intel Corporation. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. + The full GNU General Public License is included in this distribution + in the file called LICENSE.GPL. + + Contact Information: + http://software.intel.com/en-us/articles/intel-vtune-amplifier-xe/ + + BSD LICENSE + + Copyright (c) 2005-2014 Intel Corporation. All rights reserved. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _ITTNOTIFY_TYPES_H_ +#define _ITTNOTIFY_TYPES_H_ + +typedef enum ___itt_group_id +{ + __itt_group_none = 0, + __itt_group_legacy = 1<<0, + __itt_group_control = 1<<1, + __itt_group_thread = 1<<2, + __itt_group_mark = 1<<3, + __itt_group_sync = 1<<4, + __itt_group_fsync = 1<<5, + __itt_group_jit = 1<<6, + __itt_group_model = 1<<7, + __itt_group_splitter_min = 1<<7, + __itt_group_counter = 1<<8, + __itt_group_frame = 1<<9, + __itt_group_stitch = 1<<10, + __itt_group_heap = 1<<11, + __itt_group_splitter_max = 1<<12, + __itt_group_structure = 1<<12, + __itt_group_suppress = 1<<13, + __itt_group_arrays = 1<<14, + __itt_group_all = -1 +} __itt_group_id; + +#pragma pack(push, 8) + +typedef struct ___itt_group_list +{ + __itt_group_id id; + const char* name; +} __itt_group_list; + +#pragma pack(pop) + +#define ITT_GROUP_LIST(varname) \ + static __itt_group_list varname[] = { \ + { __itt_group_all, "all" }, \ + { __itt_group_control, "control" }, \ + { __itt_group_thread, "thread" }, \ + { __itt_group_mark, "mark" }, \ + { __itt_group_sync, "sync" }, \ + { __itt_group_fsync, "fsync" }, \ + { __itt_group_jit, "jit" }, \ + { __itt_group_model, "model" }, \ + { __itt_group_counter, "counter" }, \ + { __itt_group_frame, "frame" }, \ + { __itt_group_stitch, "stitch" }, \ + { __itt_group_heap, "heap" }, \ + { __itt_group_structure, "structure" }, \ + { __itt_group_suppress, "suppress" }, \ + { __itt_group_arrays, "arrays" }, \ + { __itt_group_none, NULL } \ + } + +#endif /* _ITTNOTIFY_TYPES_H_ */ diff --git a/ext/opcache/jit/vtune/jitprofiling.c b/ext/opcache/jit/vtune/jitprofiling.c new file mode 100644 index 0000000000000..1b2f04b076a27 --- /dev/null +++ b/ext/opcache/jit/vtune/jitprofiling.c @@ -0,0 +1,315 @@ +/* + This file is provided under a dual BSD/GPLv2 license. When using or + redistributing this file, you may do so under either license. + + GPL LICENSE SUMMARY + + Copyright (c) 2005-2014 Intel Corporation. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. + The full GNU General Public License is included in this distribution + in the file called LICENSE.GPL. + + Contact Information: + http://software.intel.com/en-us/articles/intel-vtune-amplifier-xe/ + + BSD LICENSE + + Copyright (c) 2005-2014 Intel Corporation. All rights reserved. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "ittnotify_config.h" + +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#include +#pragma optimize("", off) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#if ITT_PLATFORM != ITT_PLATFORM_MAC && ITT_PLATFORM != ITT_PLATFORM_FREEBSD +#include +#endif +#include + +#include "jitprofiling.h" + +static const char rcsid[] = "\n@(#) $Revision: 463960 $\n"; + +#define DLL_ENVIRONMENT_VAR "VS_PROFILER" + +#ifndef NEW_DLL_ENVIRONMENT_VAR +#if ITT_ARCH==ITT_ARCH_IA32 +#define NEW_DLL_ENVIRONMENT_VAR "INTEL_JIT_PROFILER32" +#else +#define NEW_DLL_ENVIRONMENT_VAR "INTEL_JIT_PROFILER64" +#endif +#endif /* NEW_DLL_ENVIRONMENT_VAR */ + +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define DEFAULT_DLLNAME "JitPI.dll" +HINSTANCE m_libHandle = NULL; +#elif ITT_PLATFORM==ITT_PLATFORM_MAC +#define DEFAULT_DLLNAME "libJitPI.dylib" +void* m_libHandle = NULL; +#else +#define DEFAULT_DLLNAME "libJitPI.so" +void* m_libHandle = NULL; +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ + +/* default location of JIT profiling agent on Android */ +#define ANDROID_JIT_AGENT_PATH "/data/intel/libittnotify.so" + +/* the function pointers */ +typedef unsigned int(JITAPI *TPInitialize)(void); +static TPInitialize FUNC_Initialize=NULL; + +typedef unsigned int(JITAPI *TPNotify)(unsigned int, void*); +static TPNotify FUNC_NotifyEvent=NULL; + +static iJIT_IsProfilingActiveFlags executionMode = iJIT_NOTHING_RUNNING; + +/* end collector dll part. */ + +/* loadiJIT_Funcs() : this function is called just in the beginning + * and is responsible to load the functions from BistroJavaCollector.dll + * result: + * on success: the functions loads, iJIT_DLL_is_missing=0, return value = 1 + * on failure: the functions are NULL, iJIT_DLL_is_missing=1, return value = 0 + */ +static int loadiJIT_Funcs(void); + +/* global representing whether the collector can't be loaded */ +static int iJIT_DLL_is_missing = 0; + +ITT_EXTERN_C int JITAPI +iJIT_NotifyEvent(iJIT_JVM_EVENT event_type, void *EventSpecificData) +{ + int ReturnValue; + + /* initialization part - the collector has not been loaded yet. */ + if (!FUNC_NotifyEvent) + { + if (iJIT_DLL_is_missing) + return 0; + + if (!loadiJIT_Funcs()) + return 0; + } + + if (event_type == iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED || + event_type == iJVM_EVENT_TYPE_METHOD_UPDATE) + { + if (((piJIT_Method_Load)EventSpecificData)->method_id == 0) + return 0; + } + else if (event_type == iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED_V2) + { + if (((piJIT_Method_Load_V2)EventSpecificData)->method_id == 0) + return 0; + } + else if (event_type == iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED_V3) + { + if (((piJIT_Method_Load_V3)EventSpecificData)->method_id == 0) + return 0; + } + else if (event_type == iJVM_EVENT_TYPE_METHOD_INLINE_LOAD_FINISHED) + { + if (((piJIT_Method_Inline_Load)EventSpecificData)->method_id == 0 || + ((piJIT_Method_Inline_Load)EventSpecificData)->parent_method_id == 0) + return 0; + } + + ReturnValue = (int)FUNC_NotifyEvent(event_type, EventSpecificData); + + return ReturnValue; +} + +ITT_EXTERN_C iJIT_IsProfilingActiveFlags JITAPI iJIT_IsProfilingActive() +{ + if (!iJIT_DLL_is_missing) + { + loadiJIT_Funcs(); + } + + return executionMode; +} + +/* This function loads the collector dll and the relevant functions. + * on success: all functions load, iJIT_DLL_is_missing = 0, return value = 1 + * on failure: all functions are NULL, iJIT_DLL_is_missing = 1, return value = 0 + */ +static int loadiJIT_Funcs() +{ + static int bDllWasLoaded = 0; + char *dllName = (char*)rcsid; /* !! Just to avoid unused code elimination */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN + DWORD dNameLength = 0; +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ + + if(bDllWasLoaded) + { + /* dll was already loaded, no need to do it for the second time */ + return 1; + } + + /* Assumes that the DLL will not be found */ + iJIT_DLL_is_missing = 1; + FUNC_NotifyEvent = NULL; + + if (m_libHandle) + { +#if ITT_PLATFORM==ITT_PLATFORM_WIN + FreeLibrary(m_libHandle); +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ + dlclose(m_libHandle); +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ + m_libHandle = NULL; + } + + /* Try to get the dll name from the environment */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN + dNameLength = GetEnvironmentVariableA(NEW_DLL_ENVIRONMENT_VAR, NULL, 0); + if (dNameLength) + { + DWORD envret = 0; + dllName = (char*)malloc(sizeof(char) * (dNameLength + 1)); + if(dllName != NULL) + { + envret = GetEnvironmentVariableA(NEW_DLL_ENVIRONMENT_VAR, + dllName, dNameLength); + if (envret) + { + /* Try to load the dll from the PATH... */ + m_libHandle = LoadLibraryExA(dllName, + NULL, LOAD_WITH_ALTERED_SEARCH_PATH); + } + free(dllName); + } + } else { + /* Try to use old VS_PROFILER variable */ + dNameLength = GetEnvironmentVariableA(DLL_ENVIRONMENT_VAR, NULL, 0); + if (dNameLength) + { + DWORD envret = 0; + dllName = (char*)malloc(sizeof(char) * (dNameLength + 1)); + if(dllName != NULL) + { + envret = GetEnvironmentVariableA(DLL_ENVIRONMENT_VAR, + dllName, dNameLength); + if (envret) + { + /* Try to load the dll from the PATH... */ + m_libHandle = LoadLibraryA(dllName); + } + free(dllName); + } + } + } +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ + dllName = getenv(NEW_DLL_ENVIRONMENT_VAR); + if (!dllName) + dllName = getenv(DLL_ENVIRONMENT_VAR); +#if defined(__ANDROID__) || defined(ANDROID) + if (!dllName) + dllName = ANDROID_JIT_AGENT_PATH; +#endif + if (dllName) + { + /* Try to load the dll from the PATH... */ + m_libHandle = dlopen(dllName, RTLD_LAZY); + } +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ + + if (!m_libHandle) + { +#if ITT_PLATFORM==ITT_PLATFORM_WIN + m_libHandle = LoadLibraryA(DEFAULT_DLLNAME); +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ + m_libHandle = dlopen(DEFAULT_DLLNAME, RTLD_LAZY); +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ + } + + /* if the dll wasn't loaded - exit. */ + if (!m_libHandle) + { + iJIT_DLL_is_missing = 1; /* don't try to initialize + * JIT agent the second time + */ + return 0; + } + +#if ITT_PLATFORM==ITT_PLATFORM_WIN + FUNC_NotifyEvent = (TPNotify)GetProcAddress(m_libHandle, "NotifyEvent"); +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ + FUNC_NotifyEvent = (TPNotify)dlsym(m_libHandle, "NotifyEvent"); +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ + if (!FUNC_NotifyEvent) + { + FUNC_Initialize = NULL; + return 0; + } + +#if ITT_PLATFORM==ITT_PLATFORM_WIN + FUNC_Initialize = (TPInitialize)GetProcAddress(m_libHandle, "Initialize"); +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ + FUNC_Initialize = (TPInitialize)dlsym(m_libHandle, "Initialize"); +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ + if (!FUNC_Initialize) + { + FUNC_NotifyEvent = NULL; + return 0; + } + + executionMode = (iJIT_IsProfilingActiveFlags)FUNC_Initialize(); + + bDllWasLoaded = 1; + iJIT_DLL_is_missing = 0; /* DLL is ok. */ + + return 1; +} + +ITT_EXTERN_C unsigned int JITAPI iJIT_GetNewMethodID() +{ + static unsigned int methodID = 1; + + if (methodID == 0) + return 0; /* ERROR : this is not a valid value */ + + return methodID++; +} diff --git a/ext/opcache/jit/vtune/jitprofiling.h b/ext/opcache/jit/vtune/jitprofiling.h new file mode 100644 index 0000000000000..3e166c04d4359 --- /dev/null +++ b/ext/opcache/jit/vtune/jitprofiling.h @@ -0,0 +1,694 @@ +/* + This file is provided under a dual BSD/GPLv2 license. When using or + redistributing this file, you may do so under either license. + + GPL LICENSE SUMMARY + + Copyright (c) 2005-2014 Intel Corporation. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. + The full GNU General Public License is included in this distribution + in the file called LICENSE.GPL. + + Contact Information: + http://software.intel.com/en-us/articles/intel-vtune-amplifier-xe/ + + BSD LICENSE + + Copyright (c) 2005-2014 Intel Corporation. All rights reserved. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __JITPROFILING_H__ +#define __JITPROFILING_H__ + +/** + * @brief JIT Profiling APIs + * + * The JIT Profiling API is used to report information about just-in-time + * generated code that can be used by performance tools. The user inserts + * calls in the code generator to report information before JIT-compiled + * code goes to execution. This information is collected at runtime and used + * by tools like Intel(R) VTune(TM) Amplifier to display performance metrics + * associated with JIT-compiled code. + * + * These APIs can be used to\n + * - **Profile trace-based and method-based JIT-compiled + * code**. Some examples of environments that you can profile with these APIs: + * dynamic JIT compilation of JavaScript code traces, JIT execution in OpenCL(TM) + * software technology, Java/.NET managed execution environments, and custom + * ISV JIT engines. + * @code + * #include + * + * if (iJIT_IsProfilingActive != iJIT_SAMPLING_ON) { + * return; + * } + * + * iJIT_Method_Load jmethod = {0}; + * jmethod.method_id = iJIT_GetNewMethodID(); + * jmethod.method_name = "method_name"; + * jmethod.class_file_name = "class_name"; + * jmethod.source_file_name = "source_file_name"; + * jmethod.method_load_address = code_addr; + * jmethod.method_size = code_size; + * + * iJIT_NotifyEvent(iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED, (void*)&jmethod); + * iJIT_NotifyEvent(iJVM_EVENT_TYPE_SHUTDOWN, NULL); + * @endcode + * + * * Expected behavior: + * * If any iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED event overwrites an + * already reported method, then such a method becomes invalid and its + * memory region is treated as unloaded. VTune Amplifier displays the metrics + * collected by the method until it is overwritten. + * * If supplied line number information contains multiple source lines for + * the same assembly instruction (code location), then VTune Amplifier picks up + * the first line number. + * * Dynamically generated code can be associated with a module name. + * Use the iJIT_Method_Load_V2 structure.\n + * Clarification of some cases: + * * If you register a function with the same method ID multiple times, + * specifying different module names, then the VTune Amplifier picks up + * the module name registered first. If you want to distinguish the same + * function between different JIT engines, supply different method IDs for + * each function. Other symbolic information (for example, source file) + * can be identical. + * + * - **Analyze split functions** (multiple joint or disjoint code regions + * belonging to the same function) **including re-JIT** + * with potential overlapping of code regions in time, which is common in + * resource-limited environments. + * @code + * #include + * + * unsigned int method_id = iJIT_GetNewMethodID(); + * + * iJIT_Method_Load a = {0}; + * a.method_id = method_id; + * a.method_load_address = 0x100; + * a.method_size = 0x20; + * + * iJIT_Method_Load b = {0}; + * b.method_id = method_id; + * b.method_load_address = 0x200; + * b.method_size = 0x30; + * + * iJIT_NotifyEvent(iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED, (void*)&a); + * iJIT_NotifyEvent(iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED, (void*)&b); + * @endcode + * + * * Expected behaviour: + * * If a iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED event overwrites an + * already reported method, then such a method becomes invalid and + * its memory region is treated as unloaded. + * * All code regions reported with the same method ID are considered as + * belonging to the same method. Symbolic information (method name, + * source file name) will be taken from the first notification, and all + * subsequent notifications with the same method ID will be processed + * only for line number table information. So, the VTune Amplifier will map + * samples to a source line using the line number table from the current + * notification while taking the source file name from the very first one.\n + * Clarification of some cases:\n + * * If you register a second code region with a different source file + * name and the same method ID, then this information will be saved and + * will not be considered as an extension of the first code region, but + * VTune Amplifier will use the source file of the first code region and map + * performance metrics incorrectly. + * * If you register a second code region with the same source file as + * for the first region and the same method ID, then the source file will be + * discarded but VTune Amplifier will map metrics to the source file correctly. + * * If you register a second code region with a null source file and + * the same method ID, then provided line number info will be associated + * with the source file of the first code region. + * + * - **Explore inline functions** including multi-level hierarchy of + * nested inline methods which shows how performance metrics are distributed through them. + * @code + * #include + * + * // method_id parent_id + * // [-- c --] 3000 2000 + * // [---- d -----] 2001 1000 + * // [---- b ----] 2000 1000 + * // [------------ a ----------------] 1000 n/a + * + * iJIT_Method_Load a = {0}; + * a.method_id = 1000; + * + * iJIT_Method_Inline_Load b = {0}; + * b.method_id = 2000; + * b.parent_method_id = 1000; + * + * iJIT_Method_Inline_Load c = {0}; + * c.method_id = 3000; + * c.parent_method_id = 2000; + * + * iJIT_Method_Inline_Load d = {0}; + * d.method_id = 2001; + * d.parent_method_id = 1000; + * + * iJIT_NotifyEvent(iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED, (void*)&a); + * iJIT_NotifyEvent(iJVM_EVENT_TYPE_METHOD_INLINE_LOAD_FINISHED, (void*)&b); + * iJIT_NotifyEvent(iJVM_EVENT_TYPE_METHOD_INLINE_LOAD_FINISHED, (void*)&c); + * iJIT_NotifyEvent(iJVM_EVENT_TYPE_METHOD_INLINE_LOAD_FINISHED, (void*)&d); + * @endcode + * + * * Requirements: + * * Each inline (iJIT_Method_Inline_Load) method should be associated + * with two method IDs: one for itself; one for its immediate parent. + * * Address regions of inline methods of the same parent method cannot + * overlap each other. + * * Execution of the parent method must not be started until it and all + * its inline methods are reported. + * * Expected behaviour: + * * In case of nested inline methods an order of + * iJVM_EVENT_TYPE_METHOD_INLINE_LOAD_FINISHED events is not important. + * * If any event overwrites either inline method or top parent method, + * then the parent, including inline methods, becomes invalid and its memory + * region is treated as unloaded. + * + * **Life time of allocated data**\n + * The client sends an event notification to the agent with event-specific + * data, which is a structure. The pointers in the structure refer to memory + * allocated by the client, which responsible for releasing it. The pointers are + * used by the iJIT_NotifyEvent method to copy client's data in a trace file, + * and they are not used after the iJIT_NotifyEvent method returns. + */ + +/** + * @defgroup jitapi JIT Profiling + * @ingroup internal + * @{ + */ + +/** + * @brief Enumerator for the types of notifications + */ +typedef enum iJIT_jvm_event +{ + iJVM_EVENT_TYPE_SHUTDOWN = 2, /**<\brief Send this to shutdown the agent. + * Use NULL for event data. */ + + iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED = 13, /**<\brief Send when dynamic code is + * JIT compiled and loaded into + * memory by the JIT engine, but + * before the code is executed. + * Use iJIT_Method_Load as event + * data. */ +/** @cond exclude_from_documentation */ + iJVM_EVENT_TYPE_METHOD_UNLOAD_START, /**<\brief Send when compiled dynamic + * code is being unloaded from memory. + * Use iJIT_Method_Load as event data.*/ +/** @endcond */ + + iJVM_EVENT_TYPE_METHOD_UPDATE, /**<\brief Send to provide new content for + * a previously reported dynamic code. + * The previous content will be invalidated + * starting from the time of the notification. + * Use iJIT_Method_Load as event data but + * required fields are following: + * - method_id identify the code to update. + * - method_load_address specify start address + * within identified code range + * where update should be started. + * - method_size specify length of updated code + * range. */ + + + iJVM_EVENT_TYPE_METHOD_INLINE_LOAD_FINISHED, /**<\brief Send when an inline dynamic + * code is JIT compiled and loaded + * into memory by the JIT engine, + * but before the parent code region + * starts executing. + * Use iJIT_Method_Inline_Load as event data.*/ + +/** @cond exclude_from_documentation */ + iJVM_EVENT_TYPE_METHOD_UPDATE_V2, +/** @endcond */ + + iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED_V2 = 21, /**<\brief Send when a dynamic code is + * JIT compiled and loaded into + * memory by the JIT engine, but + * before the code is executed. + * Use iJIT_Method_Load_V2 as event data. */ + + iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED_V3 /**<\brief Send when a dynamic code is + * JIT compiled and loaded into + * memory by the JIT engine, but + * before the code is executed. + * Use iJIT_Method_Load_V3 as event data. */ +} iJIT_JVM_EVENT; + +/** + * @brief Enumerator for the agent's mode + */ +typedef enum _iJIT_IsProfilingActiveFlags +{ + iJIT_NOTHING_RUNNING = 0x0000, /**<\brief The agent is not running; + * iJIT_NotifyEvent calls will + * not be processed. */ + iJIT_SAMPLING_ON = 0x0001, /**<\brief The agent is running and + * ready to process notifications. */ +} iJIT_IsProfilingActiveFlags; + +/** + * @brief Description of a single entry in the line number information of a code region. + * @details A table of line number entries gives information about how the reported code region + * is mapped to source file. + * Intel(R) VTune(TM) Amplifier uses line number information to attribute + * the samples (virtual address) to a line number. \n + * It is acceptable to report different code addresses for the same source line: + * @code + * Offset LineNumber + * 1 2 + * 12 4 + * 15 2 + * 18 1 + * 21 30 + * + * VTune Amplifier constructs the following table using the client data + * + * Code subrange Line number + * 0-1 2 + * 1-12 4 + * 12-15 2 + * 15-18 1 + * 18-21 30 + * @endcode + */ +typedef struct _LineNumberInfo +{ + unsigned int Offset; /**<\brief Offset from the begining of the code region. */ + unsigned int LineNumber; /**<\brief Matching source line number offset (from beginning of source file). */ + +} *pLineNumberInfo, LineNumberInfo; + +/** + * @brief Enumerator for the code architecture. + */ +typedef enum _iJIT_CodeArchitecture +{ + iJIT_CA_NATIVE = 0, /**<\brief Native to the process architecture that is calling it. */ + + iJIT_CA_32, /**<\brief 32-bit machine code. */ + + iJIT_CA_64 /**<\brief 64-bit machine code. */ + +} iJIT_CodeArchitecture; + +#pragma pack(push, 8) + +/** + * @brief Description of a JIT-compiled method + * @details When you use the iJIT_Method_Load structure to describe + * the JIT compiled method, use iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED + * as an event type to report it. + */ +typedef struct _iJIT_Method_Load +{ + unsigned int method_id; /**<\brief Unique method ID. Cannot be 0. + * You must either use the API function + * iJIT_GetNewMethodID to get a valid and unique + * method ID, or else manage ID uniqueness + * and correct range by yourself.\n + * You must use the same method ID for all code + * regions of the same method, otherwise different + * method IDs specify different methods. */ + + char* method_name; /**<\brief The name of the method. It can be optionally + * prefixed with its class name and appended with + * its complete signature. Can't be NULL. */ + + void* method_load_address; /**<\brief The start virtual address of the method code + * region. If NULL, data provided with + * event are not accepted. */ + + unsigned int method_size; /**<\brief The code size of the method in memory. + * If 0, then data provided with the event are not + * accepted. */ + + unsigned int line_number_size; /**<\brief The number of entries in the line number + * table.0 if none. */ + + pLineNumberInfo line_number_table; /**<\brief Pointer to the line numbers info + * array. Can be NULL if + * line_number_size is 0. See + * LineNumberInfo Structure for a + * description of a single entry in + * the line number info array */ + + unsigned int class_id; /**<\brief This field is obsolete. */ + + char* class_file_name; /**<\brief Class name. Can be NULL.*/ + + char* source_file_name; /**<\brief Source file name. Can be NULL.*/ + +} *piJIT_Method_Load, iJIT_Method_Load; + +/** + * @brief Description of a JIT-compiled method + * @details When you use the iJIT_Method_Load_V2 structure to describe + * the JIT compiled method, use iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED_V2 + * as an event type to report it. + */ +typedef struct _iJIT_Method_Load_V2 +{ + unsigned int method_id; /**<\brief Unique method ID. Cannot be 0. + * You must either use the API function + * iJIT_GetNewMethodID to get a valid and unique + * method ID, or else manage ID uniqueness + * and correct range by yourself.\n + * You must use the same method ID for all code + * regions of the same method, otherwise different + * method IDs specify different methods. */ + + char* method_name; /**<\brief The name of the method. It can be optionally + * prefixed with its class name and appended with + * its complete signature. Can't be NULL. */ + + void* method_load_address; /**<\brief The start virtual address of the method code + * region. If NULL, then data provided with the + * event are not accepted. */ + + unsigned int method_size; /**<\brief The code size of the method in memory. + * If 0, then data provided with the event are not + * accepted. */ + + unsigned int line_number_size; /**<\brief The number of entries in the line number + * table. 0 if none. */ + + pLineNumberInfo line_number_table; /**<\brief Pointer to the line numbers info + * array. Can be NULL if + * line_number_size is 0. See + * LineNumberInfo Structure for a + * description of a single entry in + * the line number info array. */ + + char* class_file_name; /**<\brief Class name. Can be NULL. */ + + char* source_file_name; /**<\brief Source file name. Can be NULL. */ + + char* module_name; /**<\brief Module name. Can be NULL. + The module name can be useful for distinguishing among + different JIT engines. VTune Amplifier will display + reported methods grouped by specific module. */ + +} *piJIT_Method_Load_V2, iJIT_Method_Load_V2; + +/** + * @brief Description of a JIT-compiled method + * @details The iJIT_Method_Load_V3 structure is the same as iJIT_Method_Load_V2 + * with a newly introduced 'arch' field that specifies architecture of the code region. + * When you use the iJIT_Method_Load_V3 structure to describe + * the JIT compiled method, use iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED_V3 + * as an event type to report it. + */ +typedef struct _iJIT_Method_Load_V3 +{ + unsigned int method_id; /**<\brief Unique method ID. Cannot be 0. + * You must either use the API function + * iJIT_GetNewMethodID to get a valid and unique + * method ID, or manage ID uniqueness + * and correct range by yourself.\n + * You must use the same method ID for all code + * regions of the same method, otherwise they are + * treated as regions of different methods. */ + + char* method_name; /**<\brief The name of the method. It can be optionally + * prefixed with its class name and appended with + * its complete signature. Cannot be NULL. */ + + void* method_load_address; /**<\brief The start virtual address of the method code + * region. If NULL, then data provided with the + * event are not accepted. */ + + unsigned int method_size; /**<\brief The code size of the method in memory. + * If 0, then data provided with the event are not + * accepted. */ + + unsigned int line_number_size; /**<\brief The number of entries in the line number + * table. 0 if none. */ + + pLineNumberInfo line_number_table; /**<\brief Pointer to the line numbers info + * array. Can be NULL if + * line_number_size is 0. See + * LineNumberInfo Structure for a + * description of a single entry in + * the line number info array. */ + + char* class_file_name; /**<\brief Class name. Can be NULL. */ + + char* source_file_name; /**<\brief Source file name. Can be NULL. */ + + char* module_name; /**<\brief Module name. Can be NULL. + * The module name can be useful for distinguishing among + * different JIT engines. VTune Amplifier will display + * reported methods grouped by specific module. */ + + iJIT_CodeArchitecture module_arch; /**<\brief Architecture of the method's code region. + * By default, it is the same as the process + * architecture that is calling it. + * For example, you can use it if your 32-bit JIT + * engine generates 64-bit code. + * + * If JIT engine reports both 32-bit and 64-bit types + * of methods then VTune Amplifier splits the methods + * with the same module name but with different + * architectures in two different modules. VTune Amplifier + * modifies the original name provided with a 64-bit method + * version by ending it with '(64)' */ + +} *piJIT_Method_Load_V3, iJIT_Method_Load_V3; + +/** + * @brief Description of an inline JIT-compiled method + * @details When you use the_iJIT_Method_Inline_Load structure to describe + * the JIT compiled method, use iJVM_EVENT_TYPE_METHOD_INLINE_LOAD_FINISHED + * as an event type to report it. + */ +typedef struct _iJIT_Method_Inline_Load +{ + unsigned int method_id; /**<\brief Unique method ID. Cannot be 0. + * You must either use the API function + * iJIT_GetNewMethodID to get a valid and unique + * method ID, or else manage ID uniqueness + * and correct range by yourself. */ + + unsigned int parent_method_id; /**<\brief Unique immediate parent's method ID. + * Cannot be 0. + * You must either use the API function + * iJIT_GetNewMethodID to get a valid and unique + * method ID, or else manage ID uniqueness + * and correct range by yourself. */ + + char* method_name; /**<\brief The name of the method. It can be optionally + * prefixed with its class name and appended with + * its complete signature. Can't be NULL. */ + + void* method_load_address; /** <\brief The virtual address on which the method + * is inlined. If NULL, then data provided with + * the event are not accepted. */ + + unsigned int method_size; /**<\brief The code size of the method in memory. + * If 0, then data provided with the event are not + * accepted. */ + + unsigned int line_number_size; /**<\brief The number of entries in the line number + * table. 0 if none. */ + + pLineNumberInfo line_number_table; /**<\brief Pointer to the line numbers info + * array. Can be NULL if + * line_number_size is 0. See + * LineNumberInfo Structure for a + * description of a single entry in + * the line number info array */ + + char* class_file_name; /**<\brief Class name. Can be NULL.*/ + + char* source_file_name; /**<\brief Source file name. Can be NULL.*/ + +} *piJIT_Method_Inline_Load, iJIT_Method_Inline_Load; + +/** @cond exclude_from_documentation */ +/** + * @brief Description of a segment type + * @details Use the segment type to specify a type of data supplied + * with the iJVM_EVENT_TYPE_METHOD_UPDATE_V2 event to be applied to + * a certain code trace. + */ +typedef enum _iJIT_SegmentType +{ + iJIT_CT_UNKNOWN = 0, + + iJIT_CT_CODE, /**<\brief Executable code. */ + + iJIT_CT_DATA, /**<\brief Data (not executable code). + * VTune Amplifier uses the format string + * (see iJIT_Method_Update) to represent + * this data in the VTune Amplifier GUI */ + + iJIT_CT_KEEP, /**<\brief Use the previous markup for the trace. + * Can be used for the following + * iJVM_EVENT_TYPE_METHOD_UPDATE_V2 events, + * if the type of the previously reported segment + * type is the same. */ + iJIT_CT_EOF +} iJIT_SegmentType; + +/** + * @brief Description of a dynamic update of the content within JIT-compiled method + * @details The JIT engine may generate the methods that are updated at runtime + * partially by mixed (data + executable code) content. When you use the iJIT_Method_Update + * structure to describe the update of the content within a JIT-compiled method, + * use iJVM_EVENT_TYPE_METHOD_UPDATE_V2 as an event type to report it. + * + * On the first Update event, VTune Amplifier copies the original code range reported by + * the iJVM_EVENT_TYPE_METHOD_LOAD event, then modifies it with the supplied bytes and + * adds the modified range to the original method. For next update events, VTune Amplifier + * does the same but it uses the latest modified version of a code region for update. + * Eventually, VTune Amplifier GUI displays multiple code ranges for the method reported by + * the iJVM_EVENT_TYPE_METHOD_LOAD event. + * Notes: + * - Multiple update events with different types for the same trace are allowed + * but they must be reported for the same code ranges. + * Example, + * @code + * [-- data---] Allowed + * [-- code --] Allowed + * [code] Ignored + * [-- data---] Allowed + * [-- code --] Allowed + * [------------ trace ---------] + * @endcode + * - The types of previously reported events can be changed but they must be reported + * for the same code ranges. + * Example, + * @code + * [-- data---] Allowed + * [-- code --] Allowed + * [-- data---] Allowed + * [-- code --] Allowed + * [------------ trace ---------] + * @endcode + */ + +typedef struct _iJIT_Method_Update +{ + void* load_address; /**<\brief Start address of the update within a method */ + + unsigned int size; /**<\brief The update size */ + + iJIT_SegmentType type; /**<\brief Type of the update */ + + const char* data_format; /**<\brief C string that contains a format string + * that follows the same specifications as format in printf. + * The format string is used for iJIT_CT_CODE only + * and cannot be NULL. + * Format can be changed on the fly. */ +} *piJIT_Method_Update, iJIT_Method_Update; + +/** @endcond */ + +#pragma pack(pop) + +/** @cond exclude_from_documentation */ +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + +#ifndef JITAPI_CDECL +# if defined WIN32 || defined _WIN32 +# define JITAPI_CDECL __cdecl +# else /* defined WIN32 || defined _WIN32 */ +# if defined _M_IX86 || defined __i386__ +# define JITAPI_CDECL __attribute__ ((cdecl)) +# else /* _M_IX86 || __i386__ */ +# define JITAPI_CDECL /* actual only on x86_64 platform */ +# endif /* _M_IX86 || __i386__ */ +# endif /* defined WIN32 || defined _WIN32 */ +#endif /* JITAPI_CDECL */ + +#define JITAPI JITAPI_CDECL +/** @endcond */ + +/** + * @brief Generates a new unique method ID. + * + * You must use this API to obtain unique and valid method IDs for methods or + * traces reported to the agent if you don't have your own mechanism to generate + * unique method IDs. + * + * @return a new unique method ID. When out of unique method IDs, this API + * returns 0, which is not an accepted value. + */ +unsigned int JITAPI iJIT_GetNewMethodID(void); + +/** + * @brief Returns the current mode of the agent. + * + * @return iJIT_SAMPLING_ON, indicating that agent is running, or + * iJIT_NOTHING_RUNNING if no agent is running. + */ +iJIT_IsProfilingActiveFlags JITAPI iJIT_IsProfilingActive(void); + +/** + * @brief Reports infomation about JIT-compiled code to the agent. + * + * The reported information is used to attribute samples obtained from any + * Intel(R) VTune(TM) Amplifier collector. This API needs to be called + * after JIT compilation and before the first entry into the JIT-compiled + * code. + * + * @param[in] event_type - type of the data sent to the agent + * @param[in] EventSpecificData - pointer to event-specific data + * + * @returns 1 on success, otherwise 0. + */ +int JITAPI iJIT_NotifyEvent(iJIT_JVM_EVENT event_type, void *EventSpecificData); + +#ifdef __cplusplus +} +#endif /* __cplusplus */ +/** @endcond */ + +/** @} jitapi group */ + +#endif /* __JITPROFILING_H__ */ diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index e0639720693d8..93e0d6f553b6e 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -81,6 +81,7 @@ static int zend_may_overflow(const zend_op *opline, zend_op_array *op_array, zen #ifdef HAVE_OPROFILE # include "jit/zend_jit_oprofile.c" #endif +#include "jit/zend_jit_vtune.c" #if _WIN32 # include @@ -144,7 +145,7 @@ static void *dasm_link_and_encode(dasm_State **dasm_state, size_t size; int ret; void *entry; -#if defined(HAVE_DISASM) || defined(HAVE_GDB) || defined(HAVE_OPROFILE) || defined(HAVE_PERFTOOLS) +#if defined(HAVE_DISASM) || defined(HAVE_GDB) || defined(HAVE_OPROFILE) || defined(HAVE_PERFTOOLS) || defined(HAVE_VTUNE) zend_string *str = NULL; #endif @@ -186,9 +187,9 @@ static void *dasm_link_and_encode(dasm_State **dasm_state, } } -#if defined(HAVE_DISASM) || defined(HAVE_GDB) || defined(HAVE_OPROFILE) || defined(HAVE_PERFTOOLS) +#if defined(HAVE_DISASM) || defined(HAVE_GDB) || defined(HAVE_OPROFILE) || defined(HAVE_PERFTOOLS) || defined(HAVE_VTUNE) if (!name) { - if (ZCG(accel_directives).jit_debug & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_OPROFILE|ZEND_JIT_DEBUG_PERF)) { + if (ZCG(accel_directives).jit_debug & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_OPROFILE|ZEND_JIT_DEBUG_PERF|ZEND_JIT_DEBUG_VTUNE)) { str = zend_jit_func_name(op_array); if (str) { name = ZSTR_VAL(str); @@ -242,7 +243,18 @@ static void *dasm_link_and_encode(dasm_State **dasm_state, } #endif -#if defined(HAVE_DISASM) || defined(HAVE_GDB) || defined(HAVE_OPROFILE) || defined(HAVE_PERFTOOLS) +#ifdef HAVE_VTUNE + if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_VTUNE) { + if (name) { + zend_jit_vtune_register( + name, + entry, + size); + } + } +#endif + +#if defined(HAVE_DISASM) || defined(HAVE_GDB) || defined(HAVE_OPROFILE) || defined(HAVE_PERFTOOLS) || defined(HAVE_VTUNE) if (str) { zend_string_release(str); } diff --git a/ext/opcache/jit/zend_jit.h b/ext/opcache/jit/zend_jit.h index 5900491e22934..5dc629bd3f7bb 100644 --- a/ext/opcache/jit/zend_jit.h +++ b/ext/opcache/jit/zend_jit.h @@ -35,6 +35,7 @@ #define ZEND_JIT_DEBUG_GDB (1<<4) #define ZEND_JIT_DEBUG_PERF (1<<5) #define ZEND_JIT_DEBUG_OPROFILE (1<<6) +#define ZEND_JIT_DEBUG_VTUNE (1<<7) ZEND_API int zend_jit_op_array(zend_op_array *op_array, zend_script *script); diff --git a/ext/opcache/jit/zend_jit_vtune.c b/ext/opcache/jit/zend_jit_vtune.c new file mode 100644 index 0000000000000..0fa2ce5a25fee --- /dev/null +++ b/ext/opcache/jit/zend_jit_vtune.c @@ -0,0 +1,49 @@ +/* + +----------------------------------------------------------------------+ + | Zend JIT | + +----------------------------------------------------------------------+ + | Copyright (c) 1998-2016 The PHP Group | + +----------------------------------------------------------------------+ + | This source file is subject to version 3.01 of the PHP license, | + | that is bundled with this package in the file LICENSE, and is | + | available through the world-wide-web at the following url: | + | http://www.php.net/license/3_01.txt | + | If you did not receive a copy of the PHP license and are unable to | + | obtain it through the world-wide-web, please send a note to | + | license@php.net so we can mail you a copy immediately. | + +----------------------------------------------------------------------+ + | Authors: Dmitry Stogov | + +----------------------------------------------------------------------+ +*/ + +#define HAVE_VTUNE 1 + +#include "jit/vtune/jitprofiling.h" +#include "jit/vtune/jitprofiling.c" + +static void zend_jit_vtune_register(const char *name, + const void *start, + size_t size) +{ + iJIT_Method_Load jmethod = {0}; + + if (iJIT_IsProfilingActive() != iJIT_SAMPLING_ON) { + return; + } + + jmethod.method_id = iJIT_GetNewMethodID(); + jmethod.method_name = (char*)name; + jmethod.class_file_name = NULL; + jmethod.source_file_name = NULL; + jmethod.method_load_address = (void*)start; + jmethod.method_size = size; + + iJIT_NotifyEvent(iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED, (void*)&jmethod); +} +/* + * Local variables: + * tab-width: 4 + * c-basic-offset: 4 + * indent-tabs-mode: t + * End: + */ From fe383a6111fab5ac002e7139a6360764e285aeef Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 25 Oct 2016 17:23:57 +0300 Subject: [PATCH 315/569] Disable JIT when caching to file-cache. --- ext/opcache/zend_persist.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/ext/opcache/zend_persist.c b/ext/opcache/zend_persist.c index e07cf86069381..2e255dec02d41 100644 --- a/ext/opcache/zend_persist.c +++ b/ext/opcache/zend_persist.c @@ -836,6 +836,10 @@ static void zend_accel_persist_class_table(HashTable *class_table) zend_persistent_script *zend_accel_script_persist(zend_persistent_script *script, char **key, unsigned int key_length) { +#ifdef HAVE_JIT + zend_long orig_jit_level = 0; +#endif + script->mem = ZCG(mem); ZEND_ASSERT(((zend_uintptr_t)ZCG(mem) & 0x7) == 0); /* should be 8 byte aligned */ @@ -859,7 +863,12 @@ zend_persistent_script *zend_accel_script_persist(zend_persistent_script *script #ifdef HAVE_JIT if (ZCG(accel_directives).jit_level) { - zend_jit_unprotect(); + if (key) { + zend_jit_unprotect(); + } else { + orig_jit_level = ZCG(accel_directives).jit_level; + ZCG(accel_directives).jit_level = 0; + } } #endif @@ -875,6 +884,8 @@ zend_persistent_script *zend_accel_script_persist(zend_persistent_script *script zend_jit_script(&script->script); } zend_jit_protect(); + } else if (!key) { + ZCG(accel_directives).jit_level = orig_jit_level; } #endif From 7cb7a0d5ee8fb91b8ab6f117c8b3568c42adb4a6 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 25 Oct 2016 17:51:00 +0300 Subject: [PATCH 316/569] Build and use call-tree, if we JIT function at once (e.g. at run-time) --- ext/opcache/jit/zend_jit.c | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 93e0d6f553b6e..d3dcee274f3b9 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -1368,6 +1368,17 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) return FAILURE; } +static int zend_jit_collect_calls(zend_op_array *op_array, zend_script *script) +{ + zend_func_info *func_info = + zend_arena_calloc(&CG(arena), 1, sizeof(zend_func_info)); + + ZEND_SET_FUNC_INFO(op_array, func_info); + func_info->num_args = -1; + func_info->return_value_used = -1; + return zend_analyze_calls(&CG(arena), script, ZEND_RT_CONSTANTS | ZEND_CALL_TREE, op_array, func_info); +} + static int zend_real_jit_func(zend_op_array *op_array, zend_script *script) { uint32_t flags = 0; @@ -1395,10 +1406,21 @@ static int zend_real_jit_func(zend_op_array *op_array, zend_script *script) zend_dump_op_array(op_array, ZEND_DUMP_HIDE_UNREACHABLE|ZEND_DUMP_RC_INFERENCE|ZEND_DUMP_SSA|ZEND_DUMP_RT_CONSTANTS, "JIT", &ssa); } + if (zend_jit_level >= ZEND_JIT_LEVEL_OPT_SCRIPT) { + if (zend_jit_collect_calls(op_array, script) != SUCCESS) { + ZEND_SET_FUNC_INFO(op_array, NULL); + goto jit_failure; + } + } + if (zend_jit(op_array, &ssa) != SUCCESS) { goto jit_failure; } + if (zend_jit_level >= ZEND_JIT_LEVEL_OPT_SCRIPT) { + ZEND_SET_FUNC_INFO(op_array, NULL); + } + zend_arena_release(&CG(arena), checkpoint); return SUCCESS; From 199f8075933f482d93e42872a406e9980c98e59b Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 26 Oct 2016 00:24:56 +0300 Subject: [PATCH 317/569] Use "opcache.jit" ini directive to define JIT level (by low decimal digit, 1-5 supported) and JIT trigger condition (by second deciaml digit, 0 - at script compilation time, 1 - at run-time, on first function execution). --- ext/opcache/ZendAccelerator.c | 6 +- ext/opcache/ZendAccelerator.h | 2 +- ext/opcache/jit/zend_jit.c | 114 +++++++++++++------------- ext/opcache/jit/zend_jit.h | 14 +++- ext/opcache/jit/zend_jit_x86.dasc | 20 ++++- ext/opcache/zend_accelerator_module.c | 2 +- ext/opcache/zend_persist.c | 18 ++-- 7 files changed, 100 insertions(+), 76 deletions(-) diff --git a/ext/opcache/ZendAccelerator.c b/ext/opcache/ZendAccelerator.c index 63e7a5c093d81..70264765710e1 100644 --- a/ext/opcache/ZendAccelerator.c +++ b/ext/opcache/ZendAccelerator.c @@ -2784,11 +2784,11 @@ static int accel_startup(zend_extension *extension) zend_shared_alloc_lock(); #ifdef HAVE_JIT - if (ZCG(accel_directives).jit_level && + if (ZCG(accel_directives).jit && ZCG(accel_directives).jit_buffer_size) { - zend_jit_startup(ZCG(accel_directives).jit_level, ZCG(accel_directives).jit_buffer_size); + zend_jit_startup(ZCG(accel_directives).jit, ZCG(accel_directives).jit_buffer_size); } else { - ZCG(accel_directives).jit_level = 0; + ZCG(accel_directives).jit = 0; ZCG(accel_directives).jit_buffer_size = 0; } #endif diff --git a/ext/opcache/ZendAccelerator.h b/ext/opcache/ZendAccelerator.h index 184cabbbd76cb..bfd629a9b472e 100644 --- a/ext/opcache/ZendAccelerator.h +++ b/ext/opcache/ZendAccelerator.h @@ -209,7 +209,7 @@ typedef struct _zend_accel_directives { zend_bool huge_code_pages; #endif #ifdef HAVE_JIT - zend_long jit_level; + zend_long jit; zend_long jit_buffer_size; zend_long jit_debug; #endif diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index d3dcee274f3b9..e3d6925cfc0c4 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -17,6 +17,7 @@ */ #include +#include "zend_shared_alloc.h" #include "Zend/zend_execute.h" #include "Zend/zend_vm.h" #include "Zend/zend_exceptions.h" @@ -34,7 +35,6 @@ //#define CONTEXT_THREADED_JIT #define PREFER_MAP_32BIT -//#define ZEND_RUNTIME_JIT //#define ZEND_JIT_RECORD //#define ZEND_JIT_FILTER #define ZEND_JIT_USE_RC_INFERENCE @@ -64,7 +64,8 @@ typedef struct _zend_jit_stub { #define JIT_STUB(name) \ {JIT_STUB_PREFIX #name, zend_jit_ ## name ## _stub} -static zend_long zend_jit_level = 0; +static zend_uchar zend_jit_level = 0; +static zend_uchar zend_jit_trigger = 0; static void *dasm_buf = NULL; static void *dasm_end = NULL; static void **dasm_ptr = NULL; @@ -99,6 +100,7 @@ ZEND_API void zend_jit_status(zval *ret) zval stats; array_init(&stats); add_assoc_long(&stats, "level", zend_jit_level); + add_assoc_long(&stats, "trigger", zend_jit_trigger); if (dasm_buf) { add_assoc_long(&stats, "buffer_size", (char*)dasm_end - (char*)dasm_buf); add_assoc_long(&stats, "buffer_free", (char*)dasm_end - (char*)*dasm_ptr); @@ -1406,7 +1408,7 @@ static int zend_real_jit_func(zend_op_array *op_array, zend_script *script) zend_dump_op_array(op_array, ZEND_DUMP_HIDE_UNREACHABLE|ZEND_DUMP_RC_INFERENCE|ZEND_DUMP_SSA|ZEND_DUMP_RT_CONSTANTS, "JIT", &ssa); } - if (zend_jit_level >= ZEND_JIT_LEVEL_OPT_SCRIPT) { + if (zend_jit_level >= ZEND_JIT_LEVEL_OPT_FUNCS) { if (zend_jit_collect_calls(op_array, script) != SUCCESS) { ZEND_SET_FUNC_INFO(op_array, NULL); goto jit_failure; @@ -1417,7 +1419,7 @@ static int zend_real_jit_func(zend_op_array *op_array, zend_script *script) goto jit_failure; } - if (zend_jit_level >= ZEND_JIT_LEVEL_OPT_SCRIPT) { + if (zend_jit_level >= ZEND_JIT_LEVEL_OPT_FUNCS) { ZEND_SET_FUNC_INFO(op_array, NULL); } @@ -1429,18 +1431,14 @@ static int zend_real_jit_func(zend_op_array *op_array, zend_script *script) return FAILURE; } - -#ifdef ZEND_RUNTIME_JIT - /* memory write protection */ -void zend_accel_shared_protect(int mode); - #define SHM_PROTECT() \ do { \ if (ZCG(accel_directives).protect_memory) { \ zend_accel_shared_protect(1); \ } \ } while (0) + #define SHM_UNPROTECT() \ do { \ if (ZCG(accel_directives).protect_memory) { \ @@ -1476,24 +1474,25 @@ static void ZEND_FASTCALL zend_runtime_jit(void) /* JIT-ed code is going to be called by VM */ } -#endif ZEND_API int zend_jit_op_array(zend_op_array *op_array, zend_script *script) { -#ifdef ZEND_RUNTIME_JIT - zend_op *opline = op_array->opcodes; + if (zend_jit_trigger == ZEND_JIT_ON_FIRST_EXEC) { + zend_op *opline = op_array->opcodes; - /* Set run-time JIT handler */ - while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) { + /* Set run-time JIT handler */ + while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) { + opline->handler = (const void*)zend_runtime_jit; + opline++; + } + ZEND_SET_FUNC_INFO(op_array, (void*)opline->handler); opline->handler = (const void*)zend_runtime_jit; - opline++; + return SUCCESS; + } else if (zend_jit_trigger == ZEND_JIT_ON_SCRIPT_LOAD) { + return zend_real_jit_func(op_array, script); + } else { + ZEND_ASSERT(0); } - ZEND_SET_FUNC_INFO(op_array, (void*)opline->handler); - opline->handler = (const void*)zend_runtime_jit; - return SUCCESS; -#else - return zend_real_jit_func(op_array, script); -#endif } ZEND_API int zend_jit_script(zend_script *script) @@ -1514,54 +1513,56 @@ ZEND_API int zend_jit_script(zend_script *script) goto jit_failure; } -#ifdef ZEND_RUNTIME_JIT - for (i = 0; i < call_graph.op_arrays_count; i++) { - ZEND_SET_FUNC_INFO(call_graph.op_arrays[i], NULL); - if (zend_jit_op_array(call_graph.op_arrays[i], script) != SUCCESS) { - goto jit_failure; - } - } -#else - for (i = 0; i < call_graph.op_arrays_count; i++) { - info = ZEND_FUNC_INFO(call_graph.op_arrays[i]); - if (info) { - if (zend_jit_op_array_analyze1(call_graph.op_arrays[i], script, &info->ssa, &info->flags) != SUCCESS) { + if (zend_jit_trigger == ZEND_JIT_ON_FIRST_EXEC) { + for (i = 0; i < call_graph.op_arrays_count; i++) { + ZEND_SET_FUNC_INFO(call_graph.op_arrays[i], NULL); + if (zend_jit_op_array(call_graph.op_arrays[i], script) != SUCCESS) { goto jit_failure; } } - } - - for (i = 0; i < call_graph.op_arrays_count; i++) { - info = ZEND_FUNC_INFO(call_graph.op_arrays[i]); - if (info) { - if (zend_jit_op_array_analyze2(call_graph.op_arrays[i], script, &info->ssa, &info->flags) != SUCCESS) { - goto jit_failure; + } else if (zend_jit_trigger == ZEND_JIT_ON_SCRIPT_LOAD) { + for (i = 0; i < call_graph.op_arrays_count; i++) { + info = ZEND_FUNC_INFO(call_graph.op_arrays[i]); + if (info) { + if (zend_jit_op_array_analyze1(call_graph.op_arrays[i], script, &info->ssa, &info->flags) != SUCCESS) { + goto jit_failure; + } } } - } - if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_SSA) { for (i = 0; i < call_graph.op_arrays_count; i++) { info = ZEND_FUNC_INFO(call_graph.op_arrays[i]); if (info) { - zend_dump_op_array(call_graph.op_arrays[i], ZEND_DUMP_HIDE_UNREACHABLE|ZEND_DUMP_RC_INFERENCE|ZEND_DUMP_SSA|ZEND_DUMP_RT_CONSTANTS, "JIT", &info->ssa); + if (zend_jit_op_array_analyze2(call_graph.op_arrays[i], script, &info->ssa, &info->flags) != SUCCESS) { + goto jit_failure; + } } } - } - for (i = 0; i < call_graph.op_arrays_count; i++) { - info = ZEND_FUNC_INFO(call_graph.op_arrays[i]); - if (info) { - if (zend_jit(call_graph.op_arrays[i], &info->ssa) != SUCCESS) { - goto jit_failure; + if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_SSA) { + for (i = 0; i < call_graph.op_arrays_count; i++) { + info = ZEND_FUNC_INFO(call_graph.op_arrays[i]); + if (info) { + zend_dump_op_array(call_graph.op_arrays[i], ZEND_DUMP_HIDE_UNREACHABLE|ZEND_DUMP_RC_INFERENCE|ZEND_DUMP_SSA|ZEND_DUMP_RT_CONSTANTS, "JIT", &info->ssa); + } } } - } - for (i = 0; i < call_graph.op_arrays_count; i++) { - ZEND_SET_FUNC_INFO(call_graph.op_arrays[i], NULL); + for (i = 0; i < call_graph.op_arrays_count; i++) { + info = ZEND_FUNC_INFO(call_graph.op_arrays[i]); + if (info) { + if (zend_jit(call_graph.op_arrays[i], &info->ssa) != SUCCESS) { + goto jit_failure; + } + } + } + + for (i = 0; i < call_graph.op_arrays_count; i++) { + ZEND_SET_FUNC_INFO(call_graph.op_arrays[i], NULL); + } + } else { + ZEND_ASSERT(0); } -#endif zend_arena_release(&CG(arena), checkpoint); return SUCCESS; @@ -1617,13 +1618,14 @@ static int zend_jit_make_stubs(void) return 1; } -ZEND_API int zend_jit_startup(zend_long jit_level, size_t size) +ZEND_API int zend_jit_startup(zend_long jit, size_t size) { size_t page_size = jit_page_size(); int shared = 1; int ret; - zend_jit_level = jit_level; + zend_jit_level = ZEND_JIT_LEVEL(jit); + zend_jit_trigger = ZEND_JIT_TRIGGER(jit); #ifdef HAVE_GDB zend_jit_gdb_init(); @@ -1725,7 +1727,7 @@ ZEND_API void zend_jit_protect(void) { } -ZEND_API int zend_jit_startup(zend_long jit_level, size_t size) +ZEND_API int zend_jit_startup(zend_long jit, size_t size) { return FAILURE; } diff --git a/ext/opcache/jit/zend_jit.h b/ext/opcache/jit/zend_jit.h index 5dc629bd3f7bb..cb3c412d0f2a9 100644 --- a/ext/opcache/jit/zend_jit.h +++ b/ext/opcache/jit/zend_jit.h @@ -25,9 +25,17 @@ #define ZEND_JIT_LEVEL_MINIMAL 1 /* minimal JIT (subroutine threading) */ #define ZEND_JIT_LEVEL_INLINE 2 /* selective inline threading */ #define ZEND_JIT_LEVEL_OPT_FUNC 3 /* optimized JIT based on Type-Inference */ -#define ZEND_JIT_LEVEL_OPT_SCRIPT 4 /* optimized JIT based on Type-Inference and inner-procedure analises */ +#define ZEND_JIT_LEVEL_OPT_FUNCS 4 /* optimized JIT based on Type-Inference and call-tree */ +#define ZEND_JIT_LEVEL_OPT_SCRIPT 5 /* optimized JIT based on Type-Inference and inner-procedure analises */ -#define ZEND_JIT_LEVEL_DEFAULT "4" +#define ZEND_JIT_LEVEL(n) ((n) % 10) + +#define ZEND_JIT_ON_SCRIPT_LOAD 0 +#define ZEND_JIT_ON_FIRST_EXEC 1 + +#define ZEND_JIT_TRIGGER(n) (((n) / 10) % 10) + +#define ZEND_JIT_DEFAULT "5" #define ZEND_JIT_DEBUG_ASM (1<<0) #define ZEND_JIT_DEBUG_SSA (1<<1) @@ -42,7 +50,7 @@ ZEND_API int zend_jit_op_array(zend_op_array *op_array, zend_script *script); ZEND_API int zend_jit_script(zend_script *script); ZEND_API void zend_jit_unprotect(void); ZEND_API void zend_jit_protect(void); -ZEND_API int zend_jit_startup(zend_long jit_level, size_t size); +ZEND_API int zend_jit_startup(zend_long jit, size_t size); ZEND_API void zend_jit_shutdown(void); ZEND_API void zend_jit_status(zval *ret); diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 5c4e1447eb5b1..30200f5aa5135 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -5044,8 +5044,14 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar | //EX_LOAD_LITERALS(op_array); |.if X64 - | LOAD_ADDR r0, func->op_array.literals - | mov EX:RX->literals, r0 + || if (zend_accel_in_shm(func->op_array.literals)) { + | LOAD_ADDR r0, func->op_array.literals + | mov EX:RX->literals, r0 + || } else { + | mov r0, EX:RX->func + | mov r0, aword [r0 + offsetof(zend_op_array, literals)] + | mov EX:RX->literals, r0 + || } |.endif | // EG(current_execute_data) = execute_data; @@ -5053,7 +5059,15 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar | mov FP, RX | // opline = op_array->opcodes; - | LOAD_ADDR IP, (func->op_array.opcodes + call_info->num_args) + if (zend_accel_in_shm(func->op_array.opcodes)) { + | LOAD_ADDR IP, (func->op_array.opcodes + call_info->num_args) + } else { + | mov r0, EX->func + | mov IP, aword [r0 + offsetof(zend_op_array, opcodes)] + if (call_info->num_args) { + | add IP, (call_info->num_args * sizeof(zend_op)) + } + } if (func && op_array == &func->op_array) { /* recursive call */ diff --git a/ext/opcache/zend_accelerator_module.c b/ext/opcache/zend_accelerator_module.c index d7ad54cad268d..73f8ed2e797b2 100644 --- a/ext/opcache/zend_accelerator_module.c +++ b/ext/opcache/zend_accelerator_module.c @@ -322,7 +322,7 @@ ZEND_INI_BEGIN() STD_PHP_INI_BOOLEAN("opcache.huge_code_pages" , "0" , PHP_INI_SYSTEM, OnUpdateBool, accel_directives.huge_code_pages, zend_accel_globals, accel_globals) #endif #ifdef HAVE_JIT - STD_PHP_INI_ENTRY("opcache.jit" , ZEND_JIT_LEVEL_DEFAULT, PHP_INI_SYSTEM, OnUpdateLong, accel_directives.jit_level, zend_accel_globals, accel_globals) + STD_PHP_INI_ENTRY("opcache.jit" , ZEND_JIT_DEFAULT, PHP_INI_SYSTEM, OnUpdateLong, accel_directives.jit, zend_accel_globals, accel_globals) STD_PHP_INI_ENTRY("opcache.jit_buffer_size" , "0" , PHP_INI_SYSTEM, OnUpdateLong, accel_directives.jit_buffer_size, zend_accel_globals, accel_globals) STD_PHP_INI_ENTRY("opcache.jit_debug" , "0" , PHP_INI_SYSTEM, OnUpdateLong, accel_directives.jit_debug, zend_accel_globals, accel_globals) #endif diff --git a/ext/opcache/zend_persist.c b/ext/opcache/zend_persist.c index 2e255dec02d41..af5267b598ebe 100644 --- a/ext/opcache/zend_persist.c +++ b/ext/opcache/zend_persist.c @@ -570,8 +570,8 @@ static void zend_persist_op_array_ex(zend_op_array *op_array, zend_persistent_sc ZCG(mem) = (void*)((char*)ZCG(mem) + ZEND_ALIGNED_SIZE(zend_extensions_op_array_persist(op_array, ZCG(mem)))); #ifdef HAVE_JIT - if (ZCG(accel_directives).jit_level && - ZCG(accel_directives).jit_level <= ZEND_JIT_LEVEL_OPT_FUNC) { + if (ZCG(accel_directives).jit && + ZEND_JIT_LEVEL(ZCG(accel_directives).jit) <= ZEND_JIT_LEVEL_OPT_FUNCS) { zend_jit_op_array(op_array, ZCG(current_persistent_script) ? &ZCG(current_persistent_script)->script : NULL); } #endif @@ -837,7 +837,7 @@ static void zend_accel_persist_class_table(HashTable *class_table) zend_persistent_script *zend_accel_script_persist(zend_persistent_script *script, char **key, unsigned int key_length) { #ifdef HAVE_JIT - zend_long orig_jit_level = 0; + zend_long orig_jit = 0; #endif script->mem = ZCG(mem); @@ -862,12 +862,12 @@ zend_persistent_script *zend_accel_script_persist(zend_persistent_script *script ZCG(mem) = (void*)((char*)ZCG(mem) + script->arena_size); #ifdef HAVE_JIT - if (ZCG(accel_directives).jit_level) { + if (ZCG(accel_directives).jit) { if (key) { zend_jit_unprotect(); } else { - orig_jit_level = ZCG(accel_directives).jit_level; - ZCG(accel_directives).jit_level = 0; + orig_jit = ZCG(accel_directives).jit; + ZCG(accel_directives).jit = 0; } } #endif @@ -879,13 +879,13 @@ zend_persistent_script *zend_accel_script_persist(zend_persistent_script *script ZCG(current_persistent_script) = NULL; #ifdef HAVE_JIT - if (ZCG(accel_directives).jit_level) { - if (ZCG(accel_directives).jit_level >= ZEND_JIT_LEVEL_OPT_SCRIPT) { + if (ZCG(accel_directives).jit) { + if (ZEND_JIT_LEVEL(ZCG(accel_directives).jit) >= ZEND_JIT_LEVEL_OPT_SCRIPT) { zend_jit_script(&script->script); } zend_jit_protect(); } else if (!key) { - ZCG(accel_directives).jit_level = orig_jit_level; + ZCG(accel_directives).jit = orig_jit; } #endif From 112c7c3b457e17e6a5b40149c9ebe27ffccb4e00 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 26 Oct 2016 00:41:53 +0300 Subject: [PATCH 318/569] Fixed compilation warning --- ext/opcache/jit/zend_jit_x86.dasc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 30200f5aa5135..d20855ea92b51 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -890,8 +890,8 @@ static void* dasm_labels[zend_lb_MAX]; // zval should be in FCARG1a |.macro ZVAL_DTOR_FUNC, var_info, filename, lineno // arg1 must be in FCARG1a ||do { -|| if (has_concrete_type(var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { -|| zend_uchar type = concrete_type(var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)); +|| if (has_concrete_type((var_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { +|| zend_uchar type = concrete_type((var_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)); || if (type == IS_STRING && !ZEND_DEBUG) { | test byte [FCARG1a + 5], IS_STR_PERSISTENT | jnz >1 From 5ba48d273e2caa2c93796f045722e0d82699b098 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 26 Oct 2016 01:18:58 +0300 Subject: [PATCH 319/569] Introduce zend_jit_activate & zend_jit_deactivate to support smarter run-time triggers. --- ext/opcache/ZendAccelerator.c | 8 ++++++++ ext/opcache/jit/zend_jit.c | 16 ++++++++++++++++ ext/opcache/jit/zend_jit.h | 5 ++++- 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/ext/opcache/ZendAccelerator.c b/ext/opcache/ZendAccelerator.c index 70264765710e1..dceec7835e089 100644 --- a/ext/opcache/ZendAccelerator.c +++ b/ext/opcache/ZendAccelerator.c @@ -2158,6 +2158,10 @@ static void accel_activate(void) } else if (reset_pcre) { accel_reset_pcre_cache(); } + +#ifdef HAVE_JIT + zend_jit_activate(); +#endif } #if !ZEND_DEBUG @@ -2377,6 +2381,10 @@ static void accel_deactivate(void) return; } +#ifdef HAVE_JIT + zend_jit_deactivate(); +#endif + #if !ZEND_DEBUG if (ZCG(accel_directives).fast_shutdown && is_zend_mm()) { zend_accel_fast_shutdown(); diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index e3d6925cfc0c4..dd64e5aa57854 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -1707,6 +1707,14 @@ ZEND_API void zend_jit_shutdown(void) } } +ZEND_API void zend_jit_activate(void) +{ +} + +ZEND_API void zend_jit_deactivate(void) +{ +} + #else /* HAVE_JIT */ ZEND_API int zend_jit_op_array(zend_op_array *op_array, zend_script *script) @@ -1736,6 +1744,14 @@ ZEND_API void zend_jit_shutdown(void) { } +ZEND_API void zend_jit_activate(void) +{ +} + +ZEND_API void zend_jit_deactivate(void) +{ +} + #endif /* HAVE_JIT */ /* diff --git a/ext/opcache/jit/zend_jit.h b/ext/opcache/jit/zend_jit.h index cb3c412d0f2a9..f2f2cce4c9e3d 100644 --- a/ext/opcache/jit/zend_jit.h +++ b/ext/opcache/jit/zend_jit.h @@ -32,6 +32,8 @@ #define ZEND_JIT_ON_SCRIPT_LOAD 0 #define ZEND_JIT_ON_FIRST_EXEC 1 +#define ZEND_JIT_ON_PROF_REQUEST 2 +#define ZEND_JIT_ON_HOT_COUNTERS 3 #define ZEND_JIT_TRIGGER(n) (((n) / 10) % 10) @@ -45,13 +47,14 @@ #define ZEND_JIT_DEBUG_OPROFILE (1<<6) #define ZEND_JIT_DEBUG_VTUNE (1<<7) - ZEND_API int zend_jit_op_array(zend_op_array *op_array, zend_script *script); ZEND_API int zend_jit_script(zend_script *script); ZEND_API void zend_jit_unprotect(void); ZEND_API void zend_jit_protect(void); ZEND_API int zend_jit_startup(zend_long jit, size_t size); ZEND_API void zend_jit_shutdown(void); +ZEND_API void zend_jit_activate(void); +ZEND_API void zend_jit_deactivate(void); ZEND_API void zend_jit_status(zval *ret); #endif /* HAVE_JIT_H */ From bf1775a079232cc4c302805bb9cea5c7aa56754d Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Wed, 26 Oct 2016 12:12:30 +0800 Subject: [PATCH 320/569] Introduced runtime-profile jit strategy (5% QPS improvement on WP) --- ext/opcache/jit/zend_jit.c | 80 ++++++++++++++++++++++++++-- ext/opcache/jit/zend_jit.h | 7 +++ ext/opcache/jit/zend_jit_vm_helper.c | 14 +++++ ext/opcache/jit/zend_jit_vm_helper.h | 1 + 4 files changed, 98 insertions(+), 4 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index dd64e5aa57854..e38ad6b8b5d83 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -66,6 +66,10 @@ typedef struct _zend_jit_stub { static zend_uchar zend_jit_level = 0; static zend_uchar zend_jit_trigger = 0; + +zend_ulong zend_jit_count = 0; +int zend_jit_counter_rid = -1; + static void *dasm_buf = NULL; static void *dasm_end = NULL; static void **dasm_ptr = NULL; @@ -1446,8 +1450,6 @@ static int zend_real_jit_func(zend_op_array *op_array, zend_script *script) } \ } while (0) -typedef void (ZEND_FASTCALL *zend_vm_opcode_handler_t)(void); - /* Run-time JIT handler */ static void ZEND_FASTCALL zend_runtime_jit(void) { @@ -1475,18 +1477,63 @@ static void ZEND_FASTCALL zend_runtime_jit(void) /* JIT-ed code is going to be called by VM */ } +void zend_jit_check_funcs(HashTable *function_table, zend_bool is_method) { + zend_op *opline; + zend_function *func; + zend_op_array *op_array; + + ZEND_HASH_REVERSE_FOREACH_PTR(function_table, func) { + if (func->type == ZEND_INTERNAL_FUNCTION) { + break; + } + op_array = &func->op_array; + opline = op_array->opcodes; + while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) { + opline++; + } + if (opline->handler == zend_jit_profile_helper) { + zend_ulong count = (zend_ulong)ZEND_COUNTER_INFO(op_array); + ZEND_COUNTER_INFO(op_array) = 0; + opline->handler = ZEND_FUNC_INFO(op_array); + ZEND_SET_FUNC_INFO(op_array, NULL); + if (((double)count / (double)zend_jit_count) > 0.005) { + zend_jit_unprotect(); + zend_real_jit_func(op_array, NULL); + zend_jit_protect(); + } + } + } ZEND_HASH_FOREACH_END(); +} + ZEND_API int zend_jit_op_array(zend_op_array *op_array, zend_script *script) { + if (dasm_ptr == NULL) { + return FAILURE; + } + if (zend_jit_trigger == ZEND_JIT_ON_FIRST_EXEC) { zend_op *opline = op_array->opcodes; /* Set run-time JIT handler */ while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) { - opline->handler = (const void*)zend_runtime_jit; + opline->handler = (void*)zend_runtime_jit; opline++; } ZEND_SET_FUNC_INFO(op_array, (void*)opline->handler); - opline->handler = (const void*)zend_runtime_jit; + opline->handler = (void*)zend_runtime_jit; + + return SUCCESS; + } else if (zend_jit_trigger == ZEND_JIT_ON_PROF_REQUEST) { + zend_op *opline = op_array->opcodes; + + if (op_array->function_name) { + while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) { + opline++; + } + ZEND_SET_FUNC_INFO(op_array, (void*)opline->handler); + opline->handler = zend_jit_profile_helper; + } + return SUCCESS; } else if (zend_jit_trigger == ZEND_JIT_ON_SCRIPT_LOAD) { return zend_real_jit_func(op_array, script); @@ -1627,6 +1674,12 @@ ZEND_API int zend_jit_startup(zend_long jit, size_t size) zend_jit_level = ZEND_JIT_LEVEL(jit); zend_jit_trigger = ZEND_JIT_TRIGGER(jit); + if (zend_jit_trigger == ZEND_JIT_ON_PROF_REQUEST) { + zend_extension dummy; + zend_jit_counter_rid = zend_get_resource_handle(&dummy); + ZEND_ASSERT(zend_jit_counter_rid != -1); + } + #ifdef HAVE_GDB zend_jit_gdb_init(); #endif @@ -1713,6 +1766,25 @@ ZEND_API void zend_jit_activate(void) ZEND_API void zend_jit_deactivate(void) { + if (zend_jit_trigger == ZEND_JIT_ON_PROF_REQUEST) { + if (!zend_jit_count) { + return; + } else { + zend_class_entry *ce; + + zend_shared_alloc_lock(); + zend_jit_check_funcs(EG(function_table), 0); + ZEND_HASH_REVERSE_FOREACH_PTR(EG(class_table), ce) { + if (ce->type == ZEND_INTERNAL_CLASS) { + break; + } + zend_jit_check_funcs(&ce->function_table, 1); + } ZEND_HASH_FOREACH_END(); + zend_shared_alloc_unlock(); + + zend_jit_count = 0; + } + } } #else /* HAVE_JIT */ diff --git a/ext/opcache/jit/zend_jit.h b/ext/opcache/jit/zend_jit.h index f2f2cce4c9e3d..1ec10b21909b9 100644 --- a/ext/opcache/jit/zend_jit.h +++ b/ext/opcache/jit/zend_jit.h @@ -47,6 +47,13 @@ #define ZEND_JIT_DEBUG_OPROFILE (1<<6) #define ZEND_JIT_DEBUG_VTUNE (1<<7) +extern zend_ulong zend_jit_count; +extern int zend_jit_counter_rid; + +#define ZEND_COUNTER_INFO(op_array) ((op_array)->reserved[zend_jit_counter_rid]) + +typedef void (ZEND_FASTCALL *zend_vm_opcode_handler_t)(void); + ZEND_API int zend_jit_op_array(zend_op_array *op_array, zend_script *script); ZEND_API int zend_jit_script(zend_script *script); ZEND_API void zend_jit_unprotect(void); diff --git a/ext/opcache/jit/zend_jit_vm_helper.c b/ext/opcache/jit/zend_jit_vm_helper.c index ae650124fa1ac..62a1f425b403f 100644 --- a/ext/opcache/jit/zend_jit_vm_helper.c +++ b/ext/opcache/jit/zend_jit_vm_helper.c @@ -19,6 +19,11 @@ #include "Zend/zend_execute.h" #include "Zend/zend_exceptions.h" + +#include +#include "Optimizer/zend_func_info.h" +#include "zend_jit.h" + #include "zend_jit_vm_helper.h" #pragma GCC diagnostic ignored "-Wvolatile-register-var" @@ -112,6 +117,15 @@ void ZEND_FASTCALL zend_jit_loop_header_helper(void) opline->lineno); } +void ZEND_FASTCALL zend_jit_profile_helper(void) +{ + zend_op_array *op_array = (zend_op_array*)EX(func); + const void *handler = (const void*)ZEND_FUNC_INFO(op_array); + ++(ZEND_COUNTER_INFO(op_array)); + ++zend_jit_count; + return ((zend_vm_opcode_handler_t)handler)(); +} + /* * Local variables: * tab-width: 4 diff --git a/ext/opcache/jit/zend_jit_vm_helper.h b/ext/opcache/jit/zend_jit_vm_helper.h index 81554008cf04c..1a2ed9497faff 100644 --- a/ext/opcache/jit/zend_jit_vm_helper.h +++ b/ext/opcache/jit/zend_jit_vm_helper.h @@ -24,6 +24,7 @@ void ZEND_FASTCALL zend_jit_leave_nested_func_helper(uint32_t call_info); void ZEND_FASTCALL zend_jit_leave_top_func_helper(uint32_t call_info); void ZEND_FASTCALL zend_jit_func_header_helper(void); void ZEND_FASTCALL zend_jit_loop_header_helper(void); +void ZEND_FASTCALL zend_jit_profile_helper(void); #endif From e9f7f746aab6f73756575fa2ac56e4a5e3fbb508 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Wed, 26 Oct 2016 12:20:37 +0800 Subject: [PATCH 321/569] Removed unecessary change --- ext/opcache/jit/zend_jit.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index e38ad6b8b5d83..23df7aaa44616 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -1516,11 +1516,11 @@ ZEND_API int zend_jit_op_array(zend_op_array *op_array, zend_script *script) /* Set run-time JIT handler */ while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) { - opline->handler = (void*)zend_runtime_jit; + opline->handler = (const void*)zend_runtime_jit; opline++; } ZEND_SET_FUNC_INFO(op_array, (void*)opline->handler); - opline->handler = (void*)zend_runtime_jit; + opline->handler = (const void*)zend_runtime_jit; return SUCCESS; } else if (zend_jit_trigger == ZEND_JIT_ON_PROF_REQUEST) { From d1217498022833e1db1dd5d6e9d001eb3ca7024f Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Wed, 26 Oct 2016 12:36:57 +0800 Subject: [PATCH 322/569] Cleanup memory protection --- ext/opcache/jit/zend_jit.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 23df7aaa44616..9a8dd6c07d53a 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -1497,9 +1497,7 @@ void zend_jit_check_funcs(HashTable *function_table, zend_bool is_method) { opline->handler = ZEND_FUNC_INFO(op_array); ZEND_SET_FUNC_INFO(op_array, NULL); if (((double)count / (double)zend_jit_count) > 0.005) { - zend_jit_unprotect(); zend_real_jit_func(op_array, NULL); - zend_jit_protect(); } } } ZEND_HASH_FOREACH_END(); @@ -1772,7 +1770,9 @@ ZEND_API void zend_jit_deactivate(void) } else { zend_class_entry *ce; - zend_shared_alloc_lock(); + SHM_UNPROTECT(); + zend_jit_unprotect(); + zend_jit_check_funcs(EG(function_table), 0); ZEND_HASH_REVERSE_FOREACH_PTR(EG(class_table), ce) { if (ce->type == ZEND_INTERNAL_CLASS) { @@ -1780,7 +1780,9 @@ ZEND_API void zend_jit_deactivate(void) } zend_jit_check_funcs(&ce->function_table, 1); } ZEND_HASH_FOREACH_END(); - zend_shared_alloc_unlock(); + + zend_jit_protect(); + SHM_PROTECT(); zend_jit_count = 0; } From c3197cd8d5fbac8e20793bad4e1ff19b0520870e Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Wed, 26 Oct 2016 12:41:34 +0800 Subject: [PATCH 323/569] Shared memory lock is needed --- ext/opcache/jit/zend_jit.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 9a8dd6c07d53a..d86e86470f755 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -1770,6 +1770,7 @@ ZEND_API void zend_jit_deactivate(void) } else { zend_class_entry *ce; + zend_shared_alloc_lock(); SHM_UNPROTECT(); zend_jit_unprotect(); @@ -1783,6 +1784,7 @@ ZEND_API void zend_jit_deactivate(void) zend_jit_protect(); SHM_PROTECT(); + zend_shared_alloc_unlock(); zend_jit_count = 0; } From d5e9541acd7cb8807c988451bf401ad5df575590 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 26 Oct 2016 10:46:32 +0300 Subject: [PATCH 324/569] Rename files and few variables --- ext/opcache/config.m4 | 2 +- ext/opcache/jit/Makefile.frag | 1 + ext/opcache/jit/zend_jit.c | 18 +++++++++--------- ext/opcache/jit/zend_jit.h | 7 ------- ...end_jit_vm_helper.h => zend_jit_internal.h} | 16 +++++++++++++--- ...d_jit_vm_helper.c => zend_jit_vm_helpers.c} | 5 ++--- 6 files changed, 26 insertions(+), 23 deletions(-) rename ext/opcache/jit/{zend_jit_vm_helper.h => zend_jit_internal.h} (81%) rename ext/opcache/jit/{zend_jit_vm_helper.c => zend_jit_vm_helpers.c} (98%) diff --git a/ext/opcache/config.m4 b/ext/opcache/config.m4 index 35f01ad99aa4f..a49e86845dac4 100644 --- a/ext/opcache/config.m4 +++ b/ext/opcache/config.m4 @@ -30,7 +30,7 @@ if test "$PHP_OPCACHE" != "no"; then if test "$PHP_OPCACHE_JIT" = "yes"; then AC_DEFINE(HAVE_JIT, 1, [Define to enable JIT]) - ZEND_JIT_SRC="jit/zend_jit.c jit/zend_jit_vm_helper.c" + ZEND_JIT_SRC="jit/zend_jit.c jit/zend_jit_vm_helpers.c" dnl Find out which ABI we are using. echo 'int i;' > conftest.$ac_ext diff --git a/ext/opcache/jit/Makefile.frag b/ext/opcache/jit/Makefile.frag index 3ef58205e9403..02e44c2654c0e 100644 --- a/ext/opcache/jit/Makefile.frag +++ b/ext/opcache/jit/Makefile.frag @@ -12,5 +12,6 @@ $(builddir)/jit/zend_jit.lo: \ $(srcdir)/jit/zend_jit_gdb.c \ $(srcdir)/jit/zend_jit_perf_dump.c \ $(srcdir)/jit/zend_jit_oprofile.c \ + $(srcdir)/jit/zend_jit_vtune.c \ $(srcdir)/jit/zend_elf.c diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index d86e86470f755..280f0f973fa5b 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -23,7 +23,7 @@ #include "Zend/zend_exceptions.h" #include "zend_smart_str.h" #include "jit/zend_jit.h" -#include "jit/zend_jit_vm_helper.h" +#include "jit/zend_jit_internal.h" #ifdef HAVE_JIT @@ -67,8 +67,8 @@ typedef struct _zend_jit_stub { static zend_uchar zend_jit_level = 0; static zend_uchar zend_jit_trigger = 0; -zend_ulong zend_jit_count = 0; -int zend_jit_counter_rid = -1; +zend_ulong zend_jit_profile_counter = 0; +int zend_jit_profile_counter_rid = -1; static void *dasm_buf = NULL; static void *dasm_end = NULL; @@ -1492,11 +1492,11 @@ void zend_jit_check_funcs(HashTable *function_table, zend_bool is_method) { opline++; } if (opline->handler == zend_jit_profile_helper) { - zend_ulong count = (zend_ulong)ZEND_COUNTER_INFO(op_array); + zend_ulong counter = (zend_ulong)ZEND_COUNTER_INFO(op_array); ZEND_COUNTER_INFO(op_array) = 0; opline->handler = ZEND_FUNC_INFO(op_array); ZEND_SET_FUNC_INFO(op_array, NULL); - if (((double)count / (double)zend_jit_count) > 0.005) { + if (((double)counter / (double)zend_jit_profile_counter) > 0.005) { zend_real_jit_func(op_array, NULL); } } @@ -1674,8 +1674,8 @@ ZEND_API int zend_jit_startup(zend_long jit, size_t size) if (zend_jit_trigger == ZEND_JIT_ON_PROF_REQUEST) { zend_extension dummy; - zend_jit_counter_rid = zend_get_resource_handle(&dummy); - ZEND_ASSERT(zend_jit_counter_rid != -1); + zend_jit_profile_counter_rid = zend_get_resource_handle(&dummy); + ZEND_ASSERT(zend_jit_profile_counter_rid != -1); } #ifdef HAVE_GDB @@ -1765,7 +1765,7 @@ ZEND_API void zend_jit_activate(void) ZEND_API void zend_jit_deactivate(void) { if (zend_jit_trigger == ZEND_JIT_ON_PROF_REQUEST) { - if (!zend_jit_count) { + if (!zend_jit_profile_counter) { return; } else { zend_class_entry *ce; @@ -1786,7 +1786,7 @@ ZEND_API void zend_jit_deactivate(void) SHM_PROTECT(); zend_shared_alloc_unlock(); - zend_jit_count = 0; + zend_jit_profile_counter = 0; } } } diff --git a/ext/opcache/jit/zend_jit.h b/ext/opcache/jit/zend_jit.h index 1ec10b21909b9..f2f2cce4c9e3d 100644 --- a/ext/opcache/jit/zend_jit.h +++ b/ext/opcache/jit/zend_jit.h @@ -47,13 +47,6 @@ #define ZEND_JIT_DEBUG_OPROFILE (1<<6) #define ZEND_JIT_DEBUG_VTUNE (1<<7) -extern zend_ulong zend_jit_count; -extern int zend_jit_counter_rid; - -#define ZEND_COUNTER_INFO(op_array) ((op_array)->reserved[zend_jit_counter_rid]) - -typedef void (ZEND_FASTCALL *zend_vm_opcode_handler_t)(void); - ZEND_API int zend_jit_op_array(zend_op_array *op_array, zend_script *script); ZEND_API int zend_jit_script(zend_script *script); ZEND_API void zend_jit_unprotect(void); diff --git a/ext/opcache/jit/zend_jit_vm_helper.h b/ext/opcache/jit/zend_jit_internal.h similarity index 81% rename from ext/opcache/jit/zend_jit_vm_helper.h rename to ext/opcache/jit/zend_jit_internal.h index 1a2ed9497faff..f0289a5863a65 100644 --- a/ext/opcache/jit/zend_jit_vm_helper.h +++ b/ext/opcache/jit/zend_jit_internal.h @@ -17,16 +17,26 @@ +----------------------------------------------------------------------+ */ -#ifndef ZEND_JIT_VM_HELPER -#define ZEND_JIT_VM_HELPER +#ifndef ZEND_JIT_INTERNAL_H +#define ZEND_JIT_INTERNAL_H +/* Profiler */ +extern zend_ulong zend_jit_profile_counter; +extern int zend_jit_profile_counter_rid; + +#define ZEND_COUNTER_INFO(op_array) ((op_array)->reserved[zend_jit_profile_counter_rid]) + +/* VM handlers */ +typedef void (ZEND_FASTCALL *zend_vm_opcode_handler_t)(void); + +/* VM helpers */ void ZEND_FASTCALL zend_jit_leave_nested_func_helper(uint32_t call_info); void ZEND_FASTCALL zend_jit_leave_top_func_helper(uint32_t call_info); void ZEND_FASTCALL zend_jit_func_header_helper(void); void ZEND_FASTCALL zend_jit_loop_header_helper(void); void ZEND_FASTCALL zend_jit_profile_helper(void); -#endif +#endif /* ZEND_JIT_INTERNAL_H */ /* * Local variables: diff --git a/ext/opcache/jit/zend_jit_vm_helper.c b/ext/opcache/jit/zend_jit_vm_helpers.c similarity index 98% rename from ext/opcache/jit/zend_jit_vm_helper.c rename to ext/opcache/jit/zend_jit_vm_helpers.c index 62a1f425b403f..ea6a1c2519220 100644 --- a/ext/opcache/jit/zend_jit_vm_helper.c +++ b/ext/opcache/jit/zend_jit_vm_helpers.c @@ -23,8 +23,7 @@ #include #include "Optimizer/zend_func_info.h" #include "zend_jit.h" - -#include "zend_jit_vm_helper.h" +#include "zend_jit_internal.h" #pragma GCC diagnostic ignored "-Wvolatile-register-var" #if defined(__x86_64__) @@ -122,7 +121,7 @@ void ZEND_FASTCALL zend_jit_profile_helper(void) zend_op_array *op_array = (zend_op_array*)EX(func); const void *handler = (const void*)ZEND_FUNC_INFO(op_array); ++(ZEND_COUNTER_INFO(op_array)); - ++zend_jit_count; + ++zend_jit_profile_counter; return ((zend_vm_opcode_handler_t)handler)(); } From f8a8a43cc6fa77e1760dbdfd2657b58c35ebf40e Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 26 Oct 2016 10:49:52 +0300 Subject: [PATCH 325/569] Make opcache.jit=25 work in the same way as opcache.jit=24 --- ext/opcache/jit/zend_jit.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 280f0f973fa5b..41fc4ab770e06 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -1558,7 +1558,8 @@ ZEND_API int zend_jit_script(zend_script *script) goto jit_failure; } - if (zend_jit_trigger == ZEND_JIT_ON_FIRST_EXEC) { + if (zend_jit_trigger == ZEND_JIT_ON_FIRST_EXEC || + zend_jit_trigger == ZEND_JIT_ON_PROF_REQUEST) { for (i = 0; i < call_graph.op_arrays_count; i++) { ZEND_SET_FUNC_INFO(call_graph.op_arrays[i], NULL); if (zend_jit_op_array(call_graph.op_arrays[i], script) != SUCCESS) { From d587c6680e1f31e1d41c676f51a684ad8341fad1 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 26 Oct 2016 10:58:07 +0300 Subject: [PATCH 326/569] Replace hard-coded constant with a macro --- ext/opcache/jit/zend_jit.c | 2 +- ext/opcache/jit/zend_jit.h | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 41fc4ab770e06..98888c0547775 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -1496,7 +1496,7 @@ void zend_jit_check_funcs(HashTable *function_table, zend_bool is_method) { ZEND_COUNTER_INFO(op_array) = 0; opline->handler = ZEND_FUNC_INFO(op_array); ZEND_SET_FUNC_INFO(op_array, NULL); - if (((double)counter / (double)zend_jit_profile_counter) > 0.005) { + if (((double)counter / (double)zend_jit_profile_counter) > ZEND_JIT_PROF_THRESHOLD) { zend_real_jit_func(op_array, NULL); } } diff --git a/ext/opcache/jit/zend_jit.h b/ext/opcache/jit/zend_jit.h index f2f2cce4c9e3d..b3ef4d39d98e5 100644 --- a/ext/opcache/jit/zend_jit.h +++ b/ext/opcache/jit/zend_jit.h @@ -39,6 +39,12 @@ #define ZEND_JIT_DEFAULT "5" +/* Makes profile based JIT (opcache.jit=2*) to generate code only for most + * offten called functions (above the threshold). + * TODO: this setting should be configurable + */ +#define ZEND_JIT_PROF_THRESHOLD 0.005 + #define ZEND_JIT_DEBUG_ASM (1<<0) #define ZEND_JIT_DEBUG_SSA (1<<1) From 45944a707acf674c1b63f7cadbb485a90b895f1b Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 26 Oct 2016 14:12:23 +0300 Subject: [PATCH 327/569] Implemented anothe JIT trigger, that starts function compilation at run-time, when its invocation/loop counter reaches some value. --- ext/opcache/jit/zend_jit.c | 170 ++++++++++++++++++++++---- ext/opcache/jit/zend_jit.h | 7 ++ ext/opcache/jit/zend_jit_internal.h | 9 ++ ext/opcache/jit/zend_jit_vm_helpers.c | 34 ++++++ 4 files changed, 193 insertions(+), 27 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 98888c0547775..504e48a00ddb7 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -70,6 +70,8 @@ static zend_uchar zend_jit_trigger = 0; zend_ulong zend_jit_profile_counter = 0; int zend_jit_profile_counter_rid = -1; +int16_t zend_jit_hot_counters[ZEND_HOT_COUNTERS_COUNT]; + static void *dasm_buf = NULL; static void *dasm_end = NULL; static void **dasm_ptr = NULL; @@ -146,6 +148,7 @@ static zend_string *zend_jit_func_name(zend_op_array *op_array) static void *dasm_link_and_encode(dasm_State **dasm_state, zend_op_array *op_array, zend_cfg *cfg, + const zend_op *rt_opline, const char *name) { size_t size; @@ -155,6 +158,23 @@ static void *dasm_link_and_encode(dasm_State **dasm_state, zend_string *str = NULL; #endif + if (rt_opline && cfg && cfg->map) { + /* Create additional entry point, to switch from interpreter to JIT-ed + * code at run-time. + */ + int b = cfg->map[rt_opline - op_array->opcodes]; + +#ifdef CONTEXT_THREADED_JIT + if (!(cfg->blocks[b].flags & (ZEND_BB_START|ZEND_BB_RECV_ENTRY))) { +#else + if (!(cfg->blocks[b].flags & (ZEND_BB_START|ZEND_BB_ENTRY|ZEND_BB_RECV_ENTRY))) { +#endif + zend_jit_label(dasm_state, cfg->blocks_count + b); + zend_jit_prologue(dasm_state); + zend_jit_jmp(dasm_state, b); + } + } + if (dasm_link(dasm_state, &size) != DASM_S_OK) { // TODO: dasm_link() failed ??? return NULL; @@ -191,6 +211,12 @@ static void *dasm_link_and_encode(dasm_State **dasm_state, dasm_getpclabel(dasm_state, cfg->blocks_count + b)); } } + if (rt_opline && cfg && cfg->map) { + int b = cfg->map[rt_opline - op_array->opcodes]; + zend_op *opline = (zend_op*)rt_opline; + opline->handler = (void*)(((char*)entry) + + dasm_getpclabel(dasm_state, cfg->blocks_count + b)); + } } #if defined(HAVE_DISASM) || defined(HAVE_GDB) || defined(HAVE_OPROFILE) || defined(HAVE_PERFTOOLS) || defined(HAVE_VTUNE) @@ -851,23 +877,32 @@ static int zend_may_overflow(const zend_op *opline, zend_op_array *op_array, zen } } -static int zend_jit_op_array_analyze1(zend_op_array *op_array, zend_script *script, zend_ssa *ssa, uint32_t *flags) +static int zend_jit_build_cfg(zend_op_array *op_array, zend_cfg *cfg, uint32_t *flags) { - if (zend_build_cfg(&CG(arena), op_array, ZEND_CFG_STACKLESS | ZEND_CFG_RECV_ENTRY | ZEND_RT_CONSTANTS | ZEND_CFG_NO_ENTRY_PREDECESSORS | ZEND_SSA_RC_INFERENCE_FLAG, &ssa->cfg, flags) != SUCCESS) { + if (zend_build_cfg(&CG(arena), op_array, ZEND_CFG_STACKLESS | ZEND_CFG_RECV_ENTRY | ZEND_RT_CONSTANTS | ZEND_CFG_NO_ENTRY_PREDECESSORS | ZEND_SSA_RC_INFERENCE_FLAG, cfg, flags) != SUCCESS) { return FAILURE; } - if (zend_cfg_build_predecessors(&CG(arena), &ssa->cfg) != SUCCESS) { + if (zend_cfg_build_predecessors(&CG(arena), cfg) != SUCCESS) { return FAILURE; } /* Compute Dominators Tree */ - if (zend_cfg_compute_dominators_tree(op_array, &ssa->cfg) != SUCCESS) { + if (zend_cfg_compute_dominators_tree(op_array, cfg) != SUCCESS) { return FAILURE; } /* Identify reducible and irreducible loops */ - if (zend_cfg_identify_loops(op_array, &ssa->cfg, flags) != SUCCESS) { + if (zend_cfg_identify_loops(op_array, cfg, flags) != SUCCESS) { + return FAILURE; + } + + return SUCCESS; +} + +static int zend_jit_op_array_analyze1(zend_op_array *op_array, zend_script *script, zend_ssa *ssa, uint32_t *flags) +{ + if (zend_jit_build_cfg(op_array, &ssa->cfg, flags) != SUCCESS) { return FAILURE; } @@ -914,7 +949,7 @@ static int zend_jit_op_array_analyze2(zend_op_array *op_array, zend_script *scri return SUCCESS; } -static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) +static int zend_jit(zend_op_array *op_array, zend_ssa *ssa, const zend_op *rt_opline) { int b, i, end; zend_op *opline; @@ -1359,7 +1394,7 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa) } } - handler = dasm_link_and_encode(&dasm_state, op_array, &ssa->cfg, NULL); + handler = dasm_link_and_encode(&dasm_state, op_array, &ssa->cfg, rt_opline, NULL); if (!handler) { goto jit_failure; } @@ -1385,7 +1420,7 @@ static int zend_jit_collect_calls(zend_op_array *op_array, zend_script *script) return zend_analyze_calls(&CG(arena), script, ZEND_RT_CONSTANTS | ZEND_CALL_TREE, op_array, func_info); } -static int zend_real_jit_func(zend_op_array *op_array, zend_script *script) +static int zend_real_jit_func(zend_op_array *op_array, zend_script *script, const zend_op *rt_opline) { uint32_t flags = 0; zend_ssa ssa; @@ -1419,7 +1454,7 @@ static int zend_real_jit_func(zend_op_array *op_array, zend_script *script) } } - if (zend_jit(op_array, &ssa) != SUCCESS) { + if (zend_jit(op_array, &ssa, rt_opline) != SUCCESS) { goto jit_failure; } @@ -1457,22 +1492,28 @@ static void ZEND_FASTCALL zend_runtime_jit(void) zend_op_array *op_array = &EX(func)->op_array; zend_op *opline = op_array->opcodes; - SHM_UNPROTECT(); - zend_jit_unprotect(); + zend_shared_alloc_lock(); - /* restore original opcode handlers */ - while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) { - zend_vm_set_opcode_handler(opline); - opline++; - } - opline->handler = ZEND_FUNC_INFO(op_array); - ZEND_SET_FUNC_INFO(op_array, NULL); + if (ZEND_FUNC_INFO(op_array)) { + SHM_UNPROTECT(); + zend_jit_unprotect(); - /* perform real JIT for this function */ - zend_real_jit_func(op_array, NULL); + /* restore original opcode handlers */ + while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) { + zend_vm_set_opcode_handler(opline); + opline++; + } + opline->handler = ZEND_FUNC_INFO(op_array); + ZEND_SET_FUNC_INFO(op_array, NULL); - zend_jit_protect(); - SHM_PROTECT(); + /* perform real JIT for this function */ + zend_real_jit_func(op_array, NULL, NULL); + + zend_jit_protect(); + SHM_PROTECT(); + } + + zend_shared_alloc_unlock(); /* JIT-ed code is going to be called by VM */ } @@ -1497,12 +1538,77 @@ void zend_jit_check_funcs(HashTable *function_table, zend_bool is_method) { opline->handler = ZEND_FUNC_INFO(op_array); ZEND_SET_FUNC_INFO(op_array, NULL); if (((double)counter / (double)zend_jit_profile_counter) > ZEND_JIT_PROF_THRESHOLD) { - zend_real_jit_func(op_array, NULL); + zend_real_jit_func(op_array, NULL, NULL); } } } ZEND_HASH_FOREACH_END(); } +void zend_jit_hot_func(zend_execute_data *execute_data, const zend_op *opline) +{ + zend_op_array *op_array = &EX(func)->op_array; + const void **orig_handlers; + uint32_t i; + + zend_shared_alloc_lock(); + orig_handlers = (const void**)ZEND_FUNC_INFO(op_array); + + if (orig_handlers) { + SHM_UNPROTECT(); + zend_jit_unprotect(); + + for (i = 0; i < op_array->last; i++) { + op_array->opcodes[i].handler = orig_handlers[i]; + } + ZEND_SET_FUNC_INFO(op_array, NULL); + + /* perform real JIT for this function */ + zend_real_jit_func(op_array, NULL, opline); + + zend_jit_protect(); + SHM_PROTECT(); + } + + zend_shared_alloc_unlock(); + + /* JIT-ed code is going to be called by VM */ +} + +static int zend_jit_setup_hot_counters(zend_op_array *op_array) +{ + zend_op *opline = op_array->opcodes; + const void **orig_handlers; + zend_cfg cfg; + uint32_t flags = 0; + uint32_t i; + + if (zend_jit_build_cfg(op_array, &cfg, &flags) != SUCCESS) { + return FAILURE; + } + + orig_handlers = (const void**)zend_shared_alloc(op_array->last * sizeof(void*)); + for (i = 0; i < op_array->last; i++) { + orig_handlers[i] = op_array->opcodes[i].handler; + } + ZEND_SET_FUNC_INFO(op_array, (void*)orig_handlers); + + while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) { + opline++; + } + + opline->handler = (const void*)zend_jit_func_counter_helper; + + for (i = 0; i < cfg.blocks_count; i++) { + if ((cfg.blocks[i].flags & ZEND_BB_REACHABLE) && + (cfg.blocks[i].flags & ZEND_BB_LOOP_HEADER)) { + op_array->opcodes[cfg.blocks[i].start].handler = + (const void*)zend_jit_loop_counter_helper; + } + } + + return SUCCESS; +} + ZEND_API int zend_jit_op_array(zend_op_array *op_array, zend_script *script) { if (dasm_ptr == NULL) { @@ -1533,8 +1639,10 @@ ZEND_API int zend_jit_op_array(zend_op_array *op_array, zend_script *script) } return SUCCESS; + } else if (zend_jit_trigger == ZEND_JIT_ON_HOT_COUNTERS) { + return zend_jit_setup_hot_counters(op_array); } else if (zend_jit_trigger == ZEND_JIT_ON_SCRIPT_LOAD) { - return zend_real_jit_func(op_array, script); + return zend_real_jit_func(op_array, script, NULL); } else { ZEND_ASSERT(0); } @@ -1559,7 +1667,8 @@ ZEND_API int zend_jit_script(zend_script *script) } if (zend_jit_trigger == ZEND_JIT_ON_FIRST_EXEC || - zend_jit_trigger == ZEND_JIT_ON_PROF_REQUEST) { + zend_jit_trigger == ZEND_JIT_ON_PROF_REQUEST || + zend_jit_trigger == ZEND_JIT_ON_HOT_COUNTERS) { for (i = 0; i < call_graph.op_arrays_count; i++) { ZEND_SET_FUNC_INFO(call_graph.op_arrays[i], NULL); if (zend_jit_op_array(call_graph.op_arrays[i], script) != SUCCESS) { @@ -1597,7 +1706,7 @@ ZEND_API int zend_jit_script(zend_script *script) for (i = 0; i < call_graph.op_arrays_count; i++) { info = ZEND_FUNC_INFO(call_graph.op_arrays[i]); if (info) { - if (zend_jit(call_graph.op_arrays[i], &info->ssa) != SUCCESS) { + if (zend_jit(call_graph.op_arrays[i], &info->ssa, NULL) != SUCCESS) { goto jit_failure; } } @@ -1656,7 +1765,7 @@ static int zend_jit_make_stubs(void) if (!zend_jit_stubs[i].stub(&dasm_state)) { return 0; } - if (!dasm_link_and_encode(&dasm_state, NULL, NULL, zend_jit_stubs[i].name)) { + if (!dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, zend_jit_stubs[i].name)) { return 0; } } @@ -1761,6 +1870,13 @@ ZEND_API void zend_jit_shutdown(void) ZEND_API void zend_jit_activate(void) { + if (zend_jit_trigger == ZEND_JIT_ON_HOT_COUNTERS) { + int i; + + for (i = 0; i < ZEND_HOT_COUNTERS_COUNT; i++) { + zend_jit_hot_counters[i] = ZEND_JIT_HOT_COUNTER_INIT; + } + } } ZEND_API void zend_jit_deactivate(void) diff --git a/ext/opcache/jit/zend_jit.h b/ext/opcache/jit/zend_jit.h index b3ef4d39d98e5..88b310cd78cc3 100644 --- a/ext/opcache/jit/zend_jit.h +++ b/ext/opcache/jit/zend_jit.h @@ -45,6 +45,13 @@ */ #define ZEND_JIT_PROF_THRESHOLD 0.005 +/* Hot Counters based JIT parameters. + * TODO: this setting should be configurable + */ +#define ZEND_JIT_HOT_FUNC_COST 1 +#define ZEND_JIT_HOT_LOOP_COST 2 +#define ZEND_JIT_HOT_COUNTER_INIT 1000 + #define ZEND_JIT_DEBUG_ASM (1<<0) #define ZEND_JIT_DEBUG_SSA (1<<1) diff --git a/ext/opcache/jit/zend_jit_internal.h b/ext/opcache/jit/zend_jit_internal.h index f0289a5863a65..54ac505942994 100644 --- a/ext/opcache/jit/zend_jit_internal.h +++ b/ext/opcache/jit/zend_jit_internal.h @@ -26,6 +26,13 @@ extern int zend_jit_profile_counter_rid; #define ZEND_COUNTER_INFO(op_array) ((op_array)->reserved[zend_jit_profile_counter_rid]) +/* Hot Counters */ +#define ZEND_HOT_COUNTERS_COUNT 64 + +extern int16_t zend_jit_hot_counters[ZEND_HOT_COUNTERS_COUNT]; + +void zend_jit_hot_func(zend_execute_data *execute_data, const zend_op *opline); + /* VM handlers */ typedef void (ZEND_FASTCALL *zend_vm_opcode_handler_t)(void); @@ -35,6 +42,8 @@ void ZEND_FASTCALL zend_jit_leave_top_func_helper(uint32_t call_info); void ZEND_FASTCALL zend_jit_func_header_helper(void); void ZEND_FASTCALL zend_jit_loop_header_helper(void); void ZEND_FASTCALL zend_jit_profile_helper(void); +void ZEND_FASTCALL zend_jit_func_counter_helper(void); +void ZEND_FASTCALL zend_jit_loop_counter_helper(void); #endif /* ZEND_JIT_INTERNAL_H */ diff --git a/ext/opcache/jit/zend_jit_vm_helpers.c b/ext/opcache/jit/zend_jit_vm_helpers.c index ea6a1c2519220..948c19875f15c 100644 --- a/ext/opcache/jit/zend_jit_vm_helpers.c +++ b/ext/opcache/jit/zend_jit_vm_helpers.c @@ -125,6 +125,40 @@ void ZEND_FASTCALL zend_jit_profile_helper(void) return ((zend_vm_opcode_handler_t)handler)(); } +void ZEND_FASTCALL zend_jit_func_counter_helper(void) +{ + unsigned int n = ((uintptr_t)EX(func) >> 3) % + (sizeof(zend_jit_hot_counters) / sizeof(zend_jit_hot_counters[0])); + + zend_jit_hot_counters[n] -= ZEND_JIT_HOT_FUNC_COST; + + if (UNEXPECTED(zend_jit_hot_counters[n] <= 0)) { + zend_jit_hot_counters[n] = ZEND_JIT_HOT_COUNTER_INIT; + zend_jit_hot_func(execute_data, opline); + } else { + zend_vm_opcode_handler_t *handlers = + (zend_vm_opcode_handler_t*)ZEND_FUNC_INFO(&EX(func)->op_array); + handlers[opline - EX(func)->op_array.opcodes](); + } +} + +void ZEND_FASTCALL zend_jit_loop_counter_helper(void) +{ + unsigned int n = ((uintptr_t)EX(func) >> 3) % + (sizeof(zend_jit_hot_counters) / sizeof(zend_jit_hot_counters[0])); + + zend_jit_hot_counters[n] -= ZEND_JIT_HOT_LOOP_COST; + + if (UNEXPECTED(zend_jit_hot_counters[n] <= 0)) { + zend_jit_hot_counters[n] = ZEND_JIT_HOT_COUNTER_INIT; + zend_jit_hot_func(execute_data, opline); + } else { + zend_vm_opcode_handler_t *handlers = + (zend_vm_opcode_handler_t*)ZEND_FUNC_INFO(&EX(func)->op_array); + handlers[opline - EX(func)->op_array.opcodes](); + } +} + /* * Local variables: * tab-width: 4 From f912ff7343dc17b1c32e5aea28916bcded9f7fa0 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 27 Oct 2016 14:24:18 +0300 Subject: [PATCH 328/569] Avoid useless checks --- ext/opcache/jit/zend_jit_x86.dasc | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index d20855ea92b51..c039e5b0432ad 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -5911,10 +5911,12 @@ static int zend_jit_return(dasm_State **Dst, const zend_op *opline, zend_op_arra | jz >1 |.cold_code |1: - || if (jit_return_label >= 0) { - | IF_NOT_Z_REFCOUNTED FP + opline->op1.var, =>jit_return_label - || } else { - | IF_NOT_Z_REFCOUNTED FP + opline->op1.var, >9 + || if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { + || if (jit_return_label >= 0) { + | IF_NOT_Z_REFCOUNTED FP + opline->op1.var, =>jit_return_label + || } else { + | IF_NOT_Z_REFCOUNTED FP + opline->op1.var, >9 + || } || } | GET_Z_PTR FCARG1a, FP + opline->op1.var | GC_DELREF FCARG1a @@ -6158,7 +6160,9 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, const zend_op *opline, zend |8: if (res_info & MAY_BE_REF) { | // ZVAL_COPY_UNREF - | IF_NOT_Z_REFCOUNTED r0, >2 + if (res_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { + | IF_NOT_Z_REFCOUNTED r0, >2 + } | GET_Z_PTR r1, r0 | IF_NOT_Z_TYPE r0, IS_REFERENCE, >1 | cmp dword [r1], 1 @@ -6655,7 +6659,9 @@ static int zend_jit_fetch_obj_r(dasm_State **Dst, zend_op *opline, zend_op_array | // ZVAL_COPY_UNREF if (res_info & MAY_BE_REF) { | // ZVAL_COPY_UNREF - | IF_NOT_Z_REFCOUNTED r0, >2 + if (res_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { + | IF_NOT_Z_REFCOUNTED r0, >2 + } | GET_Z_PTR r1, r0 | IF_NOT_Z_TYPE r0, IS_REFERENCE, >1 | cmp dword [r1], 1 From 2e4cb15c3f7b087057bb294ef67f682408c07ef5 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 28 Oct 2016 21:05:20 +0300 Subject: [PATCH 329/569] Implemented JIT for BOOL, BOOL_NOT, JMPZ_EX, JMPNZ_EX. --- ext/opcache/jit/zend_jit.c | 17 +- ext/opcache/jit/zend_jit_x86.dasc | 463 +++++++++++++++++++++++------- 2 files changed, 377 insertions(+), 103 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 504e48a00ddb7..fbf88aa68b6ea 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -1191,16 +1191,25 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa, const zend_op *rt_op goto jit_failure; } goto done; + case ZEND_BOOL: + case ZEND_BOOL_NOT: + if (!zend_jit_bool_jmpznz(&dasm_state, opline, b, op_array, ssa)) { + goto jit_failure; + } + goto done; case ZEND_JMPZ: case ZEND_JMPNZ: case ZEND_JMPZNZ: + case ZEND_JMPZ_EX: + case ZEND_JMPNZ_EX: if ((opline-1)->opcode == ZEND_IS_EQUAL || (opline-1)->opcode == ZEND_IS_NOT_EQUAL || (opline-1)->opcode == ZEND_IS_SMALLER || (opline-1)->opcode == ZEND_IS_SMALLER_OR_EQUAL || (opline-1)->opcode == ZEND_CASE) { /* skip */ - } else if ((opline->opcode != ZEND_JMPZNZ) && + } else if ((opline->opcode == ZEND_JMPZ || + (opline->opcode == ZEND_JMPNZ)) && ((opline-1)->opcode == ZEND_IS_IDENTICAL || (opline-1)->opcode == ZEND_IS_NOT_IDENTICAL || (opline-1)->opcode == ZEND_ISSET_ISEMPTY_VAR || @@ -1214,8 +1223,10 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa, const zend_op *rt_op if (!zend_jit_cond_jmp(&dasm_state, opline + 1, ssa->cfg.blocks[b].successors[0])) { goto jit_failure; } - } else if (!zend_jit_jmpznz(&dasm_state, opline, b, op_array, ssa)) { - goto jit_failure; + } else { + if (!zend_jit_bool_jmpznz(&dasm_state, opline, b, op_array, ssa)) { + goto jit_failure; + } } goto done; case ZEND_FETCH_DIM_R: diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index c039e5b0432ad..9b9abb33479bc 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -3918,7 +3918,41 @@ static int zend_jit_cmp_long_long(dasm_State **Dst, const zend_op *opline, int b | LONG_LOAD r0, opline->op1_type, opline->op1 | LONG_OP cmp, r0, opline->op2_type, opline->op2 } - if ((opline+1)->opcode == ZEND_JMPZ && + if (((opline+1)->opcode == ZEND_JMPZ_EX || + (opline+1)->opcode == ZEND_JMPNZ_EX) && + (opline+1)->op1_type == IS_TMP_VAR && + (opline+1)->op1.var == opline->result.var) { + switch (opline->opcode) { + case ZEND_IS_EQUAL: + case ZEND_CASE: + | sete al + break; + case ZEND_IS_NOT_EQUAL: + | setne al + break; + case ZEND_IS_SMALLER: + if (swap) { + | setg al + } else { + | setl al + } + break; + case ZEND_IS_SMALLER_OR_EQUAL: + if (swap) { + | setge al + } else { + | setle al + } + break; + default: + ZEND_ASSERT(0); + } + | movzx eax, al + | lea eax, [eax + 2] + | SET_Z_TYPE_INFO FP + opline->result.var, eax + } + if (((opline+1)->opcode == ZEND_JMPZ || + (opline+1)->opcode == ZEND_JMPZ_EX) && (opline+1)->op1_type == IS_TMP_VAR && (opline+1)->op1.var == opline->result.var) { target_label = ssa->cfg.blocks[b].successors[0]; @@ -3944,8 +3978,11 @@ static int zend_jit_cmp_long_long(dasm_State **Dst, const zend_op *opline, int b | jg => target_label } break; + default: + ZEND_ASSERT(0); } - } else if ((opline+1)->opcode == ZEND_JMPNZ && + } else if (((opline+1)->opcode == ZEND_JMPNZ || + (opline+1)->opcode == ZEND_JMPNZ_EX) && (opline+1)->op1_type == IS_TMP_VAR && (opline+1)->op1.var == opline->result.var) { target_label = ssa->cfg.blocks[b].successors[0]; @@ -3971,6 +4008,8 @@ static int zend_jit_cmp_long_long(dasm_State **Dst, const zend_op *opline, int b | jle => target_label } break; + default: + ZEND_ASSERT(0); } } else if ((opline+1)->opcode == ZEND_JMPZNZ && (opline+1)->op1_type == IS_TMP_VAR && @@ -3998,6 +4037,8 @@ static int zend_jit_cmp_long_long(dasm_State **Dst, const zend_op *opline, int b | jg => target_label } break; + default: + ZEND_ASSERT(0); } target_label = ssa->cfg.blocks[b].successors[1]; | jmp => target_label @@ -4006,13 +4047,9 @@ static int zend_jit_cmp_long_long(dasm_State **Dst, const zend_op *opline, int b case ZEND_IS_EQUAL: case ZEND_CASE: | sete al - | movzx eax, al - | add eax, 2 break; case ZEND_IS_NOT_EQUAL: | setne al - | movzx eax, al - | add eax, 2 break; case ZEND_IS_SMALLER: if (swap) { @@ -4020,8 +4057,6 @@ static int zend_jit_cmp_long_long(dasm_State **Dst, const zend_op *opline, int b } else { | setl al } - | movzx eax, al - | add eax, 2 break; case ZEND_IS_SMALLER_OR_EQUAL: if (swap) { @@ -4029,10 +4064,12 @@ static int zend_jit_cmp_long_long(dasm_State **Dst, const zend_op *opline, int b } else { | setle al } - | movzx eax, al - | add eax, 2 break; + default: + ZEND_ASSERT(0); } + | movzx eax, al + | add eax, 2 | SET_Z_TYPE_INFO FP + opline->result.var, eax } @@ -4065,6 +4102,8 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, i case ZEND_IS_SMALLER_OR_EQUAL: | ja => target_label break; + default: + ZEND_ASSERT(0); } } else if ((opline+1)->opcode == ZEND_JMPNZ && (opline+1)->op1_type == IS_TMP_VAR && @@ -4088,6 +4127,8 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, i case ZEND_IS_SMALLER_OR_EQUAL: | jna => target_label break; + default: + ZEND_ASSERT(0); } } else if ((opline+1)->opcode == ZEND_JMPZNZ && (opline+1)->op1_type == IS_TMP_VAR && @@ -4111,9 +4152,85 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, i case ZEND_IS_SMALLER_OR_EQUAL: | ja => target_label break; + default: + ZEND_ASSERT(0); } target_label = ssa->cfg.blocks[b].successors[1]; | jmp => target_label + } else if ((opline+1)->opcode == ZEND_JMPZ_EX && + (opline+1)->op1_type == IS_TMP_VAR && + (opline+1)->op1.var == opline->result.var) { + target_label = ssa->cfg.blocks[b].successors[0]; + switch (opline->opcode) { + case ZEND_IS_EQUAL: + case ZEND_CASE: + | SET_Z_TYPE_INFO FP + opline->result.var, IS_FALSE + | jp >1 + | jne => target_label + | SET_Z_TYPE_INFO FP + opline->result.var, IS_TRUE + |1: + break; + case ZEND_IS_NOT_EQUAL: + | jp >1 + | SET_Z_TYPE_INFO FP + opline->result.var, IS_TRUE + | je => target_label + |1: + | SET_Z_TYPE_INFO FP + opline->result.var, IS_FALSE + break; + case ZEND_IS_SMALLER: + | setnae al + | movzx eax, al + | lea eax, [eax + 2] + | SET_Z_TYPE_INFO FP + opline->result.var, eax + | jae => target_label + break; + case ZEND_IS_SMALLER_OR_EQUAL: + | setna al + | movzx eax, al + | lea eax, [eax + 2] + | SET_Z_TYPE_INFO FP + opline->result.var, eax + | ja => target_label + break; + default: + ZEND_ASSERT(0); + } + } else if ((opline+1)->opcode == ZEND_JMPNZ_EX && + (opline+1)->op1_type == IS_TMP_VAR && + (opline+1)->op1.var == opline->result.var) { + target_label = ssa->cfg.blocks[b].successors[0]; + switch (opline->opcode) { + case ZEND_IS_EQUAL: + case ZEND_CASE: + | jp >1 + | SET_Z_TYPE_INFO FP + opline->result.var, IS_TRUE + | je => target_label + |1: + | SET_Z_TYPE_INFO FP + opline->result.var, IS_FALSE + break; + case ZEND_IS_NOT_EQUAL: + | SET_Z_TYPE_INFO FP + opline->result.var, IS_TRUE + | jp >1 + | jne => target_label + | SET_Z_TYPE_INFO FP + opline->result.var, IS_FALSE + |1: + break; + case ZEND_IS_SMALLER: + | setnae al + | movzx eax, al + | lea eax, [eax + 2] + | SET_Z_TYPE_INFO FP + opline->result.var, eax + | jnae => target_label + break; + case ZEND_IS_SMALLER_OR_EQUAL: + | setna al + | movzx eax, al + | lea eax, [eax + 2] + | SET_Z_TYPE_INFO FP + opline->result.var, eax + | jna => target_label + break; + default: + ZEND_ASSERT(0); + } } else { switch (opline->opcode) { case ZEND_IS_EQUAL: @@ -4143,6 +4260,8 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, i | movzx eax, al | add eax, 2 break; + default: + ZEND_ASSERT(0); } | SET_Z_TYPE_INFO FP + opline->result.var, eax } @@ -4201,7 +4320,33 @@ static int zend_jit_cmp_slow(dasm_State **Dst, const zend_op *opline, int b, zen unsigned int target_label; | cmp aword [FP + opline->result.var], 0 - if ((opline+1)->opcode == ZEND_JMPZ && + if (((opline+1)->opcode == ZEND_JMPZ_EX || + (opline+1)->opcode == ZEND_JMPNZ_EX) && + (opline+1)->op1_type == IS_TMP_VAR && + (opline+1)->op1.var == opline->result.var) { + switch (opline->opcode) { + case ZEND_IS_EQUAL: + case ZEND_CASE: + | sete al + break; + case ZEND_IS_NOT_EQUAL: + | setne al + break; + case ZEND_IS_SMALLER: + | setl al + break; + case ZEND_IS_SMALLER_OR_EQUAL: + | setle al + break; + default: + ZEND_ASSERT(0); + } + | movzx eax, al + | lea eax, [eax + 2] + | SET_Z_TYPE_INFO FP + opline->result.var, eax + } + if (((opline+1)->opcode == ZEND_JMPZ || + (opline+1)->opcode == ZEND_JMPZ_EX) && (opline+1)->op1_type == IS_TMP_VAR && (opline+1)->op1.var == opline->result.var) { target_label = ssa->cfg.blocks[b].successors[0]; @@ -4219,8 +4364,11 @@ static int zend_jit_cmp_slow(dasm_State **Dst, const zend_op *opline, int b, zen case ZEND_IS_SMALLER_OR_EQUAL: | jg => target_label break; + default: + ZEND_ASSERT(0); } - } else if ((opline+1)->opcode == ZEND_JMPNZ && + } else if (((opline+1)->opcode == ZEND_JMPNZ || + (opline+1)->opcode == ZEND_JMPNZ_EX) && (opline+1)->op1_type == IS_TMP_VAR && (opline+1)->op1.var == opline->result.var) { target_label = ssa->cfg.blocks[b].successors[0]; @@ -4238,6 +4386,8 @@ static int zend_jit_cmp_slow(dasm_State **Dst, const zend_op *opline, int b, zen case ZEND_IS_SMALLER_OR_EQUAL: | jle => target_label break; + default: + ZEND_ASSERT(0); } } else if ((opline+1)->opcode == ZEND_JMPZNZ && (opline+1)->op1_type == IS_TMP_VAR && @@ -4257,6 +4407,8 @@ static int zend_jit_cmp_slow(dasm_State **Dst, const zend_op *opline, int b, zen case ZEND_IS_SMALLER_OR_EQUAL: | jg => target_label break; + default: + ZEND_ASSERT(0); } target_label = ssa->cfg.blocks[b].successors[1]; | jmp => target_label @@ -4265,25 +4417,21 @@ static int zend_jit_cmp_slow(dasm_State **Dst, const zend_op *opline, int b, zen case ZEND_IS_EQUAL: case ZEND_CASE: | sete al - | movzx eax, al - | add eax, 2 break; case ZEND_IS_NOT_EQUAL: | setne al - | movzx eax, al - | add eax, 2 break; case ZEND_IS_SMALLER: | setl al - | movzx eax, al - | add eax, 2 break; case ZEND_IS_SMALLER_OR_EQUAL: | setle al - | movzx eax, al - | add eax, 2 break; + default: + ZEND_ASSERT(0); } + | movzx eax, al + | add eax, 2 | SET_Z_TYPE_INFO FP + opline->result.var, eax } @@ -4497,6 +4645,8 @@ static int zend_jit_cmp(dasm_State **Dst, const zend_op *opline, int b, int *opn |6: if (((opline+1)->opcode == ZEND_JMPZ || (opline+1)->opcode == ZEND_JMPNZ || + (opline+1)->opcode == ZEND_JMPZ_EX || + (opline+1)->opcode == ZEND_JMPNZ_EX || (opline+1)->opcode == ZEND_JMPZNZ) && (opline+1)->op1_type == IS_TMP_VAR && (opline+1)->op1.var == opline->result.var) { @@ -4506,122 +4656,198 @@ static int zend_jit_cmp(dasm_State **Dst, const zend_op *opline, int b, int *opn return 1; } -static int zend_jit_jmpznz(dasm_State **Dst, const zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa) +static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa) { uint32_t op1_info = OP1_INFO(); - uint32_t true_label, false_label; + uint32_t true_label = -1; + uint32_t false_label = -1; + zend_bool set_bool = 0; + zend_bool set_bool_not = 0; + zend_bool jmp_done = 0; if (opline->opcode == ZEND_JMPZ) { - true_label = (uint32_t)-1; false_label = ssa->cfg.blocks[b].successors[0]; } else if (opline->opcode == ZEND_JMPNZ) { true_label = ssa->cfg.blocks[b].successors[0]; - false_label = (uint32_t)-1; } else if (opline->opcode == ZEND_JMPZNZ) { true_label = ssa->cfg.blocks[b].successors[1]; false_label = ssa->cfg.blocks[b].successors[0]; + } else if (opline->opcode == ZEND_BOOL) { + set_bool = 1; + } else if (opline->opcode == ZEND_BOOL_NOT) { + set_bool = 1; + set_bool_not = 1; + } else if (opline->opcode == ZEND_JMPZ_EX) { + set_bool = 1; + false_label = ssa->cfg.blocks[b].successors[0]; + } else if (opline->opcode == ZEND_JMPNZ_EX) { + set_bool = 1; + true_label = ssa->cfg.blocks[b].successors[0]; } else { ZEND_ASSERT(0); } - if ((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY)) == MAY_BE_TRUE) { - if (true_label != (uint32_t)-1) { - | jmp =>true_label; - } - } else if (!(op1_info & (MAY_BE_ANY - (MAY_BE_NULL|MAY_BE_FALSE)))) { - if (op1_info & MAY_BE_UNDEF) { - if (op1_info & MAY_BE_ANY) { - | IF_Z_TYPE FP + opline->op1.var, IS_UNDEF, >1 - |.cold_code - |1: - } - | mov FCARG1d, opline->op1.var - | SAVE_VALID_OPLINE opline - | EXT_CALL zend_jit_undefined_op_helper, r0 - if (zend_may_throw(opline, op_array, ssa)) { - if (!zend_jit_check_exception(Dst)) { - return 0; - } - } - if (op1_info & MAY_BE_ANY) { - if (false_label != (uint32_t)-1) { - | jmp =>false_label; - } else { - | jmp >9 - } - |.code - } - } - if (false_label != (uint32_t)-1) { - | jmp =>false_label; - } - } else { + if (op1_info & MAY_BE_REF) { | LOAD_ZVAL_ADDR FCARG1a, opline->op1_type, opline->op1 | ZVAL_DEREF FCARG1a, op1_info + } - if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE)) { - if (op1_info & MAY_BE_TRUE) { - if (true_label != (uint32_t)-1) { - | IF_Z_TYPE FCARG1a, IS_TRUE, =>true_label - } else if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) { - | IF_Z_TYPE FCARG1a, IS_TRUE, >9 + if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE)) { + if (!(op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)-MAY_BE_TRUE))) { + /* Always TRUE */ + if (set_bool) { + if (set_bool_not) { + | SET_Z_TYPE_INFO FP + opline->result.var, IS_FALSE } else { - | //IF_Z_TYPE FCARG1a, IS_TRUE, >9 - | cmp byte [FCARG1a + 8], IS_TRUE + | SET_Z_TYPE_INFO FP + opline->result.var, IS_TRUE + } + } + if (true_label != (uint32_t)-1) { + | jmp =>true_label; + } + } else { + if (!(op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE)))) { + /* Always FASLE */ + if (set_bool) { + if (set_bool_not) { + | SET_Z_TYPE_INFO FP + opline->result.var, IS_TRUE + } else { + | SET_Z_TYPE_INFO FP + opline->result.var, IS_FALSE + } } } else { - | cmp byte [FCARG1a + 8], IS_TRUE + if (op1_info & MAY_BE_REF) { + | cmp byte [FCARG1a + 8], IS_TRUE + } else { + | cmp byte [FP + opline->op1.var + 8], IS_TRUE + } + if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) { + if ((op1_info & MAY_BE_LONG) && + !(op1_info & MAY_BE_UNDEF) && + !set_bool) { + if (false_label != (uint32_t)-1) { + | jl =>false_label + } else { + | jl >9 + } + jmp_done = 1; + } else { + | jg >2 + } + } + if (!(op1_info & MAY_BE_TRUE)) { + /* It's FALSE */ + if (set_bool) { + if (set_bool_not) { + | SET_Z_TYPE_INFO FP + opline->result.var, IS_TRUE + } else { + | SET_Z_TYPE_INFO FP + opline->result.var, IS_FALSE + } + } + } else { + if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) { + if (set_bool) { + | jne >1 + | SET_Z_TYPE_INFO FP + opline->result.var, IS_TRUE + if (true_label != (uint32_t)-1) { + | jmp =>true_label + } else { + | jmp >9 + } + |1: + | SET_Z_TYPE_INFO FP + opline->result.var, IS_FALSE + } else { + if (true_label != (uint32_t)-1) { + | je =>true_label + } else if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_LONG))) { + | jne =>false_label + jmp_done = 1; + } else { + | je >9 + } + } + } else if (set_bool) { + | sete al + | movzx eax, al + if (set_bool_not) { + | neg eax + | add eax, 3 + } else { + | add eax, 2 + } + | SET_Z_TYPE_INFO FP + opline->result.var, eax + } + } } + /* It's FALSE, but may be UNDEF */ if (op1_info & MAY_BE_UNDEF) { - if (op1_info & (MAY_BE_ANY - (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) { - | jg >2 - } - | IF_Z_TYPE FCARG1a, IS_UNDEF, >1 - if (false_label != (uint32_t)-1) { - | jmp =>false_label - } else if (op1_info & (MAY_BE_ANY - (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) { - | jmp >9 + if (op1_info & MAY_BE_ANY) { + if (op1_info & MAY_BE_REF) { + | IF_Z_TYPE FCARG1a, IS_UNDEF, >1 + } else { + | IF_Z_TYPE FP + opline->op1.var, IS_UNDEF, >1 + } + |.cold_code + |1: } - |.cold_code - |1: | mov FCARG1d, opline->op1.var | SAVE_VALID_OPLINE opline | EXT_CALL zend_jit_undefined_op_helper, r0 + if (zend_may_throw(opline, op_array, ssa)) { if (!zend_jit_check_exception(Dst)) { return 0; } } + if (false_label != (uint32_t)-1) { | jmp =>false_label - } else { - | jmp >9 } - |.code - } else if (op1_info & (MAY_BE_NULL|MAY_BE_FALSE)) { - if (false_label != (uint32_t)-1) { - | jl =>false_label - if (!(op1_info & MAY_BE_LONG) && - (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)))) { - | jmp >2 - } - } else if (op1_info & (MAY_BE_ANY - (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) { - if (op1_info & MAY_BE_LONG) { - | jl >9 - } else { - | jg >2 + if (op1_info & MAY_BE_ANY) { + if (false_label == (uint32_t)-1) { + | jmp >9 } + |.code + } + } + + if (!jmp_done) { + if (false_label != (uint32_t)-1) { + | jmp =>false_label + } else if (op1_info & MAY_BE_LONG) { + | jmp >9 } } } + } - if (op1_info & MAY_BE_LONG) { - |2: - if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG))) { + if (op1_info & MAY_BE_LONG) { + |2: + if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG))) { + if (op1_info & MAY_BE_REF) { | IF_NOT_Z_TYPE FCARG1a, IS_LONG, >2 + } else { + | IF_NOT_Z_TYPE FP + opline->op1.var, IS_LONG, >2 } + } + if (op1_info & MAY_BE_REF) { | cmp aword [FCARG1a], 0 + } else { + | cmp aword [FP + opline->op1.var], 0 + } + if (set_bool) { + | setne al + | movzx eax, al + if (set_bool_not) { + | neg eax + | add eax, 3 + } else { + | lea eax, [eax + 2] + } + | SET_Z_TYPE_INFO FP + opline->result.var, eax + } + if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) { if (true_label != (uint32_t)-1) { | jne =>true_label if (false_label != (uint32_t)-1) { @@ -4631,14 +4857,51 @@ static int zend_jit_jmpznz(dasm_State **Dst, const zend_op *opline, int b, zend_ | je =>false_label } } + } - if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG))) { + if (op1_info & (MAY_BE_ANY - (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG))) { + if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { + |.cold_code + |2: + } + if (!(op1_info & MAY_BE_REF)) { + | LOAD_ZVAL_ADDR FCARG1a, opline->op1_type, opline->op1 + } + | SAVE_VALID_OPLINE opline + | EXT_CALL zend_is_true, r0 + + if (set_bool) { + if (set_bool_not) { + | neg eax + | add eax, 3 + } else { + | add eax, 2 + } + | SET_Z_TYPE_INFO FP + opline->result.var, eax + | FREE_OP opline->op1_type, opline->op1, op1_info, 1, op_array, opline->lineno + if (zend_may_throw(opline, op_array, ssa)) { + if (!zend_jit_check_exception(Dst)) { + return 0; + } + } + if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) { + | cmp dword [FP + opline->result.var + 8], IS_FALSE + if (true_label != (uint32_t)-1) { + | jne =>true_label + if (false_label != (uint32_t)-1) { + | jmp =>false_label + } else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { + | jmp >9 + } + } else { + | je =>false_label + } + } if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { - |.cold_code - |2: + | jmp >9 + |.code } - | SAVE_VALID_OPLINE opline - | EXT_CALL zend_is_true, r0 + } else { if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { @@ -4677,10 +4940,10 @@ static int zend_jit_jmpznz(dasm_State **Dst, const zend_op *opline, int b, zend_ |.code } } - - |9: } + |9: + return 1; } From 519fd715b3543dfdc6d91382b657c0ff98857c08 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Sun, 30 Oct 2016 14:20:33 +0800 Subject: [PATCH 330/569] Fixed section setting --- ext/opcache/jit/zend_jit_x86.dasc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 9b9abb33479bc..a09da8a8a587f 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -4878,7 +4878,7 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, int b, | add eax, 2 } | SET_Z_TYPE_INFO FP + opline->result.var, eax - | FREE_OP opline->op1_type, opline->op1, op1_info, 1, op_array, opline->lineno + | FREE_OP opline->op1_type, opline->op1, op1_info, !(op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)), op_array, opline->lineno if (zend_may_throw(opline, op_array, ssa)) { if (!zend_jit_check_exception(Dst)) { return 0; From e5b5fc4751863cebdc41189e725e83cd67750c00 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Sun, 30 Oct 2016 21:01:59 +0800 Subject: [PATCH 331/569] Implemented JIT for ZEND_FETCH_OBJ_R --- ext/opcache/jit/zend_jit.c | 2 - ext/opcache/jit/zend_jit_disasm_x86.c | 1 + ext/opcache/jit/zend_jit_helpers.c | 27 +++-- ext/opcache/jit/zend_jit_x86.dasc | 137 ++++++++++++++++---------- 4 files changed, 105 insertions(+), 62 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index fbf88aa68b6ea..1f9e43ce6ddf4 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -1240,13 +1240,11 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa, const zend_op *rt_op goto jit_failure; } goto done; - /* case ZEND_FETCH_OBJ_R: if (!zend_jit_fetch_obj_r(&dasm_state, opline, op_array, ssa)) { goto jit_failure; } goto done; - */ case ZEND_BIND_GLOBAL: if (!zend_jit_bind_global(&dasm_state, opline, op_array, ssa)) { goto jit_failure; diff --git a/ext/opcache/jit/zend_jit_disasm_x86.c b/ext/opcache/jit/zend_jit_disasm_x86.c index c08349dd57b7f..575cec48cbf73 100644 --- a/ext/opcache/jit/zend_jit_disasm_x86.c +++ b/ext/opcache/jit/zend_jit_disasm_x86.c @@ -423,6 +423,7 @@ static int zend_jit_disasm_init(void) REGISTER_HELPER(zend_jit_fetch_global_helper); REGISTER_HELPER(zend_jit_verify_arg_object); REGISTER_HELPER(zend_jit_verify_arg_slow); + REGISTER_HELPER(zend_jit_fetch_obj_r_slow); #undef REGISTER_HELPER zend_elf_load_symbols(); diff --git a/ext/opcache/jit/zend_jit_helpers.c b/ext/opcache/jit/zend_jit_helpers.c index 8f27db9506cd3..268718c40d9ff 100644 --- a/ext/opcache/jit/zend_jit_helpers.c +++ b/ext/opcache/jit/zend_jit_helpers.c @@ -1306,12 +1306,6 @@ static void ZEND_FASTCALL zend_jit_free_call_frame(zend_execute_data *call) zend_vm_stack_free_call_frame(call); } -static void ZEND_FASTCALL zend_jit_zval_copy_unref_helper(zval *dst, zval *src) -{ - ZVAL_UNREF(src); - ZVAL_COPY(dst, src); -} - static zval* ZEND_FASTCALL zend_jit_new_ref_helper(zval *value) { zend_reference *ref = (zend_reference*)emalloc(sizeof(zend_reference)); @@ -1591,6 +1585,27 @@ static void ZEND_FASTCALL zend_jit_verify_arg_slow(zval *arg, zend_op_array *op_ zend_verify_arg_error((zend_function*)op_array, arg_info, arg_num, ce, arg); } +static void ZEND_FASTCALL zend_jit_zval_copy_unref_helper(zval *dst, zval *src) +{ + ZVAL_UNREF(src); + ZVAL_COPY(dst, src); +} + +static void ZEND_FASTCALL zend_jit_fetch_obj_r_slow(zval *container, zval *offset, zval *result) +{ + zval *retval; + zend_object *zobj = Z_OBJ_P(container); + if (UNEXPECTED(zobj->handlers->read_property == NULL)) { + zend_error(E_NOTICE, "Trying to get property of non-object"); + ZVAL_NULL(result); + } else { + retval = zobj->handlers->read_property(container, offset, BP_VAR_R, NULL, result); + if (retval != result) { + ZVAL_COPY_UNREF(result, retval); + } + } +} + /* * Local variables: * tab-width: 4 diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index a09da8a8a587f..ee691e2ebeea5 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -1615,6 +1615,25 @@ static int zend_jit_cannot_add_element_stub(dasm_State **Dst) return 1; } +static int zend_jit_not_obj_stub(dasm_State **Dst) +{ + |->no_obj: + |8: + |.if X64 + | mov CARG1, 0 + | LOAD_ADDR CARG2, "Using $this when not in object context" + | EXT_CALL zend_throw_error, r0 + |.else + | sub r4, 8 + | push "Using $this when not in object context" + | push 0 + | EXT_CALL zend_throw_error, r0 + | add r4, 16 + |.endif + | jmp ->exception_handler + return 1; +} + static const zend_jit_stub zend_jit_stubs[] = { JIT_STUB(interrupt_handler), JIT_STUB(exception_handler), @@ -1628,6 +1647,7 @@ static const zend_jit_stub zend_jit_stubs[] = { JIT_STUB(undefined_offset_ex), JIT_STUB(undefined_index_ex), JIT_STUB(cannot_add_element_ex), + JIT_STUB(not_obj), }; static int zend_jit_align_func(dasm_State **Dst) @@ -6872,25 +6892,32 @@ static int zend_jit_recv_init(dasm_State **Dst, const zend_op *opline, zend_op_a static int zend_jit_fetch_obj_r(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) { - uint32_t op1_info, res_info; - zend_ssa_var_info *op1_ssa; + uint32_t op1_info; zval *member, *zv; + zend_class_entry *ce; zend_property_info *info; + zend_ssa_var_info *op1_ssa; - /* TODO: supports IS_UNUSED? */ - if (opline->op1_type != IS_CV || opline->op2_type != IS_CONST || !ssa->ops || !ssa->var_info) { + if (opline->op2_type != IS_CONST || !ssa->ops || !ssa->var_info) { goto fallback; } - res_info = RES_INFO(); - op1_info = OP1_INFO(); - if ((op1_info & MAY_BE_ANY) != MAY_BE_OBJECT) { - goto fallback; - } + if (opline->op1_type == IS_UNUSED) { + if (!op_array->scope) { + goto fallback; + } + ce = op_array->scope; + } else { + op1_info = OP1_INFO(); + if ((op1_info & MAY_BE_ANY) != MAY_BE_OBJECT) { + goto fallback; + } - op1_ssa = &ssa->var_info[ssa->ops[opline - op_array->opcodes].op1_use]; - if (!op1_ssa->ce || op1_ssa->is_instanceof || op1_ssa->ce->create_object) { - goto fallback; + op1_ssa = &ssa->var_info[ssa->ops[opline - op_array->opcodes].op1_use]; + if (!op1_ssa->ce || op1_ssa->is_instanceof || op1_ssa->ce->create_object) { + goto fallback; + } + ce = op1_ssa->ce; } member = RT_CONSTANT(op_array, opline->op2); @@ -6898,69 +6925,71 @@ static int zend_jit_fetch_obj_r(dasm_State **Dst, zend_op *opline, zend_op_array goto fallback; } - zv = zend_hash_find(&op1_ssa->ce->properties_info, Z_STR_P(member)); + zv = zend_hash_find(&ce->properties_info, Z_STR_P(member)); if (zv == NULL) { goto fallback; } info = (zend_property_info*)Z_PTR_P(zv); - if (!(info->flags & ZEND_ACC_PUBLIC) || info->offset == ZEND_WRONG_PROPERTY_OFFSET) { + if (info->offset == ZEND_WRONG_PROPERTY_OFFSET) { + goto fallback; + } + if (!(info->flags & ZEND_ACC_PUBLIC) && (opline->op2_type != IS_UNUSED || info->ce != ce)) { goto fallback; } - if (opline->op1_type == IS_CV) { - | lea r0, [FP + opline->op1.var] - | ZVAL_DEREF r0, op1_info + if (opline->op1_type == IS_UNUSED) { + | lea r1, EX->This + | IF_Z_TYPE r1, IS_UNDEF, ->not_obj } else { - | lea r0, EX->This - } - if (op1_info & MAY_BE_UNDEF) { - | IF_Z_TYPE r0, IS_UNDEF, >8 + | lea r1, [FP + opline->op1.var] + | ZVAL_DEREF r1, op1_info } - | mov r0, [r0] + | mov r0, [r1] | lea r0, [r0 + info->offset] - | // ZVAL_COPY_UNREF - if (res_info & MAY_BE_REF) { - | // ZVAL_COPY_UNREF - if (res_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { - | IF_NOT_Z_REFCOUNTED r0, >2 - } + | mov edx, dword [r0 + 8] + | IF_TYPE dl, IS_UNDEF, >5 | GET_Z_PTR r1, r0 - | IF_NOT_Z_TYPE r0, IS_REFERENCE, >1 - | cmp dword [r1], 1 - | jne >1 - | lea FCARG1a, [FP + opline->result.var] - | mov FCARG2a, r0 - | EXT_CALL zend_jit_zval_copy_unref_helper, r0 - | jmp >9 + | IF_NOT_REFCOUNTED dh, >2 + | IF_TYPE dl, IS_REFERENCE, >6 |1: | GC_ADDREF r1 |2: - | ZVAL_COPY_VALUE FP + opline->result.var, r0, MAY_BE_ANY, r1, ecx, r2 - } else { - | // ZVAL_COPY - | ZVAL_COPY_VALUE FP + opline->result.var, r0, MAY_BE_ANY, r1, ecx, r2 - | TRY_ADDREF res_info, ch, r2 - } - |9: // END + |.if X64 + | SET_Z_PTR FP + opline->result.var, r1 + |.else + | SET_Z_PTR FP + opline->result.var, r1 + | GET_Z_W2 r1, r0 + | SET_Z_W2 FP + opline->result.var, r1 + |.endif + | SET_Z_TYPE_INFO FP + opline->result.var, edx + |4: // END + + | FREE_OP opline->op1_type, opline->op1, op1_info, 1, op_array, opline->lineno - if (op1_info & MAY_BE_UNDEF) { |.cold_code - |8: + |5: + | mov FCARG1a, r1 + | LOAD_ADDR FCARG2a, member |.if X64 - | mov CARG1, 0 - | LOAD_ADDR CARG2, "Using $this when not in object context" - | EXT_CALL zend_throw_error, r0 + | lea CARG3, [FP + opline->result.var] + | EXT_CALL zend_jit_fetch_obj_r_slow, r0 |.else - | sub r4, 8 - | push "Using $this when not in object context" - | push 0 - | EXT_CALL zend_throw_error, r0 + | sub r4, 12 + | lea r0, [FP + opline->result.var] + | push r0 + | EXT_CALL zend_jit_fetch_obj_r_slow, r0 | add r4, 16 |.endif - | jmp <9 - |.code - } + | jmp <4 + |6: + | cmp dword [r1], 1 + | jne <1 + | lea FCARG1a, [FP + opline->result.var] + | mov FCARG2a, r0 + | EXT_CALL zend_jit_zval_copy_unref_helper, r0 + | jmp <4 + |.code; return 1; From c448d8c7e9f58e67d4853931aca6f76a2de43cac Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Mon, 31 Oct 2016 16:53:05 +0800 Subject: [PATCH 332/569] Fixed typo & cleanup --- ext/opcache/jit/zend_jit_x86.dasc | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index ee691e2ebeea5..607be3fd131fb 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -6893,7 +6893,7 @@ static int zend_jit_recv_init(dasm_State **Dst, const zend_op *opline, zend_op_a static int zend_jit_fetch_obj_r(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) { uint32_t op1_info; - zval *member, *zv; + zval *member; zend_class_entry *ce; zend_property_info *info; zend_ssa_var_info *op1_ssa; @@ -6909,7 +6909,7 @@ static int zend_jit_fetch_obj_r(dasm_State **Dst, zend_op *opline, zend_op_array ce = op_array->scope; } else { op1_info = OP1_INFO(); - if ((op1_info & MAY_BE_ANY) != MAY_BE_OBJECT) { + if ((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != MAY_BE_OBJECT) { goto fallback; } @@ -6925,16 +6925,12 @@ static int zend_jit_fetch_obj_r(dasm_State **Dst, zend_op *opline, zend_op_array goto fallback; } - zv = zend_hash_find(&ce->properties_info, Z_STR_P(member)); - if (zv == NULL) { + info = (zend_property_info*)zend_hash_find_ptr(&ce->properties_info, Z_STR_P(member)); + if (info == NULL || info->offset == ZEND_WRONG_PROPERTY_OFFSET) { goto fallback; } - info = (zend_property_info*)Z_PTR_P(zv); - if (info->offset == ZEND_WRONG_PROPERTY_OFFSET) { - goto fallback; - } - if (!(info->flags & ZEND_ACC_PUBLIC) && (opline->op2_type != IS_UNUSED || info->ce != ce)) { + if (!(info->flags & ZEND_ACC_PUBLIC) && (opline->op1_type != IS_UNUSED || info->ce != ce)) { goto fallback; } From 4b0d168bbf9904299d694b65155ee5931179879c Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 31 Oct 2016 13:10:40 +0300 Subject: [PATCH 333/569] Removed "dead" jmp --- ext/opcache/jit/zend_jit_x86.dasc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 607be3fd131fb..71bc61d529191 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -6123,7 +6123,9 @@ static int zend_jit_leave_func(dasm_State **Dst, const zend_op *opline, zend_op_ } | // OBJ_RELEASE(object); | OBJ_RELEASE r0, ecx, >4 - | jmp >4 + if (op_array->fn_flags & (ZEND_ACC_CLOSURE|ZEND_ACC_STATIC)) { + | jmp >4 + } if (op_array->fn_flags & ZEND_ACC_STATIC) { |.code } else { From 66fd926e70d7f286bf2c30b7a71b680915097541 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Mon, 31 Oct 2016 18:25:38 +0800 Subject: [PATCH 334/569] Disable fetch_obj_r for now --- ext/opcache/jit/zend_jit.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 1f9e43ce6ddf4..fbf88aa68b6ea 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -1240,11 +1240,13 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa, const zend_op *rt_op goto jit_failure; } goto done; + /* case ZEND_FETCH_OBJ_R: if (!zend_jit_fetch_obj_r(&dasm_state, opline, op_array, ssa)) { goto jit_failure; } goto done; + */ case ZEND_BIND_GLOBAL: if (!zend_jit_bind_global(&dasm_state, opline, op_array, ssa)) { goto jit_failure; From 9dd1655530bfc5d41fb9ace562f6d8d6686f3bb8 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 31 Oct 2016 14:22:49 +0300 Subject: [PATCH 335/569] Fixed support for references --- ext/opcache/jit/zend_jit_helpers.c | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/ext/opcache/jit/zend_jit_helpers.c b/ext/opcache/jit/zend_jit_helpers.c index 268718c40d9ff..5478ab688112d 100644 --- a/ext/opcache/jit/zend_jit_helpers.c +++ b/ext/opcache/jit/zend_jit_helpers.c @@ -229,6 +229,12 @@ static void ZEND_FASTCALL zend_jit_fetch_dim_r_helper(zend_array *ht, zval *dim, } switch (Z_TYPE_P(dim)) { + case IS_LONG: + hval = Z_LVAL_P(dim); + goto num_index; + case IS_STRING: + offset_key = Z_STR_P(dim); + goto str_index; case IS_UNDEF: zend_jit_undefined_op_helper(EG(current_execute_data)->opline->op2.var); /* break missing intentionally */ @@ -295,6 +301,12 @@ static void ZEND_FASTCALL zend_jit_fetch_dim_is_helper(zend_array *ht, zval *dim } switch (Z_TYPE_P(dim)) { + case IS_LONG: + hval = Z_LVAL_P(dim); + goto num_index; + case IS_STRING: + offset_key = Z_STR_P(dim); + goto str_index; case IS_UNDEF: zend_jit_undefined_op_helper(EG(current_execute_data)->opline->op2.var); /* break missing intentionally */ @@ -358,6 +370,12 @@ static int ZEND_FASTCALL zend_jit_fetch_dim_isset_helper(zend_array *ht, zval *d } switch (Z_TYPE_P(dim)) { + case IS_LONG: + hval = Z_LVAL_P(dim); + goto num_index; + case IS_STRING: + offset_key = Z_STR_P(dim); + goto str_index; case IS_UNDEF: zend_jit_undefined_op_helper(EG(current_execute_data)->opline->op2.var); /* break missing intentionally */ @@ -419,6 +437,12 @@ static zval* ZEND_FASTCALL zend_jit_fetch_dim_rw_helper(zend_array *ht, zval *di } switch (Z_TYPE_P(dim)) { + case IS_LONG: + hval = Z_LVAL_P(dim); + goto num_index; + case IS_STRING: + offset_key = Z_STR_P(dim); + goto str_index; case IS_UNDEF: zend_jit_undefined_op_helper(EG(current_execute_data)->opline->op2.var); /* break missing intentionally */ @@ -481,6 +505,12 @@ static zval* ZEND_FASTCALL zend_jit_fetch_dim_w_helper(zend_array *ht, zval *dim } switch (Z_TYPE_P(dim)) { + case IS_LONG: + hval = Z_LVAL_P(dim); + goto num_index; + case IS_STRING: + offset_key = Z_STR_P(dim); + goto str_index; case IS_UNDEF: zend_jit_undefined_op_helper(EG(current_execute_data)->opline->op2.var); /* break missing intentionally */ From 32db99ed0fb736f0b59751d4b8c38a87dfc4f215 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 31 Oct 2016 19:17:53 +0300 Subject: [PATCH 336/569] Fixed incorrect JIT code-genertion for TYPE_CHECK opcode --- ext/opcache/jit/zend_jit_x86.dasc | 24 ++++-------------------- 1 file changed, 4 insertions(+), 20 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 71bc61d529191..02a8ef0891154 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -5839,20 +5839,12 @@ static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, int b, i (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { | // if (Z_REFCOUNTED_P(cv)) { - if (op1_info & MAY_BE_REF) { - | IF_Z_REFCOUNTED r0, >1 - } else { - | IF_Z_REFCOUNTED FP + opline->op1.var, >1 - } + | IF_Z_REFCOUNTED FP + opline->op1.var, >1 |.cold_code |1: } | // if (!Z_DELREF_P(cv)) { - if (op1_info & MAY_BE_REF) { - | GET_Z_PTR FCARG1a, r0 - } else { - | GET_Z_PTR FCARG1a, FP + opline->op1.var - } + | GET_Z_PTR FCARG1a, FP + opline->op1.var | GC_DELREF FCARG1a if (RC_MAY_BE_1(op1_info)) { if (RC_MAY_BE_N(op1_info)) { @@ -5924,20 +5916,12 @@ static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, int b, i (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { | // if (Z_REFCOUNTED_P(cv)) { - if (op1_info & MAY_BE_REF) { - | IF_Z_REFCOUNTED r0, >1 - } else { - | IF_Z_REFCOUNTED FP + opline->op1.var, >1 - } + | IF_Z_REFCOUNTED FP + opline->op1.var, >1 |.cold_code |1: } | // if (!Z_DELREF_P(cv)) { - if (op1_info & MAY_BE_REF) { - | GET_Z_PTR FCARG1a, r0 - } else { - | GET_Z_PTR FCARG1a, FP + opline->op1.var - } + | GET_Z_PTR FCARG1a, FP + opline->op1.var | GC_DELREF FCARG1a if (RC_MAY_BE_1(op1_info)) { if (RC_MAY_BE_N(op1_info)) { From 95da6f92bc9645d7609f84a3bedb377620abb179 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 31 Oct 2016 21:29:23 +0300 Subject: [PATCH 337/569] don't clobber used register --- ext/opcache/jit/zend_jit_x86.dasc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 02a8ef0891154..dea6005bdd741 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -6729,7 +6729,7 @@ static int zend_jit_bind_global(dasm_State **Dst, const zend_op *opline, zend_op if (op1_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) { |3: // GC_ZVAL_CHECK_POSSIBLE_ROOT(variable_ptr) - | IF_GC_MAY_NOT_LEAK FCARG1a, eax, >5 + | IF_GC_MAY_NOT_LEAK FCARG1a, edx, >5 | mov aword [r4], r0 //save | EXT_CALL gc_possible_root, r1 | mov r0, aword [r4] // restore From 229aec318dee21a1dafe888d36c03052b4f1d5b1 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 1 Nov 2016 01:45:39 +0300 Subject: [PATCH 338/569] Fixed JIT for ASSIGN_ADD on arrays --- ext/opcache/jit/zend_jit_x86.dasc | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index dea6005bdd741..29a05f2569372 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -2369,11 +2369,6 @@ static int zend_jit_math_helper(dasm_State **Dst, /* Labels: 1,2,3,4,5,6 */ { zend_bool same_ops = (op1_type == op2_type) && (op1.var == op2.var); - zend_bool has_slow = - (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && - (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && - ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) || - (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))); if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)) { if (op1_info & (MAY_BE_ANY-MAY_BE_LONG)) { @@ -2506,8 +2501,12 @@ static int zend_jit_math_helper(dasm_State **Dst, |5: - if (has_slow) { - |.cold_code + if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) || + (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { + if ((op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && + (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { + |.cold_code + } |6: | SAVE_VALID_OPLINE opline if (separate_op1) { @@ -2555,8 +2554,11 @@ static int zend_jit_math_helper(dasm_State **Dst, || if (zend_may_throw(opline, op_array, ssa)) { || zend_jit_check_exception(Dst); || } - | jmp <5 - |.code + if ((op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && + (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { + | jmp <5 + |.code + } } return 1; From 90685a050a0655c78a0a5ee1588310486edd491b Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 2 Nov 2016 01:43:40 +0300 Subject: [PATCH 339/569] JIT for FETCH_OBJ_R (incomplete) --- ext/opcache/jit/zend_jit.c | 4 +- ext/opcache/jit/zend_jit_x86.dasc | 177 +++++++++++++++++++++++------- 2 files changed, 137 insertions(+), 44 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index fbf88aa68b6ea..ab62c2dcc9e95 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -1230,7 +1230,7 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa, const zend_op *rt_op } goto done; case ZEND_FETCH_DIM_R: - case ZEND_FETCH_DIM_IS: + case ZEND_FETCH_DIM_IS: if (!zend_jit_fetch_dim_read(&dasm_state, opline, op_array, ssa)) { goto jit_failure; } @@ -1240,13 +1240,11 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa, const zend_op *rt_op goto jit_failure; } goto done; - /* case ZEND_FETCH_OBJ_R: if (!zend_jit_fetch_obj_r(&dasm_state, opline, op_array, ssa)) { goto jit_failure; } goto done; - */ case ZEND_BIND_GLOBAL: if (!zend_jit_bind_global(&dasm_state, opline, op_array, ssa)) { goto jit_failure; diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 29a05f2569372..2590b5f38a96e 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -1617,7 +1617,7 @@ static int zend_jit_cannot_add_element_stub(dasm_State **Dst) static int zend_jit_not_obj_stub(dasm_State **Dst) { - |->no_obj: + |->not_obj: |8: |.if X64 | mov CARG1, 0 @@ -6878,101 +6878,196 @@ static int zend_jit_recv_init(dasm_State **Dst, const zend_op *opline, zend_op_a return 1; } +static uint32_t zend_get_known_property_offset(zend_class_entry *ce, zend_string *member, zend_bool on_this, zend_string *filename) +{ + zend_property_info *info; + + if (!ce || (ce->ce_flags & ZEND_ACC_TRAIT)) { + return ZEND_WRONG_PROPERTY_OFFSET; + } + + if (ce->info.user.filename != filename) { + /* class declaration might be changed infdependently */ + return ZEND_WRONG_PROPERTY_OFFSET; + } + + if (ce->ce_flags & ZEND_ACC_INHERITED) { + if (!ce->parent) { + /* propery offests may be changed by inheritance */ + return ZEND_WRONG_PROPERTY_OFFSET; + } else { + zend_class_entry *parent = ce->parent; + + do { + if (parent->type == ZEND_INTERNAL_CLASS) { + break; + } else if (parent->info.user.filename != filename) { + /* some of parents class declarations might be changed infdependently */ + /* TODO: this check may be not enough, because even + * in the same it's possible to conditionnaly define + * few classes with the same name, and "parent" may + * change from request to request. + */ + return ZEND_WRONG_PROPERTY_OFFSET; + } + parent = parent->parent; + } while (parent); + } + } + + info = (zend_property_info*)zend_hash_find_ptr(&ce->properties_info, member); + if (info == NULL || + info->offset == ZEND_WRONG_PROPERTY_OFFSET || + (info->flags & ZEND_ACC_STATIC)) { + return ZEND_WRONG_PROPERTY_OFFSET; + } + + if (!(info->flags & ZEND_ACC_PUBLIC) && + (!on_this || info->ce != ce)) { + return ZEND_WRONG_PROPERTY_OFFSET; + } + + return info->offset; +} + static int zend_jit_fetch_obj_r(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) { uint32_t op1_info; + zend_class_entry *ce = NULL; zval *member; - zend_class_entry *ce; - zend_property_info *info; - zend_ssa_var_info *op1_ssa; + uint32_t offset; - if (opline->op2_type != IS_CONST || !ssa->ops || !ssa->var_info) { + if (opline->op2_type != IS_CONST) { + goto fallback; + } + + member = RT_CONSTANT(op_array, opline->op2); + if (Z_TYPE_P(member) != IS_STRING || Z_STRVAL_P(member)[0] == '\0') { goto fallback; } if (opline->op1_type == IS_UNUSED) { - if (!op_array->scope) { - goto fallback; - } + op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN; ce = op_array->scope; } else { op1_info = OP1_INFO(); - if ((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != MAY_BE_OBJECT) { - goto fallback; - } + if (ssa->var_info && ssa->ops) { + zend_ssa_var_info *op1_ssa = + ssa->var_info + ssa->ops[opline - op_array->opcodes].op1_use; - op1_ssa = &ssa->var_info[ssa->ops[opline - op_array->opcodes].op1_use]; - if (!op1_ssa->ce || op1_ssa->is_instanceof || op1_ssa->ce->create_object) { - goto fallback; + if (op1_ssa->ce && !op1_ssa->is_instanceof && !op1_ssa->ce->create_object) { + ce = op1_ssa->ce; + } } - ce = op1_ssa->ce; } - member = RT_CONSTANT(op_array, opline->op2); - if (Z_TYPE_P(member) != IS_STRING || Z_STRVAL_P(member)[0] == '\0') { + if (!(op1_info & MAY_BE_OBJECT)) { goto fallback; } - info = (zend_property_info*)zend_hash_find_ptr(&ce->properties_info, Z_STR_P(member)); - if (info == NULL || info->offset == ZEND_WRONG_PROPERTY_OFFSET) { - goto fallback; - } - - if (!(info->flags & ZEND_ACC_PUBLIC) && (opline->op1_type != IS_UNUSED || info->ce != ce)) { + offset = zend_get_known_property_offset(ce, Z_STR_P(member), opline->op1_type == IS_UNUSED, op_array->filename); + if (offset == ZEND_WRONG_PROPERTY_OFFSET) { + // TODO: JIT using run_time_cache ??? goto fallback; } if (opline->op1_type == IS_UNUSED) { - | lea r1, EX->This - | IF_Z_TYPE r1, IS_UNDEF, ->not_obj + | IF_Z_TYPE FP + offsetof(zend_execute_data, This), IS_UNDEF, ->not_obj + | GET_Z_PTR FCARG1a, FP + offsetof(zend_execute_data, This) + } else if (op1_info & MAY_BE_REF) { + | lea r0, [FP + opline->op1.var] + | ZVAL_DEREF r0, op1_info + | IF_NOT_Z_TYPE r0, IS_OBJECT, >7 + | GET_Z_PTR FCARG1a, r0 } else { - | lea r1, [FP + opline->op1.var] - | ZVAL_DEREF r1, op1_info + if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { + | IF_NOT_Z_TYPE FP + opline->op1.var, IS_OBJECT, >7 + } + | GET_Z_PTR FCARG1a, FP + opline->op1.var } - | mov r0, [r1] - | lea r0, [r0 + info->offset] - | mov edx, dword [r0 + 8] + + | mov edx, dword [FCARG1a + offset + 8] | IF_TYPE dl, IS_UNDEF, >5 - | GET_Z_PTR r1, r0 + | GET_Z_PTR r0, FCARG1a + offset | IF_NOT_REFCOUNTED dh, >2 | IF_TYPE dl, IS_REFERENCE, >6 |1: - | GC_ADDREF r1 + | GC_ADDREF r0 |2: |.if X64 - | SET_Z_PTR FP + opline->result.var, r1 + | SET_Z_PTR FP + opline->result.var, r0 |.else - | SET_Z_PTR FP + opline->result.var, r1 - | GET_Z_W2 r1, r0 - | SET_Z_W2 FP + opline->result.var, r1 + | SET_Z_PTR FP + opline->result.var, r0 + | GET_Z_W2 r0, FCARG1a + offset + | SET_Z_W2 FP + opline->result.var, r0 |.endif | SET_Z_TYPE_INFO FP + opline->result.var, edx |4: // END | FREE_OP opline->op1_type, opline->op1, op1_info, 1, op_array, opline->lineno + if (zend_may_throw(opline, op_array, ssa)) { + if (!zend_jit_check_exception(Dst)) { + return 0; + } + } + |.cold_code |5: - | mov FCARG1a, r1 + if (opline->op1_type == IS_UNUSED) { + | lea FCARG1a, EX->This + } else if (op1_info & MAY_BE_REF) { + | mov FCARG1a, r0 + } else { + | lea FCARG1a, [FP + opline->op1.var] + } | LOAD_ADDR FCARG2a, member |.if X64 | lea CARG3, [FP + opline->result.var] | EXT_CALL zend_jit_fetch_obj_r_slow, r0 |.else - | sub r4, 12 | lea r0, [FP + opline->result.var] | push r0 | EXT_CALL zend_jit_fetch_obj_r_slow, r0 - | add r4, 16 |.endif | jmp <4 |6: - | cmp dword [r1], 1 + | cmp dword [r0], 1 | jne <1 + | lea FCARG2a, [FCARG1a + offset] | lea FCARG1a, [FP + opline->result.var] - | mov FCARG2a, r0 | EXT_CALL zend_jit_zval_copy_unref_helper, r0 | jmp <4 + + if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)- MAY_BE_OBJECT)) { + |7: + | SAVE_VALID_OPLINE opline + if (op1_info & MAY_BE_UNDEF) { + if (op1_info & MAY_BE_ANY) { + if (op1_info & MAY_BE_REF) { + | IF_NOT_Z_TYPE FCARG1a, IS_UNDEF, >1 + } else { + | IF_NOT_Z_TYPE FP + opline->op1.var, IS_UNDEF, >1 + } + } + | mov FCARG1d, opline->op1.var + | EXT_CALL zend_jit_undefined_op_helper, r0 + |1: + } + |.if X64 + | mov CARG1, E_NOTICE + | LOAD_ADDR CARG2, "Trying to get property of non-object" + | EXT_CALL zend_error, r0 + |.else + | sub r4, 8 + | push "Trying to get property of non-object" + | push E_NOTICE + | EXT_CALL zend_error, r0 + | add r4, 26 + |.endif + | SET_Z_TYPE_INFO FP + opline->result.var, IS_NULL + | jmp <4 + } |.code; return 1; From 4460aea4d21ad51faf27ea76e2b80f13a50f676a Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 2 Nov 2016 14:02:15 +0300 Subject: [PATCH 340/569] JIT for FETCH_OBJ_R --- ext/opcache/jit/zend_jit_disasm_x86.c | 1 + ext/opcache/jit/zend_jit_helpers.c | 22 ++++++- ext/opcache/jit/zend_jit_x86.dasc | 89 +++++++++++++++++---------- 3 files changed, 77 insertions(+), 35 deletions(-) diff --git a/ext/opcache/jit/zend_jit_disasm_x86.c b/ext/opcache/jit/zend_jit_disasm_x86.c index 575cec48cbf73..da83b4c36b127 100644 --- a/ext/opcache/jit/zend_jit_disasm_x86.c +++ b/ext/opcache/jit/zend_jit_disasm_x86.c @@ -424,6 +424,7 @@ static int zend_jit_disasm_init(void) REGISTER_HELPER(zend_jit_verify_arg_object); REGISTER_HELPER(zend_jit_verify_arg_slow); REGISTER_HELPER(zend_jit_fetch_obj_r_slow); + REGISTER_HELPER(zend_jit_fetch_obj_r_dynamic); #undef REGISTER_HELPER zend_elf_load_symbols(); diff --git a/ext/opcache/jit/zend_jit_helpers.c b/ext/opcache/jit/zend_jit_helpers.c index 5478ab688112d..acdca1610c184 100644 --- a/ext/opcache/jit/zend_jit_helpers.c +++ b/ext/opcache/jit/zend_jit_helpers.c @@ -1621,21 +1621,37 @@ static void ZEND_FASTCALL zend_jit_zval_copy_unref_helper(zval *dst, zval *src) ZVAL_COPY(dst, src); } -static void ZEND_FASTCALL zend_jit_fetch_obj_r_slow(zval *container, zval *offset, zval *result) +static void ZEND_FASTCALL zend_jit_fetch_obj_r_slow(zend_object *zobj, zval *offset, zval *result) { zval *retval; - zend_object *zobj = Z_OBJ_P(container); + zend_execute_data *execute_data = EG(current_execute_data); + zval tmp; + if (UNEXPECTED(zobj->handlers->read_property == NULL)) { zend_error(E_NOTICE, "Trying to get property of non-object"); ZVAL_NULL(result); } else { - retval = zobj->handlers->read_property(container, offset, BP_VAR_R, NULL, result); + ZVAL_OBJ(&tmp, zobj); + retval = zobj->handlers->read_property(&tmp, offset, BP_VAR_R, CACHE_ADDR(Z_CACHE_SLOT_P(offset)), result); if (retval != result) { ZVAL_COPY_UNREF(result, retval); } } } +static void ZEND_FASTCALL zend_jit_fetch_obj_r_dynamic(zend_object *zobj, zval *offset, zval *result) +{ + if (zobj->properties) { + zval *retval = zend_hash_find(zobj->properties, Z_STR_P(offset)); + + if (EXPECTED(retval)) { + ZVAL_COPY_UNREF(result, retval); + return; + } + } + zend_jit_fetch_obj_r_slow(zobj, offset, result); +} + /* * Local variables: * tab-width: 4 diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 2590b5f38a96e..f7d3180479d89 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -6966,10 +6966,6 @@ static int zend_jit_fetch_obj_r(dasm_State **Dst, zend_op *opline, zend_op_array } offset = zend_get_known_property_offset(ce, Z_STR_P(member), opline->op1_type == IS_UNUSED, op_array->filename); - if (offset == ZEND_WRONG_PROPERTY_OFFSET) { - // TODO: JIT using run_time_cache ??? - goto fallback; - } if (opline->op1_type == IS_UNUSED) { | IF_Z_TYPE FP + offsetof(zend_execute_data, This), IS_UNDEF, ->not_obj @@ -6986,9 +6982,23 @@ static int zend_jit_fetch_obj_r(dasm_State **Dst, zend_op *opline, zend_op_array | GET_Z_PTR FCARG1a, FP + opline->op1.var } - | mov edx, dword [FCARG1a + offset + 8] - | IF_TYPE dl, IS_UNDEF, >5 - | GET_Z_PTR r0, FCARG1a + offset + if (offset == ZEND_WRONG_PROPERTY_OFFSET) { + | mov r0, EX->run_time_cache + | mov r2, aword [r0 + Z_CACHE_SLOT_P(member)] + | cmp r2, aword [FCARG1a + offsetof(zend_object, ce)] + | jne >5 + | mov r0, aword [r0 + Z_CACHE_SLOT_P(member) + sizeof(void*)] + | cmp r0, (uint32_t)ZEND_DYNAMIC_PROPERTY_OFFSET + | je >8 // dynamic property + | mov edx, dword [FCARG1a + r0 + 8] + | IF_TYPE dl, IS_UNDEF, >5 + | add FCARG1a, r0 + | GET_Z_PTR r0, FCARG1a + } else { + | mov edx, dword [FCARG1a + offset + 8] + | IF_TYPE dl, IS_UNDEF, >5 + | GET_Z_PTR r0, FCARG1a + offset + } | IF_NOT_REFCOUNTED dh, >2 | IF_TYPE dl, IS_REFERENCE, >6 |1: @@ -6998,46 +7008,38 @@ static int zend_jit_fetch_obj_r(dasm_State **Dst, zend_op *opline, zend_op_array | SET_Z_PTR FP + opline->result.var, r0 |.else | SET_Z_PTR FP + opline->result.var, r0 - | GET_Z_W2 r0, FCARG1a + offset + if (offset == ZEND_WRONG_PROPERTY_OFFSET) { + | GET_Z_W2 r0, FCARG1a + } else { + | GET_Z_W2 r0, FCARG1a + offset + } | SET_Z_W2 FP + opline->result.var, r0 |.endif | SET_Z_TYPE_INFO FP + opline->result.var, edx - |4: // END - - | FREE_OP opline->op1_type, opline->op1, op1_info, 1, op_array, opline->lineno - - if (zend_may_throw(opline, op_array, ssa)) { - if (!zend_jit_check_exception(Dst)) { - return 0; - } - } |.cold_code |5: - if (opline->op1_type == IS_UNUSED) { - | lea FCARG1a, EX->This - } else if (op1_info & MAY_BE_REF) { - | mov FCARG1a, r0 - } else { - | lea FCARG1a, [FP + opline->op1.var] - } | LOAD_ADDR FCARG2a, member |.if X64 | lea CARG3, [FP + opline->result.var] - | EXT_CALL zend_jit_fetch_obj_r_slow, r0 |.else | lea r0, [FP + opline->result.var] | push r0 - | EXT_CALL zend_jit_fetch_obj_r_slow, r0 |.endif - | jmp <4 + | SAVE_VALID_OPLINE opline + | EXT_CALL zend_jit_fetch_obj_r_slow, r0 + | jmp >9 |6: | cmp dword [r0], 1 | jne <1 - | lea FCARG2a, [FCARG1a + offset] + if (offset == ZEND_WRONG_PROPERTY_OFFSET) { + | mov FCARG2a, FCARG1a + } else { + | lea FCARG2a, [FCARG1a + offset] + } | lea FCARG1a, [FP + opline->result.var] | EXT_CALL zend_jit_zval_copy_unref_helper, r0 - | jmp <4 + | jmp >9 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)- MAY_BE_OBJECT)) { |7: @@ -7045,7 +7047,7 @@ static int zend_jit_fetch_obj_r(dasm_State **Dst, zend_op *opline, zend_op_array if (op1_info & MAY_BE_UNDEF) { if (op1_info & MAY_BE_ANY) { if (op1_info & MAY_BE_REF) { - | IF_NOT_Z_TYPE FCARG1a, IS_UNDEF, >1 + | IF_NOT_Z_TYPE r0, IS_UNDEF, >1 } else { | IF_NOT_Z_TYPE FP + opline->op1.var, IS_UNDEF, >1 } @@ -7063,12 +7065,35 @@ static int zend_jit_fetch_obj_r(dasm_State **Dst, zend_op *opline, zend_op_array | push "Trying to get property of non-object" | push E_NOTICE | EXT_CALL zend_error, r0 - | add r4, 26 + | add r4, 16 |.endif | SET_Z_TYPE_INFO FP + opline->result.var, IS_NULL - | jmp <4 + | jmp >9 } + + if (offset == ZEND_WRONG_PROPERTY_OFFSET) { + |8: + | LOAD_ADDR FCARG2a, member + |.if X64 + | lea CARG3, [FP + opline->result.var] + |.else + | lea r0, [FP + opline->result.var] + | push r0 + |.endif + | SAVE_VALID_OPLINE opline + | EXT_CALL zend_jit_fetch_obj_r_dynamic, r0 + | jmp >9 + } + |.code; + |9: // END + | FREE_OP opline->op1_type, opline->op1, op1_info, 1, op_array, opline->lineno + + if (zend_may_throw(opline, op_array, ssa)) { + if (!zend_jit_check_exception(Dst)) { + return 0; + } + } return 1; From 06742036a19645addfbbf495bdc511e7d9405192 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Wed, 2 Nov 2016 19:16:36 +0800 Subject: [PATCH 341/569] Fixed segfault --- ext/opcache/jit/zend_jit_x86.dasc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index f7d3180479d89..6b83eceb87f20 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -6987,8 +6987,8 @@ static int zend_jit_fetch_obj_r(dasm_State **Dst, zend_op *opline, zend_op_array | mov r2, aword [r0 + Z_CACHE_SLOT_P(member)] | cmp r2, aword [FCARG1a + offsetof(zend_object, ce)] | jne >5 - | mov r0, aword [r0 + Z_CACHE_SLOT_P(member) + sizeof(void*)] - | cmp r0, (uint32_t)ZEND_DYNAMIC_PROPERTY_OFFSET + | mov eax, dword [r0 + Z_CACHE_SLOT_P(member) + sizeof(void*)] + | cmp eax, (uint32_t)ZEND_DYNAMIC_PROPERTY_OFFSET | je >8 // dynamic property | mov edx, dword [FCARG1a + r0 + 8] | IF_TYPE dl, IS_UNDEF, >5 From 4988a96fca6ef63622ec57482ea4fa9b401bac0c Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Wed, 2 Nov 2016 19:26:59 +0800 Subject: [PATCH 342/569] This should still be r0 --- ext/opcache/jit/zend_jit_x86.dasc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 6b83eceb87f20..fe2158e888f8e 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -6987,7 +6987,7 @@ static int zend_jit_fetch_obj_r(dasm_State **Dst, zend_op *opline, zend_op_array | mov r2, aword [r0 + Z_CACHE_SLOT_P(member)] | cmp r2, aword [FCARG1a + offsetof(zend_object, ce)] | jne >5 - | mov eax, dword [r0 + Z_CACHE_SLOT_P(member) + sizeof(void*)] + | mov r0, aword [r0 + Z_CACHE_SLOT_P(member) + sizeof(void*)] | cmp eax, (uint32_t)ZEND_DYNAMIC_PROPERTY_OFFSET | je >8 // dynamic property | mov edx, dword [FCARG1a + r0 + 8] From a41d15755fb0691d20a0c7fe08af4d94d1ec11eb Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 2 Nov 2016 16:35:44 +0300 Subject: [PATCH 343/569] JIT for FETCH_OBJ_IS --- ext/opcache/jit/zend_jit.c | 3 +- ext/opcache/jit/zend_jit_disasm_x86.c | 2 + ext/opcache/jit/zend_jit_helpers.c | 30 ++++++++++ ext/opcache/jit/zend_jit_x86.dasc | 85 ++++++++++++++++----------- 4 files changed, 85 insertions(+), 35 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index ab62c2dcc9e95..685ad2cf40d0e 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -1241,7 +1241,8 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa, const zend_op *rt_op } goto done; case ZEND_FETCH_OBJ_R: - if (!zend_jit_fetch_obj_r(&dasm_state, opline, op_array, ssa)) { + case ZEND_FETCH_OBJ_IS: + if (!zend_jit_fetch_obj_read(&dasm_state, opline, op_array, ssa)) { goto jit_failure; } goto done; diff --git a/ext/opcache/jit/zend_jit_disasm_x86.c b/ext/opcache/jit/zend_jit_disasm_x86.c index da83b4c36b127..b14c81d634cee 100644 --- a/ext/opcache/jit/zend_jit_disasm_x86.c +++ b/ext/opcache/jit/zend_jit_disasm_x86.c @@ -425,6 +425,8 @@ static int zend_jit_disasm_init(void) REGISTER_HELPER(zend_jit_verify_arg_slow); REGISTER_HELPER(zend_jit_fetch_obj_r_slow); REGISTER_HELPER(zend_jit_fetch_obj_r_dynamic); + REGISTER_HELPER(zend_jit_fetch_obj_is_slow); + REGISTER_HELPER(zend_jit_fetch_obj_is_dynamic); #undef REGISTER_HELPER zend_elf_load_symbols(); diff --git a/ext/opcache/jit/zend_jit_helpers.c b/ext/opcache/jit/zend_jit_helpers.c index acdca1610c184..9d9270d9d19a9 100644 --- a/ext/opcache/jit/zend_jit_helpers.c +++ b/ext/opcache/jit/zend_jit_helpers.c @@ -1652,6 +1652,36 @@ static void ZEND_FASTCALL zend_jit_fetch_obj_r_dynamic(zend_object *zobj, zval * zend_jit_fetch_obj_r_slow(zobj, offset, result); } +static void ZEND_FASTCALL zend_jit_fetch_obj_is_slow(zend_object *zobj, zval *offset, zval *result) +{ + zval *retval; + zend_execute_data *execute_data = EG(current_execute_data); + zval tmp; + + if (UNEXPECTED(zobj->handlers->read_property == NULL)) { + ZVAL_NULL(result); + } else { + ZVAL_OBJ(&tmp, zobj); + retval = zobj->handlers->read_property(&tmp, offset, BP_VAR_IS, CACHE_ADDR(Z_CACHE_SLOT_P(offset)), result); + if (retval != result) { + ZVAL_COPY(result, retval); + } + } +} + +static void ZEND_FASTCALL zend_jit_fetch_obj_is_dynamic(zend_object *zobj, zval *offset, zval *result) +{ + if (zobj->properties) { + zval *retval = zend_hash_find(zobj->properties, Z_STR_P(offset)); + + if (EXPECTED(retval)) { + ZVAL_COPY(result, retval); + return; + } + } + zend_jit_fetch_obj_is_slow(zobj, offset, result); +} + /* * Local variables: * tab-width: 4 diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index fe2158e888f8e..013ef43e1843d 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -6930,7 +6930,7 @@ static uint32_t zend_get_known_property_offset(zend_class_entry *ce, zend_string return info->offset; } -static int zend_jit_fetch_obj_r(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) +static int zend_jit_fetch_obj_read(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) { uint32_t op1_info; zend_class_entry *ce = NULL; @@ -7000,7 +7000,9 @@ static int zend_jit_fetch_obj_r(dasm_State **Dst, zend_op *opline, zend_op_array | GET_Z_PTR r0, FCARG1a + offset } | IF_NOT_REFCOUNTED dh, >2 - | IF_TYPE dl, IS_REFERENCE, >6 + if (opline->opcode == ZEND_FETCH_OBJ_R) { + | IF_TYPE dl, IS_REFERENCE, >6 + } |1: | GC_ADDREF r0 |2: @@ -7027,46 +7029,57 @@ static int zend_jit_fetch_obj_r(dasm_State **Dst, zend_op *opline, zend_op_array | push r0 |.endif | SAVE_VALID_OPLINE opline - | EXT_CALL zend_jit_fetch_obj_r_slow, r0 - | jmp >9 - |6: - | cmp dword [r0], 1 - | jne <1 - if (offset == ZEND_WRONG_PROPERTY_OFFSET) { - | mov FCARG2a, FCARG1a + if (opline->opcode == ZEND_FETCH_OBJ_R) { + | EXT_CALL zend_jit_fetch_obj_r_slow, r0 + } else if (opline->opcode == ZEND_FETCH_OBJ_IS) { + | EXT_CALL zend_jit_fetch_obj_is_slow, r0 } else { - | lea FCARG2a, [FCARG1a + offset] + ZEND_ASSERT(0); } - | lea FCARG1a, [FP + opline->result.var] - | EXT_CALL zend_jit_zval_copy_unref_helper, r0 | jmp >9 + if (opline->opcode == ZEND_FETCH_OBJ_R) { + |6: + | cmp dword [r0], 1 + | jne <1 + if (offset == ZEND_WRONG_PROPERTY_OFFSET) { + | mov FCARG2a, FCARG1a + } else { + | lea FCARG2a, [FCARG1a + offset] + } + | lea FCARG1a, [FP + opline->result.var] + | EXT_CALL zend_jit_zval_copy_unref_helper, r0 + | jmp >9 + } + if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)- MAY_BE_OBJECT)) { |7: - | SAVE_VALID_OPLINE opline - if (op1_info & MAY_BE_UNDEF) { - if (op1_info & MAY_BE_ANY) { - if (op1_info & MAY_BE_REF) { - | IF_NOT_Z_TYPE r0, IS_UNDEF, >1 - } else { - | IF_NOT_Z_TYPE FP + opline->op1.var, IS_UNDEF, >1 + if (opline->opcode == ZEND_FETCH_OBJ_R) { + | SAVE_VALID_OPLINE opline + if (op1_info & MAY_BE_UNDEF) { + if (op1_info & MAY_BE_ANY) { + if (op1_info & MAY_BE_REF) { + | IF_NOT_Z_TYPE r0, IS_UNDEF, >1 + } else { + | IF_NOT_Z_TYPE FP + opline->op1.var, IS_UNDEF, >1 + } } + | mov FCARG1d, opline->op1.var + | EXT_CALL zend_jit_undefined_op_helper, r0 + |1: } - | mov FCARG1d, opline->op1.var - | EXT_CALL zend_jit_undefined_op_helper, r0 - |1: + |.if X64 + | mov CARG1, E_NOTICE + | LOAD_ADDR CARG2, "Trying to get property of non-object" + | EXT_CALL zend_error, r0 + |.else + | sub r4, 8 + | push "Trying to get property of non-object" + | push E_NOTICE + | EXT_CALL zend_error, r0 + | add r4, 16 + |.endif } - |.if X64 - | mov CARG1, E_NOTICE - | LOAD_ADDR CARG2, "Trying to get property of non-object" - | EXT_CALL zend_error, r0 - |.else - | sub r4, 8 - | push "Trying to get property of non-object" - | push E_NOTICE - | EXT_CALL zend_error, r0 - | add r4, 16 - |.endif | SET_Z_TYPE_INFO FP + opline->result.var, IS_NULL | jmp >9 } @@ -7081,7 +7094,11 @@ static int zend_jit_fetch_obj_r(dasm_State **Dst, zend_op *opline, zend_op_array | push r0 |.endif | SAVE_VALID_OPLINE opline - | EXT_CALL zend_jit_fetch_obj_r_dynamic, r0 + if (opline->opcode == ZEND_FETCH_OBJ_R) { + | EXT_CALL zend_jit_fetch_obj_r_dynamic, r0 + } else if (opline->opcode == ZEND_FETCH_OBJ_IS) { + | EXT_CALL zend_jit_fetch_obj_is_dynamic, r0 + } | jmp >9 } From 52f14f544ff244c01d30c2d496db4749e3a286a7 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 10 Nov 2016 17:26:26 +0300 Subject: [PATCH 344/569] Tuned JIT trigger based on software counters --- ext/opcache/jit/zend_jit.h | 2 +- ext/opcache/jit/zend_jit_internal.h | 2 +- ext/opcache/jit/zend_jit_vm_helpers.c | 25 +++++++++++++++++++++++-- 3 files changed, 25 insertions(+), 4 deletions(-) diff --git a/ext/opcache/jit/zend_jit.h b/ext/opcache/jit/zend_jit.h index 88b310cd78cc3..f7bfa7c2ad536 100644 --- a/ext/opcache/jit/zend_jit.h +++ b/ext/opcache/jit/zend_jit.h @@ -50,7 +50,7 @@ */ #define ZEND_JIT_HOT_FUNC_COST 1 #define ZEND_JIT_HOT_LOOP_COST 2 -#define ZEND_JIT_HOT_COUNTER_INIT 1000 +#define ZEND_JIT_HOT_COUNTER_INIT 127 #define ZEND_JIT_DEBUG_ASM (1<<0) #define ZEND_JIT_DEBUG_SSA (1<<1) diff --git a/ext/opcache/jit/zend_jit_internal.h b/ext/opcache/jit/zend_jit_internal.h index 54ac505942994..282c045439de9 100644 --- a/ext/opcache/jit/zend_jit_internal.h +++ b/ext/opcache/jit/zend_jit_internal.h @@ -27,7 +27,7 @@ extern int zend_jit_profile_counter_rid; #define ZEND_COUNTER_INFO(op_array) ((op_array)->reserved[zend_jit_profile_counter_rid]) /* Hot Counters */ -#define ZEND_HOT_COUNTERS_COUNT 64 +#define ZEND_HOT_COUNTERS_COUNT 128 extern int16_t zend_jit_hot_counters[ZEND_HOT_COUNTERS_COUNT]; diff --git a/ext/opcache/jit/zend_jit_vm_helpers.c b/ext/opcache/jit/zend_jit_vm_helpers.c index 948c19875f15c..6a730c1253244 100644 --- a/ext/opcache/jit/zend_jit_vm_helpers.c +++ b/ext/opcache/jit/zend_jit_vm_helpers.c @@ -125,9 +125,30 @@ void ZEND_FASTCALL zend_jit_profile_helper(void) return ((zend_vm_opcode_handler_t)handler)(); } +static zend_always_inline zend_long _op_array_hash(const zend_op_array *op_array) +{ + uintptr_t x; + + if (op_array->function_name) { + x = (uintptr_t)op_array >> 3; + } else { + x = (uintptr_t)op_array->filename >> 3; + } +#if SIZEOF_SIZE_T == 4 + x = ((x >> 16) ^ x) * 0x45d9f3b; + x = ((x >> 16) ^ x) * 0x45d9f3b; + x = (x >> 16) ^ x; +#elif SIZEOF_SIZE_T == 8 + x = (x ^ (x >> 30)) * 0xbf58476d1ce4e5b9; + x = (x ^ (x >> 27)) * 0x94d049bb133111eb; + x = x ^ (x >> 31); +#endif + return x; +} + void ZEND_FASTCALL zend_jit_func_counter_helper(void) { - unsigned int n = ((uintptr_t)EX(func) >> 3) % + unsigned int n = _op_array_hash(&EX(func)->op_array) % (sizeof(zend_jit_hot_counters) / sizeof(zend_jit_hot_counters[0])); zend_jit_hot_counters[n] -= ZEND_JIT_HOT_FUNC_COST; @@ -144,7 +165,7 @@ void ZEND_FASTCALL zend_jit_func_counter_helper(void) void ZEND_FASTCALL zend_jit_loop_counter_helper(void) { - unsigned int n = ((uintptr_t)EX(func) >> 3) % + unsigned int n = _op_array_hash(&EX(func)->op_array) % (sizeof(zend_jit_hot_counters) / sizeof(zend_jit_hot_counters[0])); zend_jit_hot_counters[n] -= ZEND_JIT_HOT_LOOP_COST; From df5d423e0fb943c83a3971b999d729d60428a6d4 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 11 Nov 2016 01:19:40 +0300 Subject: [PATCH 345/569] JIT for DO_FCALL_BY_NAME (incomplete). --- ext/opcache/jit/zend_jit.c | 1 + ext/opcache/jit/zend_jit_disasm_x86.c | 2 + ext/opcache/jit/zend_jit_helpers.c | 5 + ext/opcache/jit/zend_jit_internal.h | 1 + ext/opcache/jit/zend_jit_vm_helpers.c | 37 ++++++ ext/opcache/jit/zend_jit_x86.dasc | 161 ++++++++++++++++++++------ 6 files changed, 170 insertions(+), 37 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 685ad2cf40d0e..b1d302637587d 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -1168,6 +1168,7 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa, const zend_op *rt_op goto done; case ZEND_DO_UCALL: case ZEND_DO_ICALL: +// case ZEND_DO_FCALL_BY_NAME: if (!zend_jit_do_fcall(&dasm_state, opline, op_array, ssa, call_level)) { goto jit_failure; } diff --git a/ext/opcache/jit/zend_jit_disasm_x86.c b/ext/opcache/jit/zend_jit_disasm_x86.c index b14c81d634cee..05eabba05b1ec 100644 --- a/ext/opcache/jit/zend_jit_disasm_x86.c +++ b/ext/opcache/jit/zend_jit_disasm_x86.c @@ -427,6 +427,8 @@ static int zend_jit_disasm_init(void) REGISTER_HELPER(zend_jit_fetch_obj_r_dynamic); REGISTER_HELPER(zend_jit_fetch_obj_is_slow); REGISTER_HELPER(zend_jit_fetch_obj_is_dynamic); + REGISTER_HELPER(zend_jit_vm_stack_free_args_helper); + REGISTER_HELPER(zend_jit_copy_extra_args_helper); #undef REGISTER_HELPER zend_elf_load_symbols(); diff --git a/ext/opcache/jit/zend_jit_helpers.c b/ext/opcache/jit/zend_jit_helpers.c index 9d9270d9d19a9..256849df8fe5b 100644 --- a/ext/opcache/jit/zend_jit_helpers.c +++ b/ext/opcache/jit/zend_jit_helpers.c @@ -1682,6 +1682,11 @@ static void ZEND_FASTCALL zend_jit_fetch_obj_is_dynamic(zend_object *zobj, zval zend_jit_fetch_obj_is_slow(zobj, offset, result); } +static void ZEND_FASTCALL zend_jit_vm_stack_free_args_helper(zend_execute_data *call) +{ + zend_vm_stack_free_args(call); +} + /* * Local variables: * tab-width: 4 diff --git a/ext/opcache/jit/zend_jit_internal.h b/ext/opcache/jit/zend_jit_internal.h index 282c045439de9..bb315301fb182 100644 --- a/ext/opcache/jit/zend_jit_internal.h +++ b/ext/opcache/jit/zend_jit_internal.h @@ -39,6 +39,7 @@ typedef void (ZEND_FASTCALL *zend_vm_opcode_handler_t)(void); /* VM helpers */ void ZEND_FASTCALL zend_jit_leave_nested_func_helper(uint32_t call_info); void ZEND_FASTCALL zend_jit_leave_top_func_helper(uint32_t call_info); +void ZEND_FASTCALL zend_jit_copy_extra_args_helper(void); void ZEND_FASTCALL zend_jit_func_header_helper(void); void ZEND_FASTCALL zend_jit_loop_header_helper(void); void ZEND_FASTCALL zend_jit_profile_helper(void); diff --git a/ext/opcache/jit/zend_jit_vm_helpers.c b/ext/opcache/jit/zend_jit_vm_helpers.c index 6a730c1253244..2e30f9efdee6c 100644 --- a/ext/opcache/jit/zend_jit_vm_helpers.c +++ b/ext/opcache/jit/zend_jit_vm_helpers.c @@ -87,6 +87,43 @@ void ZEND_FASTCALL zend_jit_leave_top_func_helper(uint32_t call_info) opline = NULL; } +void ZEND_FASTCALL zend_jit_copy_extra_args_helper(void) +{ + zend_op_array *op_array = &EX(func)->op_array; + + if (EXPECTED(!(op_array->fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE))) { + uint32_t first_extra_arg = op_array->num_args; + uint32_t num_args = EX_NUM_ARGS(); + zval *end, *src, *dst; + uint32_t type_flags = 0; + + if (EXPECTED((op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0)) { + /* Skip useless ZEND_RECV and ZEND_RECV_INIT opcodes */ + opline += first_extra_arg; + } + + /* move extra args into separate array after all CV and TMP vars */ + end = EX_VAR_NUM(first_extra_arg - 1); + src = end + (num_args - first_extra_arg); + dst = src + (op_array->last_var + op_array->T - first_extra_arg); + if (EXPECTED(src != dst)) { + do { + type_flags |= Z_TYPE_INFO_P(src); + ZVAL_COPY_VALUE(dst, src); + ZVAL_UNDEF(src); + src--; + dst--; + } while (src != end); + } else { + do { + type_flags |= Z_TYPE_INFO_P(src); + src--; + } while (src != end); + } + ZEND_ADD_CALL_FLAG(execute_data, ((type_flags >> Z_TYPE_FLAGS_SHIFT) & IS_TYPE_REFCOUNTED)); + } +} + /* The recorded log may be postprocessed to identify the hot functions and * loops. * diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 013ef43e1843d..b3e5d86a1725e 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -5241,8 +5241,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar } } if (!func) { - // TODO: add support for unknown functions ??? - goto fallback; + /* resolve function ar run time */ } else if (func->type == ZEND_USER_FUNCTION) { if (call_info->num_args > func->op_array.num_args || (opline-1)->opcode == ZEND_SEND_UNPACK || @@ -5287,29 +5286,33 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar | //call->prev_execute_data = execute_data; | mov EX:RX->prev_execute_data, EX - | - if (func && func->type == ZEND_USER_FUNCTION) { - | // EX(call) = NULL; - | mov aword EX:RX->call, 0 + if (!func) { + | mov r0, EX:RX->func + | cmp byte [r0 + offsetof(zend_function, type)], ZEND_USER_FUNCTION + | jne >8 } - if (func && func->type == ZEND_USER_FUNCTION) { + if (!func || func->type == ZEND_USER_FUNCTION) { + | // EX(call) = NULL; + | mov aword EX:RX->call, 0 if (RETURN_VALUE_USED(opline)) { | // ZVAL_NULL(EX_VAR(opline->result.var)); | SET_Z_TYPE_INFO FP + opline->result.var, IS_NULL | // EX(return_value) = EX_VAR(opline->result.var); - | lea r0, aword [FP + opline->result.var] - | mov aword EX:RX->return_value, r0 + | lea r2, aword [FP + opline->result.var] + | mov aword EX:RX->return_value, r2 } else { | // EX(return_value) = 0; | mov aword EX:RX->return_value, 0 } - for (i = call_info->num_args; i < func->op_array.last_var; i++) { - uint32_t n = (uint32_t)(uintptr_t)ZEND_CALL_VAR_NUM(NULL, i); - | SET_Z_TYPE_INFO RX + n, IS_UNDEF + if (func) { + for (i = call_info->num_args; i < func->op_array.last_var; i++) { + uint32_t n = (uint32_t)(uintptr_t)ZEND_CALL_VAR_NUM(NULL, i); + | SET_Z_TYPE_INFO RX + n, IS_UNDEF + } } //EX_LOAD_RUN_TIME_CACHE(op_array); @@ -5317,25 +5320,29 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar if (func && op_array == &func->op_array) { /* recursive call */ if (func->op_array.cache_size > sizeof(void*)) { - | mov r0, EX->run_time_cache - | mov EX:RX->run_time_cache, r0 + | mov r2, EX->run_time_cache + | mov EX:RX->run_time_cache, r2 } } else { - | mov r2, EX:RX->func - | mov r0, aword [r2 + offsetof(zend_op_array, run_time_cache)] - | mov EX:RX->run_time_cache, r0 + if (func) { + | mov r0, EX:RX->func + } + | mov r2, aword [r0 + offsetof(zend_op_array, run_time_cache)] + | mov EX:RX->run_time_cache, r2 } } | //EX_LOAD_LITERALS(op_array); |.if X64 - || if (zend_accel_in_shm(func->op_array.literals)) { - | LOAD_ADDR r0, func->op_array.literals - | mov EX:RX->literals, r0 + || if (func && zend_accel_in_shm(func->op_array.literals)) { + | LOAD_ADDR r2, func->op_array.literals + | mov EX:RX->literals, r2 || } else { - | mov r0, EX:RX->func - | mov r0, aword [r0 + offsetof(zend_op_array, literals)] - | mov EX:RX->literals, r0 + || if (func) { + | mov r0, EX:RX->func + || } + | mov r2, aword [r0 + offsetof(zend_op_array, literals)] + | mov EX:RX->literals, r2 || } |.endif @@ -5344,14 +5351,63 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar | mov FP, RX | // opline = op_array->opcodes; - if (zend_accel_in_shm(func->op_array.opcodes)) { - | LOAD_ADDR IP, (func->op_array.opcodes + call_info->num_args) + if (func) { + if (func && zend_accel_in_shm(func->op_array.opcodes)) { + | LOAD_ADDR IP, (func->op_array.opcodes + call_info->num_args) + } else { + if (func) { + | mov r0, EX->func + } + | mov IP, aword [r0 + offsetof(zend_op_array, opcodes)] + if (call_info->num_args) { + | add IP, (call_info->num_args * sizeof(zend_op)) + } + } } else { - | mov r0, EX->func + | // opline = op_array->opcodes | mov IP, aword [r0 + offsetof(zend_op_array, opcodes)] - if (call_info->num_args) { - | add IP, (call_info->num_args * sizeof(zend_op)) - } + | // first_extra_arg = op_array->num_args; + | mov edx, dword [r0 + offsetof(zend_op_array, num_args)] + | // num_args = EX_NUM_ARGS(); + | mov ecx, dword [FP + offsetof(zend_execute_data, This.u2.num_args)] + | // if (UNEXPECTED(num_args > first_extra_arg)) + | cmp edx, ecx + | jl >1 + |.cold_code + |1: + | EXT_CALL zend_jit_copy_extra_args_helper, r0 + | mov r0, EX->func // reload + | mov ecx, dword [FP + offsetof(zend_execute_data, This.u2.num_args)] // reload + | jmp >1 + |.code + | // if (EXPECTED((op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0)) + | test dword [r0 + offsetof(zend_op_array, fn_flags)], ZEND_ACC_HAS_TYPE_HINTS + | jnz >1 + | // opline += num_args; + |.if X64 + | movsd r2, ecx + | imul r2, r2, sizeof(zend_op) + |.else + | imul r2, ecx, sizeof(zend_op) + |.endif + | add IP, r2 + |1: + | // if (EXPECTED((int)num_args < op_array->last_var)) { + | mov eax, dword [r0 + offsetof(zend_op_array, last_var)] + | sub eax, ecx + | jle >3 //??? + | // zval *var = EX_VAR_NUM(num_args); + |.if X64 + | movsd r1, ecx + |.endif + | shl r1, 4 + | lea r1, [FP + r1 + (ZEND_CALL_FRAME_SLOT * sizeof(zval))] + |2: + | SET_Z_TYPE_INFO r1, IS_UNDEF + | sub eax, 1 + | lea r1, [r1 + 16] + | jne <2 + |3: } if (func && op_array == &func->op_array) { @@ -5369,7 +5425,12 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar | jmp aword [IP] #endif } - } else if (func && func->type == ZEND_INTERNAL_FUNCTION) { + } + + if (!func || func->type == ZEND_INTERNAL_FUNCTION) { + if (!func) { + |8: + } if (RETURN_VALUE_USED(opline)) { | // ZVAL_NULL(EX_VAR(opline->result.var)); | lea FCARG2a, aword [FP + opline->result.var] @@ -5388,12 +5449,20 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar | // fbc->internal_function.handler(call, ret); |.if X64 | mov FCARG1a, RX - | EXT_CALL func->internal_function.handler, r0 + if (func) { + | EXT_CALL func->internal_function.handler, r0 + } else { + | call aword [r0 + offsetof(zend_internal_function, handler)] + } |.else | sub r4, 8 | push FCARG2a | push RX - | EXT_CALL func->internal_function.handler, r0 + if (func) { + | EXT_CALL func->internal_function.handler, r0 + } else { + | call aword [r0 + offsetof(zend_internal_function, handler)] + } | add r4, 16 |.endif @@ -5401,9 +5470,14 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar | mov aword [&EG(current_execute_data)], FP | // zend_vm_stack_free_args(call); - for (i = 0; i < call_info->num_args; i++ ) { - uint32_t offset = (uint32_t)(uintptr_t)ZEND_CALL_VAR_NUM(NULL, i); - | ZVAL_PTR_DTOR RX + offset, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN, 0, 1, 1, op_array->filename, opline->lineno + if (func) { + for (i = 0; i < call_info->num_args; i++ ) { + uint32_t offset = (uint32_t)(uintptr_t)ZEND_CALL_VAR_NUM(NULL, i); + | ZVAL_PTR_DTOR RX + offset, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN, 0, 1, 1, op_array->filename, opline->lineno + } + } else { + | mov FCARG1a, RX + | EXT_CALL zend_jit_vm_stack_free_args_helper, r0 } | // zend_vm_stack_free_call_frame(call); @@ -5419,7 +5493,9 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar |1: if (opline->result_type == IS_UNUSED) { - uint32_t func_info = zend_get_func_info(call_info, ssa); + uint32_t func_info = call_info ? + zend_get_func_info(call_info, ssa) : + (MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN); if (func_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { | ZVAL_PTR_DTOR r4 + 8, func_info, 1, 1, 0, op_array->filename, opline->lineno @@ -5432,7 +5508,18 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar | jne ->icall_throw_handler // TODO: Can we avoid checking for interrupts after each call ??? - return zend_jit_check_timeout(Dst, opline + 1); + if (!zend_jit_check_timeout(Dst, opline + 1)) { + return 0; + } + if (opline->opcode != ZEND_DO_ICALL) { + uint32_t target_label = ssa->cfg.map[opline - op_array->opcodes] + 1; + + | jmp =>target_label + } + } + + if (!func) { + |9: } return 1; From c9281e8e6cd87940353067adc22c27cc8adc518c Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 11 Nov 2016 11:10:59 +0300 Subject: [PATCH 346/569] Fixed 64-bit build --- ext/opcache/jit/zend_jit_x86.dasc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index b3e5d86a1725e..183aebddd8e7a 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -5385,7 +5385,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar | jnz >1 | // opline += num_args; |.if X64 - | movsd r2, ecx + | movsxd r2, ecx | imul r2, r2, sizeof(zend_op) |.else | imul r2, ecx, sizeof(zend_op) @@ -5398,7 +5398,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar | jle >3 //??? | // zval *var = EX_VAR_NUM(num_args); |.if X64 - | movsd r1, ecx + | movsxd r1, ecx |.endif | shl r1, 4 | lea r1, [FP + r1 + (ZEND_CALL_FRAME_SLOT * sizeof(zval))] From 7edc287832319fe5c1769e8b3cf2e29a4e561c5a Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 11 Nov 2016 19:56:45 +0300 Subject: [PATCH 347/569] JIT for DO_FCALL_BY_NAME and DO_FCALL (incomplete). --- ext/opcache/jit/zend_jit.c | 1 + ext/opcache/jit/zend_jit_disasm_x86.c | 1 + ext/opcache/jit/zend_jit_internal.h | 1 + ext/opcache/jit/zend_jit_vm_helpers.c | 14 ++ ext/opcache/jit/zend_jit_x86.dasc | 176 ++++++++++++++++++++++---- 5 files changed, 167 insertions(+), 26 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index b1d302637587d..bbb78059557e5 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -1169,6 +1169,7 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa, const zend_op *rt_op case ZEND_DO_UCALL: case ZEND_DO_ICALL: // case ZEND_DO_FCALL_BY_NAME: +// case ZEND_DO_FCALL: if (!zend_jit_do_fcall(&dasm_state, opline, op_array, ssa, call_level)) { goto jit_failure; } diff --git a/ext/opcache/jit/zend_jit_disasm_x86.c b/ext/opcache/jit/zend_jit_disasm_x86.c index 05eabba05b1ec..3374ab192efff 100644 --- a/ext/opcache/jit/zend_jit_disasm_x86.c +++ b/ext/opcache/jit/zend_jit_disasm_x86.c @@ -429,6 +429,7 @@ static int zend_jit_disasm_init(void) REGISTER_HELPER(zend_jit_fetch_obj_is_dynamic); REGISTER_HELPER(zend_jit_vm_stack_free_args_helper); REGISTER_HELPER(zend_jit_copy_extra_args_helper); + REGISTER_HELPER(zend_jit_deprecated_or_abstract_helper); #undef REGISTER_HELPER zend_elf_load_symbols(); diff --git a/ext/opcache/jit/zend_jit_internal.h b/ext/opcache/jit/zend_jit_internal.h index bb315301fb182..f8776b5edb09b 100644 --- a/ext/opcache/jit/zend_jit_internal.h +++ b/ext/opcache/jit/zend_jit_internal.h @@ -40,6 +40,7 @@ typedef void (ZEND_FASTCALL *zend_vm_opcode_handler_t)(void); void ZEND_FASTCALL zend_jit_leave_nested_func_helper(uint32_t call_info); void ZEND_FASTCALL zend_jit_leave_top_func_helper(uint32_t call_info); void ZEND_FASTCALL zend_jit_copy_extra_args_helper(void); +void ZEND_FASTCALL zend_jit_deprecated_or_abstract_helper(void); void ZEND_FASTCALL zend_jit_func_header_helper(void); void ZEND_FASTCALL zend_jit_loop_header_helper(void); void ZEND_FASTCALL zend_jit_profile_helper(void); diff --git a/ext/opcache/jit/zend_jit_vm_helpers.c b/ext/opcache/jit/zend_jit_vm_helpers.c index 2e30f9efdee6c..649f300ca76e5 100644 --- a/ext/opcache/jit/zend_jit_vm_helpers.c +++ b/ext/opcache/jit/zend_jit_vm_helpers.c @@ -124,6 +124,20 @@ void ZEND_FASTCALL zend_jit_copy_extra_args_helper(void) } } +void ZEND_FASTCALL zend_jit_deprecated_or_abstract_helper(void) +{ + zend_function *fbc = ((zend_execute_data*)(opline))->func; + + if (UNEXPECTED((fbc->common.fn_flags & ZEND_ACC_ABSTRACT) != 0)) { + zend_throw_error(NULL, "Cannot call abstract method %s::%s()", ZSTR_VAL(fbc->common.scope->name), ZSTR_VAL(fbc->common.function_name)); + } else if (UNEXPECTED((fbc->common.fn_flags & ZEND_ACC_DEPRECATED) != 0)) { + zend_error(E_DEPRECATED, "Function %s%s%s() is deprecated", + fbc->common.scope ? ZSTR_VAL(fbc->common.scope->name) : "", + fbc->common.scope ? "::" : "", + ZSTR_VAL(fbc->common.function_name)); + } +} + /* The recorded log may be postprocessed to identify the hot functions and * loops. * diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 183aebddd8e7a..664f805e73b5a 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -5100,27 +5100,21 @@ static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, zen return 1; } -static int zend_jit_needs_call_chain(zend_call_info *call_info, uint32_t b, zend_op_array *op_array, zend_ssa *ssa) +static int zend_jit_needs_call_chain(zend_call_info *call_info, uint32_t b, zend_op_array *op_array, zend_ssa *ssa, const zend_op *opline) { - const zend_op *opline, *end; int skip; if (!call_info) { - return 1; - } - - opline = call_info->caller_init_opline; - end = call_info->caller_call_opline; - - if (end - op_array->opcodes >= ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len) { - /* INIT_FCALL and DO_FCALL in different BasicBlocks */ - return 1; - } + const zend_op *end = op_array->opcodes + op_array->last; - opline++; - skip = 1; - while (opline != end) { - if (skip) { + opline++; + skip = 1; + while (opline != end) { + if (!skip) { + if (zend_may_throw(opline, op_array, ssa)) { + return 1; + } + } switch (opline->opcode) { case ZEND_SEND_VAL: case ZEND_SEND_VAR: @@ -5134,17 +5128,84 @@ static int zend_jit_needs_call_chain(zend_call_info *call_info, uint32_t b, zend case ZEND_SEND_ARRAY: case ZEND_SEND_USER: case ZEND_SEND_UNPACK: + case ZEND_INIT_FCALL: + case ZEND_INIT_METHOD_CALL: + case ZEND_INIT_STATIC_METHOD_CALL: + case ZEND_INIT_FCALL_BY_NAME: + case ZEND_INIT_NS_FCALL_BY_NAME: + case ZEND_INIT_DYNAMIC_CALL: + case ZEND_NEW: + case ZEND_INIT_USER_CALL: + case ZEND_FAST_CALL: + case ZEND_JMP: + case ZEND_JMPZNZ: + case ZEND_JMPZ: + case ZEND_JMPNZ: + case ZEND_JMPZ_EX: + case ZEND_JMPNZ_EX: + case ZEND_FE_RESET_R: + case ZEND_FE_RESET_RW: + case ZEND_JMP_SET: + case ZEND_COALESCE: + case ZEND_ASSERT_CHECK: + case ZEND_CATCH: + case ZEND_DECLARE_ANON_CLASS: + case ZEND_DECLARE_ANON_INHERITED_CLASS: + case ZEND_FE_FETCH_R: + case ZEND_FE_FETCH_RW: return 1; + case ZEND_DO_ICALL: + case ZEND_DO_UCALL: + case ZEND_DO_FCALL_BY_NAME: + case ZEND_DO_FCALL: + end = opline; + if (end - op_array->opcodes >= ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len) { + /* INIT_FCALL and DO_FCALL in different BasicBlocks */ + return 1; + } + return 0; } - } else { - if (zend_may_throw(opline, op_array, ssa)) { - return 1; - } + opline++; } + + return 1; + } else { + const zend_op *end = call_info->caller_call_opline; + + if (end - op_array->opcodes >= ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len) { + /* INIT_FCALL and DO_FCALL in different BasicBlocks */ + return 1; + } + opline++; - } + skip = 1; + while (opline != end) { + if (skip) { + switch (opline->opcode) { + case ZEND_SEND_VAL: + case ZEND_SEND_VAR: + case ZEND_SEND_VAL_EX: + case ZEND_SEND_VAR_EX: + case ZEND_SEND_REF: + case ZEND_SEND_VAR_NO_REF: + case ZEND_SEND_VAR_NO_REF_EX: + skip = 0; + break; + case ZEND_SEND_ARRAY: + case ZEND_SEND_USER: + case ZEND_SEND_UNPACK: + return 1; + } + } else { + if (zend_may_throw(opline, op_array, ssa)) { + return 1; + } + } + opline++; + } - return 0; + return 0; + } } static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, uint32_t b, zend_op_array *op_array, zend_ssa *ssa, int call_level) @@ -5197,6 +5258,7 @@ static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, uint32_t ZEND_ASSERT(0); } if (opline->opcode != ZEND_INIT_FCALL) { + // TODO: INIT_FCALL may fail in some cases ??? | test r0, r0 | jz ->exception_handler } @@ -5212,7 +5274,7 @@ static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, uint32_t return 0; } - if (zend_jit_needs_call_chain(call_info, b, op_array, ssa)) { + if (zend_jit_needs_call_chain(call_info, b, op_array, ssa, opline)) { if (!zend_jit_save_call_chain(Dst, call_level)) { return 0; } @@ -5293,6 +5355,30 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar | jne >8 } + if (opline->opcode == ZEND_DO_FCALL) { + if (!func) { + | test dword [r0 + offsetof(zend_op_array, fn_flags)], (ZEND_ACC_DEPRECATED|ZEND_ACC_ABSTRACT) + | jnz >1 + |.cold_code + |1: + | EXT_CALL zend_jit_deprecated_or_abstract_helper, r0 + | cmp aword [&EG(exception)], 0 + | jne ->exception_handler + | mov r0, EX->func // reload + | jmp >1 + |.code + |1: + } else if (func->common.fn_flags & ZEND_ACC_ABSTRACT) { + | EXT_CALL zend_jit_deprecated_or_abstract_helper, r0 + | jmp ->exception_handler + } else if (func->common.fn_flags & ZEND_ACC_DEPRECATED) { + | EXT_CALL zend_jit_deprecated_or_abstract_helper, r0 + | cmp aword [&EG(exception)], 0 + | jne ->exception_handler + | mov r0, EX->func // reload + } + } + if (!func || func->type == ZEND_USER_FUNCTION) { | // EX(call) = NULL; | mov aword EX:RX->call, 0 @@ -5393,8 +5479,8 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar | add IP, r2 |1: | // if (EXPECTED((int)num_args < op_array->last_var)) { - | mov eax, dword [r0 + offsetof(zend_op_array, last_var)] - | sub eax, ecx + | mov edx, dword [r0 + offsetof(zend_op_array, last_var)] + | sub edx, ecx | jle >3 //??? | // zval *var = EX_VAR_NUM(num_args); |.if X64 @@ -5404,7 +5490,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar | lea r1, [FP + r1 + (ZEND_CALL_FRAME_SLOT * sizeof(zval))] |2: | SET_Z_TYPE_INFO r1, IS_UNDEF - | sub eax, 1 + | sub edx, 1 | lea r1, [r1 + 16] | jne <2 |3: @@ -5431,6 +5517,29 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar if (!func) { |8: } + if (opline->opcode == ZEND_DO_FCALL_BY_NAME) { + if (!func) { + | test dword [r0 + offsetof(zend_op_array, fn_flags)], (ZEND_ACC_DEPRECATED|ZEND_ACC_ABSTRACT) + | jnz >1 + |.cold_code + |1: + | EXT_CALL zend_jit_deprecated_or_abstract_helper, r0 + | cmp aword [&EG(exception)], 0 + | jne ->exception_handler + | mov r0, EX->func // reload + | jmp >1 + |.code + |1: + } else if (func->common.fn_flags & ZEND_ACC_ABSTRACT) { + | EXT_CALL zend_jit_deprecated_or_abstract_helper, r0 + | jmp ->exception_handler + } else if (func->common.fn_flags & ZEND_ACC_DEPRECATED) { + | EXT_CALL zend_jit_deprecated_or_abstract_helper, r0 + | cmp aword [&EG(exception)], 0 + | jne ->exception_handler + | mov r0, EX->func // reload + } + } if (RETURN_VALUE_USED(opline)) { | // ZVAL_NULL(EX_VAR(opline->result.var)); | lea FCARG2a, aword [FP + opline->result.var] @@ -5480,6 +5589,21 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar | EXT_CALL zend_jit_vm_stack_free_args_helper, r0 } + if (opline->opcode == ZEND_DO_FCALL) { + // TODO: optimize ??? + | // if (UNEXPECTED(ZEND_CALL_INFO(call) & ZEND_CALL_RELEASE_THIS)) + | test byte [RX + offsetof(zend_execute_data, This.u1.type_info) + 2], ZEND_CALL_RELEASE_THIS + | jnz >1 + |.cold_code + |1: + | // OBJ_RELEASE(object); + | GET_Z_PTR r0, RX + offsetof(zend_execute_data, This) + | OBJ_RELEASE r0, ecx, >2 + | jmp >2 + |.code + |2: + } + | // zend_vm_stack_free_call_frame(call); | test byte [RX + offsetof(zend_execute_data, This.u1.type_info) + 2], ZEND_CALL_ALLOCATED | jnz >1 From c5d4b958b0ad24d8cb28c67bea3e078ebd921f6e Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 21 Nov 2016 13:51:11 +0300 Subject: [PATCH 348/569] JIT for DO_FCALL and DO_FCALL_BY_NAME (incomplete). --- ext/opcache/jit/zend_jit_x86.dasc | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 664f805e73b5a..0eaaa75cd726d 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -5596,8 +5596,19 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar | jnz >1 |.cold_code |1: - | // OBJ_RELEASE(object); | GET_Z_PTR r0, RX + offsetof(zend_execute_data, This) + if (opline->op1.num & ZEND_CALL_CTOR) { + | // if (UNEXPECTED(EG(exception) + | cmp aword [&EG(exception)], 0 + | je >1 + | // GC_REFCOUNT(object)--; + | GC_DELREF r0 + | // zend_object_store_ctor_failed(object); + | // GC_FLAGS(obj) |= IS_OBJ_DESTRUCTOR_CALLED; + | or byte [r0 + offsetof(zend_object, gc.u.v.flags)], IS_OBJ_DESTRUCTOR_CALLED + |1: + } + | // OBJ_RELEASE(object); | OBJ_RELEASE r0, ecx, >2 | jmp >2 |.code @@ -6303,17 +6314,8 @@ static int zend_jit_leave_func(dasm_State **Dst, const zend_op *opline, zend_op_ | // GC_REFCOUNT(object)--; | GC_DELREF r0 | // zend_object_store_ctor_failed(object); - |.if X64 - | mov CARG1, r0 - | EXT_CALL zend_object_store_ctor_failed, r0 - |.else - | sub r4, 12 - | push r0 - | EXT_CALL zend_object_store_ctor_failed, r0 - | add r4, 16 - |.endif - | // reload registers - | mov r0, EX->This.value.obj + | // GC_FLAGS(obj) |= IS_OBJ_DESTRUCTOR_CALLED; + | or byte [r0 + offsetof(zend_object, gc.u.v.flags)], IS_OBJ_DESTRUCTOR_CALLED | jmp >5 |.code |5: From 0f4dae6baf35662adb5ba3e1cd758d2050cf207f Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 21 Nov 2016 18:17:32 +0300 Subject: [PATCH 349/569] JIT for DO_FCALL and DO_FCALL_BY_NAME (incomplete). --- ext/opcache/jit/zend_jit_disasm_x86.c | 1 + ext/opcache/jit/zend_jit_helpers.c | 19 +++++++++++++++ ext/opcache/jit/zend_jit_x86.dasc | 35 ++++++++++++++++++++++++--- 3 files changed, 51 insertions(+), 4 deletions(-) diff --git a/ext/opcache/jit/zend_jit_disasm_x86.c b/ext/opcache/jit/zend_jit_disasm_x86.c index 3374ab192efff..f724232420f28 100644 --- a/ext/opcache/jit/zend_jit_disasm_x86.c +++ b/ext/opcache/jit/zend_jit_disasm_x86.c @@ -430,6 +430,7 @@ static int zend_jit_disasm_init(void) REGISTER_HELPER(zend_jit_vm_stack_free_args_helper); REGISTER_HELPER(zend_jit_copy_extra_args_helper); REGISTER_HELPER(zend_jit_deprecated_or_abstract_helper); + REGISTER_HELPER(zend_jit_verify_internal_arg_types_helper); #undef REGISTER_HELPER zend_elf_load_symbols(); diff --git a/ext/opcache/jit/zend_jit_helpers.c b/ext/opcache/jit/zend_jit_helpers.c index 256849df8fe5b..445147089f0d7 100644 --- a/ext/opcache/jit/zend_jit_helpers.c +++ b/ext/opcache/jit/zend_jit_helpers.c @@ -1687,6 +1687,25 @@ static void ZEND_FASTCALL zend_jit_vm_stack_free_args_helper(zend_execute_data * zend_vm_stack_free_args(call); } +static int ZEND_FASTCALL zend_jit_verify_internal_arg_types_helper(zend_execute_data *call) +{ + zend_function *fbc = call->func; + uint32_t i; + uint32_t num_args = ZEND_CALL_NUM_ARGS(call); + zval *p = ZEND_CALL_ARG(call, 1); + + for (i = 0; i < num_args; ++i) { + zend_check_internal_arg_type(fbc, i + 1, p); + if (UNEXPECTED(EG(exception))) { + EG(current_execute_data) = call->prev_execute_data; + zend_vm_stack_free_args(call); + return 0; + } + p++; + } + return 1; +} + /* * Local variables: * tab-width: 4 diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 0eaaa75cd726d..267a792eb9d7e 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -5364,7 +5364,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar | EXT_CALL zend_jit_deprecated_or_abstract_helper, r0 | cmp aword [&EG(exception)], 0 | jne ->exception_handler - | mov r0, EX->func // reload + | mov r0, EX:RX->func // reload | jmp >1 |.code |1: @@ -5375,7 +5375,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar | EXT_CALL zend_jit_deprecated_or_abstract_helper, r0 | cmp aword [&EG(exception)], 0 | jne ->exception_handler - | mov r0, EX->func // reload + | mov r0, EX:RX->func // reload } } @@ -5526,7 +5526,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar | EXT_CALL zend_jit_deprecated_or_abstract_helper, r0 | cmp aword [&EG(exception)], 0 | jne ->exception_handler - | mov r0, EX->func // reload + | mov r0, EX:RX->func // reload | jmp >1 |.code |1: @@ -5537,9 +5537,10 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar | EXT_CALL zend_jit_deprecated_or_abstract_helper, r0 | cmp aword [&EG(exception)], 0 | jne ->exception_handler - | mov r0, EX->func // reload + | mov r0, EX:RX->func // reload } } + if (RETURN_VALUE_USED(opline)) { | // ZVAL_NULL(EX_VAR(opline->result.var)); | lea FCARG2a, aword [FP + opline->result.var] @@ -5553,6 +5554,31 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar | // EG(current_execute_data) = execute_data; | mov aword [&EG(current_execute_data)], RX + if (!func || (func->common.fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) { + if (!func) { + | test dword [r0 + offsetof(zend_op_array, fn_flags)], ZEND_ACC_HAS_TYPE_HINTS + | jnz >1 + |.cold_code + } + |1: + | mov FCARG1a, RX + | EXT_CALL zend_jit_verify_internal_arg_types_helper, r0 + | test r0, r0 + | je >8 + | // reload + if (RETURN_VALUE_USED(opline)) { + | lea FCARG2a, aword [FP + opline->result.var] + } else { + | lea FCARG2a, aword [r4 + 8] + } + if (!func) { + | mov r0, EX:RX->func // reload + | jmp >1 + |.code + } + |1: + } + zend_jit_reset_opline(Dst, NULL); | // fbc->internal_function.handler(call, ret); @@ -5589,6 +5615,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar | EXT_CALL zend_jit_vm_stack_free_args_helper, r0 } + |8: if (opline->opcode == ZEND_DO_FCALL) { // TODO: optimize ??? | // if (UNEXPECTED(ZEND_CALL_INFO(call) & ZEND_CALL_RELEASE_THIS)) From dc18f75016d7418bf7994f0facdc8980950f5b8e Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 22 Nov 2016 18:42:12 +0300 Subject: [PATCH 350/569] JIT for DO_FCALL and DO_FCALL_BY_NAME (incomplete) --- ext/opcache/jit/zend_jit_x86.dasc | 33 ++++++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 267a792eb9d7e..4bbb225da0cab 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -5307,6 +5307,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar } else if (func->type == ZEND_USER_FUNCTION) { if (call_info->num_args > func->op_array.num_args || (opline-1)->opcode == ZEND_SEND_UNPACK || + (opline-1)->opcode == ZEND_SEND_ARRAY || (func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) != 0) { goto fallback; } @@ -5316,7 +5317,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar goto fallback; } #endif - if ((opline-1)->opcode == ZEND_SEND_UNPACK) { + if ((opline-1)->opcode == ZEND_SEND_UNPACK || (opline-1)->opcode == ZEND_SEND_ARRAY) { goto fallback; } } else { @@ -5351,8 +5352,6 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar if (!func) { | mov r0, EX:RX->func - | cmp byte [r0 + offsetof(zend_function, type)], ZEND_USER_FUNCTION - | jne >8 } if (opline->opcode == ZEND_DO_FCALL) { @@ -5379,6 +5378,11 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar } } + if (!func) { + | cmp byte [r0 + offsetof(zend_function, type)], ZEND_USER_FUNCTION + | jne >8 + } + if (!func || func->type == ZEND_USER_FUNCTION) { | // EX(call) = NULL; | mov aword EX:RX->call, 0 @@ -5516,6 +5520,29 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar if (!func || func->type == ZEND_INTERNAL_FUNCTION) { if (!func) { |8: + if (opline->opcode == ZEND_DO_FCALL) { + | jg >8 + |.cold_code + |8: + if (RETURN_VALUE_USED(opline)) { + | // ZVAL_NULL(EX_VAR(opline->result.var)); + | lea FCARG2a, aword [FP + opline->result.var] + | SET_Z_TYPE_INFO FCARG2a, IS_NULL + } else { + | sub r4, 16 /* alloca() */ + | lea FCARG2a, aword [r4 + 8] + | SET_Z_TYPE_INFO FCARG2a, IS_NULL + } + | mov FCARG1a, RX + | EXT_CALL zend_do_fcall_overloaded, r0 + | test r0, r0 + | jne >8 + if (!RETURN_VALUE_USED(opline)) { + | add r4, 16 /* revert alloca() */ + } + | jmp ->exception_handler + |.code + } } if (opline->opcode == ZEND_DO_FCALL_BY_NAME) { if (!func) { From 7d170e4eff78e575aa4d1a1cbf8976f6bfd12a33 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 23 Nov 2016 17:58:09 +0300 Subject: [PATCH 351/569] JIT for DO_FCALL and DO_FCALL_BY_NAME (done). --- ext/opcache/jit/zend_jit.c | 4 ++-- ext/opcache/jit/zend_jit_x86.dasc | 15 ++++++++++++--- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index bbb78059557e5..80573571fd983 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -1168,8 +1168,8 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa, const zend_op *rt_op goto done; case ZEND_DO_UCALL: case ZEND_DO_ICALL: -// case ZEND_DO_FCALL_BY_NAME: -// case ZEND_DO_FCALL: + case ZEND_DO_FCALL_BY_NAME: + case ZEND_DO_FCALL: if (!zend_jit_do_fcall(&dasm_state, opline, op_array, ssa, call_level)) { goto jit_failure; } diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 4bbb225da0cab..7de098d2d54e4 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -1379,7 +1379,6 @@ static int zend_jit_leave_throw_stub(dasm_State **Dst) static int zend_jit_icall_throw_stub(dasm_State **Dst) { |->icall_throw_handler: - | mov IP, EX->opline | // zend_throw_exception_internal(NULL); |.if X64 | xor CARG1, CARG1 @@ -5694,7 +5693,12 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar | // if (UNEXPECTED(EG(exception) != NULL)) { | cmp aword [&EG(exception)], 0 - | jne ->icall_throw_handler + | jne >1 + |.cold_code + |1: + | LOAD_ADDR IP, opline + | jmp ->icall_throw_handler + |.code // TODO: Can we avoid checking for interrupts after each call ??? if (!zend_jit_check_timeout(Dst, opline + 1)) { @@ -7235,7 +7239,12 @@ static int zend_jit_fetch_obj_read(dasm_State **Dst, zend_op *opline, zend_op_ar offset = zend_get_known_property_offset(ce, Z_STR_P(member), opline->op1_type == IS_UNUSED, op_array->filename); if (opline->op1_type == IS_UNUSED) { - | IF_Z_TYPE FP + offsetof(zend_execute_data, This), IS_UNDEF, ->not_obj + | IF_Z_TYPE FP + offsetof(zend_execute_data, This), IS_UNDEF, >1 + |.cold_code + |1: + | SAVE_VALID_OPLINE opline + | jmp ->not_obj + |.code | GET_Z_PTR FCARG1a, FP + offsetof(zend_execute_data, This) } else if (op1_info & MAY_BE_REF) { | lea r0, [FP + opline->op1.var] From 96392ef04ff33d5bebd536fc0f3271df3e7333cf Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 24 Nov 2016 17:46:15 +0300 Subject: [PATCH 352/569] Implemented new JIT trigger - ON_DOC_COMMENT. If "opcache.jit=45" (from 41 to 45), only functions with @jit tag in doc_comments are going to be compiled, at script loading time. --- ext/opcache/jit/zend_jit.c | 52 +++++++++++++++++++++++++++++++++++++- ext/opcache/jit/zend_jit.h | 5 ++-- 2 files changed, 54 insertions(+), 3 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 80573571fd983..debc8f7a1a0e3 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -1621,6 +1621,24 @@ static int zend_jit_setup_hot_counters(zend_op_array *op_array) return SUCCESS; } +static int zend_needs_manual_jit(const zend_op_array *op_array) +{ + if (op_array->doc_comment) { + const char *s = ZSTR_VAL(op_array->doc_comment); + const char *p = strstr(s, "@jit"); + + if (p) { + size_t l = ZSTR_LEN(op_array->doc_comment); + + if ((p == s + 3 || *(p-1) <= ' ') && + (p + 6 == s + l || *(p+4) <= ' ')) { + return 1; + } + } + } + return 0; +} + ZEND_API int zend_jit_op_array(zend_op_array *op_array, zend_script *script) { if (dasm_ptr == NULL) { @@ -1655,6 +1673,12 @@ ZEND_API int zend_jit_op_array(zend_op_array *op_array, zend_script *script) return zend_jit_setup_hot_counters(op_array); } else if (zend_jit_trigger == ZEND_JIT_ON_SCRIPT_LOAD) { return zend_real_jit_func(op_array, script, NULL); + } else if (zend_jit_trigger == ZEND_JIT_ON_DOC_COMMENT) { + if (zend_needs_manual_jit(op_array)) { + return zend_real_jit_func(op_array, script, NULL); + } else { + return SUCCESS; + } } else { ZEND_ASSERT(0); } @@ -1687,7 +1711,21 @@ ZEND_API int zend_jit_script(zend_script *script) goto jit_failure; } } - } else if (zend_jit_trigger == ZEND_JIT_ON_SCRIPT_LOAD) { + } else if (zend_jit_trigger == ZEND_JIT_ON_SCRIPT_LOAD || + zend_jit_trigger == ZEND_JIT_ON_DOC_COMMENT) { + + if (zend_jit_trigger == ZEND_JIT_ON_DOC_COMMENT) { + int do_jit = 0; + for (i = 0; i < call_graph.op_arrays_count; i++) { + if (zend_needs_manual_jit(call_graph.op_arrays[i])) { + do_jit = 1; + break; + } + } + if (!do_jit) { + goto jit_failure; + } + } for (i = 0; i < call_graph.op_arrays_count; i++) { info = ZEND_FUNC_INFO(call_graph.op_arrays[i]); if (info) { @@ -1698,6 +1736,10 @@ ZEND_API int zend_jit_script(zend_script *script) } for (i = 0; i < call_graph.op_arrays_count; i++) { + if (zend_jit_trigger == ZEND_JIT_ON_DOC_COMMENT && + !zend_needs_manual_jit(call_graph.op_arrays[i])) { + continue; + } info = ZEND_FUNC_INFO(call_graph.op_arrays[i]); if (info) { if (zend_jit_op_array_analyze2(call_graph.op_arrays[i], script, &info->ssa, &info->flags) != SUCCESS) { @@ -1708,6 +1750,10 @@ ZEND_API int zend_jit_script(zend_script *script) if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_SSA) { for (i = 0; i < call_graph.op_arrays_count; i++) { + if (zend_jit_trigger == ZEND_JIT_ON_DOC_COMMENT && + !zend_needs_manual_jit(call_graph.op_arrays[i])) { + continue; + } info = ZEND_FUNC_INFO(call_graph.op_arrays[i]); if (info) { zend_dump_op_array(call_graph.op_arrays[i], ZEND_DUMP_HIDE_UNREACHABLE|ZEND_DUMP_RC_INFERENCE|ZEND_DUMP_SSA|ZEND_DUMP_RT_CONSTANTS, "JIT", &info->ssa); @@ -1716,6 +1762,10 @@ ZEND_API int zend_jit_script(zend_script *script) } for (i = 0; i < call_graph.op_arrays_count; i++) { + if (zend_jit_trigger == ZEND_JIT_ON_DOC_COMMENT && + !zend_needs_manual_jit(call_graph.op_arrays[i])) { + continue; + } info = ZEND_FUNC_INFO(call_graph.op_arrays[i]); if (info) { if (zend_jit(call_graph.op_arrays[i], &info->ssa, NULL) != SUCCESS) { diff --git a/ext/opcache/jit/zend_jit.h b/ext/opcache/jit/zend_jit.h index f7bfa7c2ad536..5568dc880f287 100644 --- a/ext/opcache/jit/zend_jit.h +++ b/ext/opcache/jit/zend_jit.h @@ -32,8 +32,9 @@ #define ZEND_JIT_ON_SCRIPT_LOAD 0 #define ZEND_JIT_ON_FIRST_EXEC 1 -#define ZEND_JIT_ON_PROF_REQUEST 2 -#define ZEND_JIT_ON_HOT_COUNTERS 3 +#define ZEND_JIT_ON_PROF_REQUEST 2 /* compile the most frequently caled on first requrest functions */ +#define ZEND_JIT_ON_HOT_COUNTERS 3 /* compile functions after N calls or loop iterations */ +#define ZEND_JIT_ON_DOC_COMMENT 4 /* compile functions with "@jit" tag in doc-comments */ #define ZEND_JIT_TRIGGER(n) (((n) / 10) % 10) From 39a5bd92bc360eae3b417fb64b82c0582b9c740e Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 25 Nov 2016 12:30:41 +0300 Subject: [PATCH 353/569] JIT for FREE and FE_FREE. --- ext/opcache/jit/zend_jit.c | 6 ++++++ ext/opcache/jit/zend_jit_x86.dasc | 29 +++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index debc8f7a1a0e3..e69c7e1955031 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -1258,6 +1258,12 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa, const zend_op *rt_op goto jit_failure; } goto done; + case ZEND_FREE: + case ZEND_FE_FREE: + if (!zend_jit_free(&dasm_state, opline, op_array, ssa)) { + goto jit_failure; + } + goto done; default: break; } diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 7de098d2d54e4..64230347379ef 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -7395,6 +7395,35 @@ fallback: return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); } +static int zend_jit_free(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) +{ + uint32_t op1_info = OP1_INFO(); + + if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { + if (zend_may_throw(opline, op_array, ssa)) { + | SAVE_VALID_OPLINE, opline + } + if (opline->opcode == ZEND_FE_FREE && (op1_info & (MAY_BE_OBJECT|MAY_BE_REF))) { + if (op1_info & MAY_BE_ARRAY) { + | IF_Z_TYPE FP + opline->op1.var, IS_ARRAY, >7 + } + | mov FCARG1d, dword [FP + opline->op1.var + offsetof(zval, u2.fe_iter_idx)] + | cmp FCARG1d, -1 + | je >7 + | EXT_CALL zend_hash_iterator_del, r0 + |7: + } + | ZVAL_PTR_DTOR FP + opline->op1.var, op1_info, 0, 0, 0, op_array->filename, opline->lineno + if (zend_may_throw(opline, op_array, ssa)) { + if (!zend_jit_check_exception(Dst)) { + return 0; + } + } + } + + return 1; +} + #ifdef ZEND_JIT_RECORD static int zend_jit_func_header(dasm_State **Dst, zend_op_array *op_array) { From 4941c0137e554282cb21c45f97fc48073c0d25c4 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 28 Nov 2016 23:21:56 +0300 Subject: [PATCH 354/569] Removed IS_TYPE_IMMUTABLE --- ext/opcache/jit/zend_jit_x86.dasc | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 64230347379ef..5a0874054e49d 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -865,8 +865,8 @@ static void* dasm_labels[zend_lb_MAX]; |.macro ZVAL_COPY_CTOR, val_info, type_flags_reg, value_ptr_reg, filename, lineno ||if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { || if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY)) { -| IF_NOT_FLAGS type_flags_reg, IS_TYPE_REFCOUNTED + IS_TYPE_IMMUTABLE, >2 -| IF_NOT_FLAGS type_flags_reg, IS_TYPE_COPYABLE + IS_TYPE_IMMUTABLE, >1 +| IF_NOT_FLAGS type_flags_reg, IS_TYPE_REFCOUNTED + IS_TYPE_COPYABLE, >2 +| IF_NOT_FLAGS type_flags_reg, IS_TYPE_COPYABLE, >1 | GC_ADDREF value_ptr_reg | jmp >2 |1: @@ -1000,18 +1000,18 @@ static void* dasm_labels[zend_lb_MAX]; |.macro SEPARATE_ZVAL_NOREF, zv, op_info, cold, filename, lineno || if ((op_info & (MAY_BE_STRING|MAY_BE_ARRAY)) && RC_MAY_BE_N(op_info)) { || if (cold) { -| IF_Z_FLAGS zv, IS_TYPE_COPYABLE + IS_TYPE_IMMUTABLE, >1 +| IF_Z_FLAGS zv, IS_TYPE_COPYABLE, >1 |.cold_code |1: || } else { -| IF_NOT_Z_FLAGS zv, IS_TYPE_COPYABLE + IS_TYPE_IMMUTABLE, >2 +| IF_NOT_Z_FLAGS zv, IS_TYPE_COPYABLE, >2 || } | GET_Z_PTR r0, zv || if (RC_MAY_BE_1(op_info)) { | cmp dword [r0], 1 // if (GC_REFCOUNTED() > 1) | jbe >2 || } -| IF_Z_FLAGS zv, IS_TYPE_IMMUTABLE, >1 +| IF_NOT_Z_FLAGS zv, IS_TYPE_REFCOUNTED, >1 | GC_DELREF r0 |1: | lea FCARG1a, [zv] @@ -1027,18 +1027,18 @@ static void* dasm_labels[zend_lb_MAX]; |.macro SEPARATE_ZVAL_NOREF_REG, op_info, cold, filename, lineno || if ((op_info & (MAY_BE_STRING|MAY_BE_ARRAY)) && RC_MAY_BE_N(op_info)) { || if (cold) { -| IF_Z_FLAGS FCARG1a, IS_TYPE_COPYABLE + IS_TYPE_IMMUTABLE, >1 +| IF_Z_FLAGS FCARG1a, IS_TYPE_COPYABLE, >1 |.cold_code |1: || } else { -| IF_NOT_Z_FLAGS FCARG1a, IS_TYPE_COPYABLE + IS_TYPE_IMMUTABLE, >2 +| IF_NOT_Z_FLAGS FCARG1a, IS_TYPE_COPYABLE, >2 || } | GET_Z_PTR r0, FCARG1a || if (RC_MAY_BE_1(op_info)) { | cmp dword [r0], 1 // if (GC_REFCOUNTED() > 1) | jbe >2 || } -| IF_Z_FLAGS FCARG1a, IS_TYPE_IMMUTABLE, >1 +| IF_NOT_Z_FLAGS FCARG1a, IS_TYPE_REFCOUNTED, >1 | GC_DELREF r0 |1: | mov aword [r4], FCARG1a // save @@ -1065,7 +1065,7 @@ static void* dasm_labels[zend_lb_MAX]; | jbe >2 || } || } -| IF_Z_FLAGS FP + op.var, IS_TYPE_IMMUTABLE, >1 +| IF_NOT_Z_FLAGS FP + op.var, IS_TYPE_REFCOUNTED, >1 | GC_DELREF FCARG1a |1: | LOAD_ZVAL_ADDR FCARG1a, op_type, op @@ -1094,7 +1094,7 @@ static void* dasm_labels[zend_lb_MAX]; | jbe >2 || } || } -| IF_Z_FLAGS FCARG1a, IS_TYPE_IMMUTABLE, >1 +| IF_NOT_Z_FLAGS FCARG1a, IS_TYPE_REFCOUNTED, >1 | GC_DELREF r0 |1: | mov aword [r4], FCARG1a // save @@ -1977,8 +1977,8 @@ static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, zend_op_arr | //ZVAL_COPY_CTOR op1_info, ah, r2, op_array->filename, opline->lineno || if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { || if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY)) { - | IF_NOT_FLAGS ah, IS_TYPE_REFCOUNTED + IS_TYPE_IMMUTABLE, >2 - | IF_FLAGS ah, IS_TYPE_COPYABLE + IS_TYPE_IMMUTABLE, >1 + | IF_NOT_FLAGS ah, IS_TYPE_REFCOUNTED + IS_TYPE_COPYABLE, >2 + | IF_FLAGS ah, IS_TYPE_COPYABLE, >1 | GC_ADDREF r2 | jmp >2 |1: From 3d7101df02ab32215ac1a37ea6820464077c9b22 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 29 Nov 2016 11:12:04 +0300 Subject: [PATCH 355/569] JIT for defined --- ext/opcache/jit/zend_jit.c | 6 +++ ext/opcache/jit/zend_jit_x86.dasc | 75 ++++++++++++++++++++++++++++++- 2 files changed, 80 insertions(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index e69c7e1955031..12a1ef6af4555 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -21,6 +21,7 @@ #include "Zend/zend_execute.h" #include "Zend/zend_vm.h" #include "Zend/zend_exceptions.h" +#include "Zend/zend_constants.h" #include "zend_smart_str.h" #include "jit/zend_jit.h" #include "jit/zend_jit_internal.h" @@ -1183,6 +1184,11 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa, const zend_op *rt_op goto jit_failure; } goto done; + case ZEND_DEFINED: + if (!zend_jit_defined(&dasm_state, opline, b, &i, op_array, ssa)) { + goto jit_failure; + } + goto done; case ZEND_TYPE_CHECK: if (!zend_jit_type_check(&dasm_state, opline, b, &i, op_array, ssa)) { goto jit_failure; diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 5a0874054e49d..36a8a7eda9162 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -5637,7 +5637,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar | ZVAL_PTR_DTOR RX + offset, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN, 0, 1, 1, op_array->filename, opline->lineno } } else { - | mov FCARG1a, RX + | mov FCARG1a, RX | EXT_CALL zend_jit_vm_stack_free_args_helper, r0 } @@ -6047,6 +6047,79 @@ static int zend_jit_smart_false(dasm_State **Dst, const zend_op *opline, int b, return 1; } +static int zend_jit_defined(dasm_State **Dst, const zend_op *opline, int b, int *opnum, zend_op_array *op_array, zend_ssa *ssa) +{ + zend_bool smart_branch = 0; + uint32_t defined_label = (uint32_t)-1; + uint32_t undefined_label = (uint32_t)-1; + zval *zv = RT_CONSTANT(op_array, opline->op1); + + if (((opline+1)->opcode == ZEND_JMPZ || + (opline+1)->opcode == ZEND_JMPNZ || + (opline+1)->opcode == ZEND_JMPZNZ) && + (opline+1)->op1_type == IS_TMP_VAR && + (opline+1)->op1.var == opline->result.var) { + (*opnum)++; + smart_branch = 1; + } + + if (smart_branch) { + if ((opline+1)->opcode == ZEND_JMPZ) { + undefined_label = ssa->cfg.blocks[b].successors[0]; + } else if ((opline+1)->opcode == ZEND_JMPNZ) { + defined_label = ssa->cfg.blocks[b].successors[0]; + } else if ((opline+1)->opcode == ZEND_JMPZNZ) { + undefined_label = ssa->cfg.blocks[b].successors[0]; + defined_label = ssa->cfg.blocks[b].successors[1]; + } else { + ZEND_ASSERT(0); + } + } + + | // if (CACHED_PTR(Z_CACHE_SLOT_P(EX_CONSTANT(opline->op1)))) { + | mov r0, EX->run_time_cache + | mov r0, aword [r0 + Z_CACHE_SLOT_P(zv)] + | test r0, r0 + | jz >1 + |.cold_code + |1: + | LOAD_ADDR FCARG1a, zv + | mov FCARG2d, 0 + | EXT_CALL zend_quick_get_constant, r0 + | test r0, r0 + if (smart_branch) { + if (undefined_label != (uint32_t)-1) { + | jz =>undefined_label + } else { + | jz >3 + } + } + | mov r0, EX->run_time_cache + | mov r0, aword [r0 + Z_CACHE_SLOT_P(zv)] + if (smart_branch) { + if (defined_label != (uint32_t)-1) { + | jmp =>defined_label + } else { + | jmp >3 + } + } else { + | add eax, IS_FALSE + | SET_Z_TYPE_INFO FP + opline->result.var, eax + | jmp >3 + } + |.code + if (smart_branch) { + if (defined_label != (uint32_t)-1) { + | jmp =>defined_label + } + } else { + | SET_Z_TYPE_INFO FP + opline->result.var, IS_TRUE + } + |3: + + return 1; +} + static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, int b, int *opnum, zend_op_array *op_array, zend_ssa *ssa) { uint32_t op1_info, mask; From a38f65ab85071196fb4d2aadf2a2f4be0446d362 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 29 Nov 2016 22:53:50 +0300 Subject: [PATCH 356/569] JIT for IDENTICAL and NOT_IDENTICAL. --- ext/opcache/jit/zend_jit.c | 6 + ext/opcache/jit/zend_jit_x86.dasc | 308 ++++++++++++++++++++++++++++++ 2 files changed, 314 insertions(+) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 12a1ef6af4555..ee866590db128 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -1184,6 +1184,12 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa, const zend_op *rt_op goto jit_failure; } goto done; + case ZEND_IS_IDENTICAL: + case ZEND_IS_NOT_IDENTICAL: + if (!zend_jit_identical(&dasm_state, opline, b, &i, op_array, ssa)) { + goto jit_failure; + } + goto done; case ZEND_DEFINED: if (!zend_jit_defined(&dasm_state, opline, b, &i, op_array, ssa)) { goto jit_failure; diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 36a8a7eda9162..09bc81d1a2853 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -4677,6 +4677,314 @@ static int zend_jit_cmp(dasm_State **Dst, const zend_op *opline, int b, int *opn return 1; } +static int zend_jit_identical(dasm_State **Dst, const zend_op *opline, int b, int *opnum, zend_op_array *op_array, zend_ssa *ssa) +{ + zend_bool smart_branch = 0; + uint32_t identical_label = (uint32_t)-1; + uint32_t not_identical_label = (uint32_t)-1; + uint32_t op1_info, op2_info; + + op1_info = OP1_INFO(); + op2_info = OP2_INFO(); + if (((opline+1)->opcode == ZEND_JMPZ || + (opline+1)->opcode == ZEND_JMPNZ || + (opline+1)->opcode == ZEND_JMPZNZ) && + (opline+1)->op1_type == IS_TMP_VAR && + (opline+1)->op1.var == opline->result.var) { + (*opnum)++; + smart_branch = 1; + } + + if (smart_branch) { + if (opline->opcode == ZEND_IS_IDENTICAL) { + if ((opline+1)->opcode == ZEND_JMPZ) { + not_identical_label = ssa->cfg.blocks[b].successors[0]; + } else if ((opline+1)->opcode == ZEND_JMPNZ) { + identical_label = ssa->cfg.blocks[b].successors[0]; + } else if ((opline+1)->opcode == ZEND_JMPZNZ) { + not_identical_label = ssa->cfg.blocks[b].successors[0]; + identical_label = ssa->cfg.blocks[b].successors[1]; + } else { + ZEND_ASSERT(0); + } + } else if (opline->opcode == ZEND_IS_NOT_IDENTICAL) { + if ((opline+1)->opcode == ZEND_JMPZ) { + identical_label = ssa->cfg.blocks[b].successors[0]; + } else if ((opline+1)->opcode == ZEND_JMPNZ) { + not_identical_label = ssa->cfg.blocks[b].successors[0]; + } else if ((opline+1)->opcode == ZEND_JMPZNZ) { + identical_label = ssa->cfg.blocks[b].successors[0]; + not_identical_label = ssa->cfg.blocks[b].successors[1]; + } else { + ZEND_ASSERT(0); + } + } else { + ZEND_ASSERT(0); + } + } + + if ((op1_info & MAY_BE_UNDEF) && (op2_info & MAY_BE_UNDEF)) { + op1_info |= MAY_BE_NULL; + op2_info |= MAY_BE_NULL; + | LOAD_ZVAL_ADDR FCARG1a, IS_CV, opline->op1 + | IF_Z_TYPE FCARG1a, IS_UNDEF, >1 + |.cold_code + |1: + | // zend_error(E_NOTICE, "Undefined variable: %s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); + | SAVE_VALID_OPLINE opline + | mov FCARG1d, opline->op1.var + | EXT_CALL zend_jit_undefined_op_helper, r0 + || if (zend_may_throw(opline, op_array, ssa)) { + || zend_jit_check_exception(Dst); + || } + | LOAD_ADDR FCARG1a, &EG(uninitialized_zval) + | jmp >1 + |.code + |1: + | LOAD_ZVAL_ADDR FCARG2a, IS_CV, opline->op2 + | IF_Z_TYPE FCARG2a, IS_UNDEF, >1 + |.cold_code + |1: + | // zend_error(E_NOTICE, "Undefined variable: %s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); + | SAVE_VALID_OPLINE opline + | mov aword [r4], FCARG1a // save + | mov FCARG1d, opline->op2.var + | EXT_CALL zend_jit_undefined_op_helper, r0 + || if (zend_may_throw(opline, op_array, ssa)) { + || zend_jit_check_exception(Dst); + || } + | mov FCARG1a, aword [r4] // restore + | LOAD_ADDR FCARG2a, &EG(uninitialized_zval) + | jmp >1 + |.code + |1: + } else if (op1_info & MAY_BE_UNDEF) { + op1_info |= MAY_BE_NULL; + | LOAD_ZVAL_ADDR FCARG1a, IS_CV, opline->op1 + | IF_Z_TYPE FCARG1a, IS_UNDEF, >1 + |.cold_code + |1: + | // zend_error(E_NOTICE, "Undefined variable: %s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); + | SAVE_VALID_OPLINE opline + | mov FCARG1d, opline->op1.var + | EXT_CALL zend_jit_undefined_op_helper, r0 + || if (zend_may_throw(opline, op_array, ssa)) { + || zend_jit_check_exception(Dst); + || } + | LOAD_ADDR FCARG1a, &EG(uninitialized_zval) + | jmp >1 + |.code + |1: + if (opline->op2_type != IS_CONST) { + | LOAD_ZVAL_ADDR FCARG2a, opline->op2_type, opline->op2 + } + } else if (op2_info & MAY_BE_UNDEF) { + op2_info |= MAY_BE_NULL; + | LOAD_ZVAL_ADDR FCARG2a, IS_CV, opline->op2 + | IF_Z_TYPE FCARG2a, IS_UNDEF, >1 + |.cold_code + |1: + | // zend_error(E_NOTICE, "Undefined variable: %s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); + | SAVE_VALID_OPLINE opline + | mov FCARG1d, opline->op2.var + | EXT_CALL zend_jit_undefined_op_helper, r0 + || if (zend_may_throw(opline, op_array, ssa)) { + || zend_jit_check_exception(Dst); + || } + | LOAD_ADDR FCARG2a, &EG(uninitialized_zval) + | jmp >1 + |.code + |1: + if (opline->op1_type != IS_CONST) { + | LOAD_ZVAL_ADDR FCARG1a, opline->op1_type, opline->op1 + } + } else { + if (opline->op1_type != IS_CONST) { + | LOAD_ZVAL_ADDR FCARG1a, opline->op1_type, opline->op1 + } + if (opline->op2_type != IS_CONST) { + | LOAD_ZVAL_ADDR FCARG2a, opline->op2_type, opline->op2 + } + } + if (opline->op1_type & (IS_CV|IS_VAR)) { + | ZVAL_DEREF FCARG1a, op1_info + } + if (opline->op2_type & (IS_CV|IS_VAR)) { + | ZVAL_DEREF FCARG2a, op2_info + } + + if ((op1_info & op2_info & MAY_BE_ANY) == 0) { + if (((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && + (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) || + ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) && + (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)))) { + | SAVE_VALID_OPLINE opline + | FREE_OP opline->op1_type, opline->op1, op1_info, 1, op_array, opline->lineno + | FREE_OP opline->op2_type, opline->op2, op2_info, 1, op_array, opline->lineno + zend_jit_check_exception(Dst); + } + if (smart_branch) { + if (not_identical_label != (uint32_t)-1) { + | jmp =>not_identical_label + } + } else { + | SET_Z_TYPE_INFO FP + opline->result.var, (opline->opcode == ZEND_IS_IDENTICAL ? IS_FALSE : IS_TRUE) + } + } else if (has_concrete_type(op1_info) && + has_concrete_type(op2_info) && + concrete_type(op1_info) == concrete_type(op2_info) && + concrete_type(op1_info) <= IS_TRUE) { + if (smart_branch) { + if (identical_label != (uint32_t)-1) { + | jmp =>identical_label + } + } else { + | SET_Z_TYPE_INFO FP + opline->result.var, (opline->opcode == ZEND_IS_IDENTICAL ? IS_TRUE : IS_FALSE) + } + } else if (opline->op1_type == IS_CONST && opline->op2_type == IS_CONST) { + if (zend_is_identical(RT_CONSTANT(op_array, opline->op1), RT_CONSTANT(op_array, opline->op2))) { + if (smart_branch) { + if (identical_label != (uint32_t)-1) { + | jmp =>identical_label + } + } else { + | SET_Z_TYPE_INFO FP + opline->result.var, (opline->opcode == ZEND_IS_IDENTICAL ? IS_TRUE : IS_FALSE) + } + } else { + if (smart_branch) { + if (not_identical_label != (uint32_t)-1) { + | jmp =>not_identical_label + } + } else { + | SET_Z_TYPE_INFO FP + opline->result.var, (opline->opcode == ZEND_IS_IDENTICAL ? IS_FALSE : IS_TRUE) + } + } + } else if (opline->op1_type == IS_CONST && Z_TYPE_P(RT_CONSTANT(op_array, opline->op1)) <= IS_TRUE) { + zval *val = RT_CONSTANT(op_array, opline->op1); + + | cmp byte [FCARG2a + offsetof(zval, u1.v.type)], Z_TYPE_P(val) + if (smart_branch) { + if (opline->op2_type == IS_VAR && (op2_info & MAY_BE_REF)) { + | jne >8 + | SAVE_VALID_OPLINE opline + | FREE_OP opline->op2_type, opline->op2, op2_info, 1, op_array, opline->lineno + zend_jit_check_exception(Dst); + if (identical_label != (uint32_t)-1) { + | jmp =>identical_label + } else { + | jmp >9 + } + |8: + } else if (identical_label != (uint32_t)-1) { + | je =>identical_label + } else { + | je >9 + } + } else { + if (opline->opcode == ZEND_IS_IDENTICAL) { + | sete al + } else { + | setne al + } + | movzx eax, al + | lea eax, [eax + 2] + | SET_Z_TYPE_INFO FP + opline->result.var, eax + } + if ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) && + (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) { + | SAVE_VALID_OPLINE opline + | FREE_OP opline->op2_type, opline->op2, op2_info, 1, op_array, opline->lineno + zend_jit_check_exception(Dst); + } + if (smart_branch && not_identical_label != (uint32_t)-1) { + | jmp =>not_identical_label + } + } else if (opline->op2_type == IS_CONST && Z_TYPE_P(RT_CONSTANT(op_array, opline->op2)) <= IS_TRUE) { + zval *val = RT_CONSTANT(op_array, opline->op2); + + | cmp byte [FCARG1a + offsetof(zval, u1.v.type)], Z_TYPE_P(val) + if (smart_branch) { + if (opline->op1_type == IS_VAR && (op1_info & MAY_BE_REF)) { + | jne >8 + | SAVE_VALID_OPLINE opline + | FREE_OP opline->op1_type, opline->op1, op1_info, 1, op_array, opline->lineno + zend_jit_check_exception(Dst); + if (identical_label != (uint32_t)-1) { + | jmp =>identical_label + } else { + | jmp >9 + } + |8: + } else if (identical_label != (uint32_t)-1) { + | je =>identical_label + } else { + | je >9 + } + } else { + if (opline->opcode == ZEND_IS_IDENTICAL) { + | sete al + } else { + | setne al + } + | movzx eax, al + | lea eax, [eax + 2] + | SET_Z_TYPE_INFO FP + opline->result.var, eax + } + if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && + (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) { + | SAVE_VALID_OPLINE opline + | FREE_OP opline->op1_type, opline->op1, op1_info, 1, op_array, opline->lineno + zend_jit_check_exception(Dst); + } + if (smart_branch && not_identical_label != (uint32_t)-1) { + | jmp =>not_identical_label + } + } else { + if (opline->op1_type == IS_CONST) { + | LOAD_ZVAL_ADDR FCARG1a, opline->op1_type, opline->op1 + } + if (opline->op2_type == IS_CONST) { + | LOAD_ZVAL_ADDR FCARG2a, opline->op2_type, opline->op2 + } + | EXT_CALL zend_is_identical, r0 + if (((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && + (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) || + ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) && + (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)))) { + | mov aword [r4], r0 // save + | SAVE_VALID_OPLINE opline + | FREE_OP opline->op1_type, opline->op1, op1_info, 1, op_array, opline->lineno + | FREE_OP opline->op2_type, opline->op2, op2_info, 1, op_array, opline->lineno + zend_jit_check_exception(Dst); + | mov r0, aword [r4] // restore + } + | test r0, r0 + if (smart_branch) { + if (not_identical_label != (uint32_t)-1) { + | jz =>not_identical_label + if (identical_label != (uint32_t)-1) { + | jmp =>identical_label + } + } else if (identical_label != (uint32_t)-1) { + | jnz =>identical_label + } + } else { + if (opline->opcode == ZEND_IS_IDENTICAL) { + | setnz al + } else { + | setz al + } + | movzx eax, al + | lea eax, [eax + 2] + | SET_Z_TYPE_INFO FP + opline->result.var, eax + } + } + + |9: + + return 1; +} + static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa) { uint32_t op1_info = OP1_INFO(); From ce00b8325b5b0d6ec705d0da38712b998e0b3784 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 30 Nov 2016 17:37:19 +0300 Subject: [PATCH 357/569] JIT for SEND_VAR_NO_REF/SEND_VAR_NO_REF_EX --- ext/opcache/jit/zend_jit.c | 2 + ext/opcache/jit/zend_jit_x86.dasc | 72 ++++++++++++++++++++++++++++--- 2 files changed, 68 insertions(+), 6 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index ee866590db128..6ce841140d7ed 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -1163,6 +1163,8 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa, const zend_op *rt_op goto done; case ZEND_SEND_VAR: case ZEND_SEND_VAR_EX: + case ZEND_SEND_VAR_NO_REF: + case ZEND_SEND_VAR_NO_REF_EX: if (!zend_jit_send_var(&dasm_state, opline, op_array, ssa)) { goto jit_failure; } diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 09bc81d1a2853..97992a8d6a911 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -6200,7 +6200,9 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, zend_op_ar uint32_t op1_info; uint32_t arg_num = opline->op2.num; - if (opline->opcode == ZEND_SEND_VAR_EX && arg_num > MAX_ARG_FLAG_NUM) { + if ((opline->opcode == ZEND_SEND_VAR_EX || + opline->opcode == ZEND_SEND_VAR_NO_REF_EX) && + arg_num > MAX_ARG_FLAG_NUM) { goto fallback; } @@ -6212,8 +6214,14 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, zend_op_ar | mov RX, EX->call } - if (opline->opcode == ZEND_SEND_VAR_EX) { - uint32_t mask = ZEND_SEND_BY_REF << ((arg_num + 3) * 2); + if (opline->opcode == ZEND_SEND_VAR_EX || opline->opcode == ZEND_SEND_VAR_NO_REF_EX) { + uint32_t mask; + + if (opline->opcode == ZEND_SEND_VAR_EX) { + mask = ZEND_SEND_BY_REF << ((arg_num + 3) * 2); + } else { + mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2); + } | mov r0, EX:RX->func if (arg_num <= MAX_ARG_FLAG_NUM) { @@ -6226,8 +6234,38 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, zend_op_ar |.cold_code |1: - if (!zend_jit_send_ref(Dst, opline, op_array, ssa, 1)) { - return 0; + if (opline->opcode == ZEND_SEND_VAR_EX) { + if (!zend_jit_send_ref(Dst, opline, op_array, ssa, 1)) { + return 0; + } + } else if (opline->opcode == ZEND_SEND_VAR_NO_REF_EX) { + mask = ZEND_SEND_PREFER_REF << ((arg_num + 3) * 2); + + | ZVAL_COPY_VALUE RX + opline->result.var, FP + opline->op1.var, op1_info, r1, ecx, r2 + if (op1_info & MAY_BE_REF) { + | cmp cl, IS_REFERENCE + | je >7 + } + | test dword [r0 + offsetof(zend_function, quick_arg_flags)], mask + | jnz >7 + | SAVE_VALID_OPLINE opline + |.if X64 + | mov CARG1, E_NOTICE + | LOAD_ADDR CARG2, "Only variables should be passed by reference" + | EXT_CALL zend_error, r0 + | add r4, 8 + |.else + | sub r4, 8 + | push "Only variables should be passed by reference" + | push E_NOTICE + | EXT_CALL zend_error, r0 + | add r4, 16 + |.endif + if (!zend_jit_check_exception(Dst)) { + return 0; + } + } else { + ZEND_ASSERT(0); } | jmp >7 @@ -6252,7 +6290,29 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, zend_op_ar } } - if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { + if (opline->opcode == ZEND_SEND_VAR_NO_REF) { + | ZVAL_COPY_VALUE RX + opline->result.var, FP + opline->op1.var, op1_info, r1, ecx, r2 + if (op1_info & MAY_BE_REF) { + | cmp cl, IS_REFERENCE + | je >7 + } + | SAVE_VALID_OPLINE opline + |.if X64 + | mov CARG1, E_NOTICE + | LOAD_ADDR CARG2, "Only variables should be passed by reference" + | EXT_CALL zend_error, r0 + | add r4, 8 + |.else + | sub r4, 8 + | push "Only variables should be passed by reference" + | push E_NOTICE + | EXT_CALL zend_error, r0 + | add r4, 16 + |.endif + if (!zend_jit_check_exception(Dst)) { + return 0; + } + } else if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { if (op1_info & MAY_BE_REF) { if (opline->op1_type == IS_CV) { | lea FCARG1a, [FP + opline->op1.var] From 2bcd8d47ef214d32a3069ea970c89933e3c53cbf Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 7 Dec 2016 02:36:47 +0300 Subject: [PATCH 358/569] Fixed memory leaks caused by exceptions thrown from destructors (JIT fixes). --- ext/opcache/jit/zend_jit_x86.dasc | 217 +++++++++++++++--------------- 1 file changed, 110 insertions(+), 107 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 97992a8d6a911..1e0481b8c9436 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -887,9 +887,29 @@ static void* dasm_labels[zend_lb_MAX]; || } |.endmacro +|.macro SAVE_VALID_OPLINE, op +|| if (op == last_valid_opline) { +| mov aword EX->opline, IP +|| } else { +|.if X64 +|| if (IS_32BIT(op)) { +| mov aword EX->opline, ((ptrdiff_t)op) +|| } else { +| mov64 r0, ((ptrdiff_t)op) +| mov aword EX->opline, r0 +|| } +|.else +| mov aword EX->opline, op +|.endif +|| } +|.endmacro + // zval should be in FCARG1a -|.macro ZVAL_DTOR_FUNC, var_info, filename, lineno // arg1 must be in FCARG1a +|.macro ZVAL_DTOR_FUNC, var_info, filename, opline // arg1 must be in FCARG1a ||do { +|| if (opline) { +| SAVE_VALID_OPLINE opline +|| } || if (has_concrete_type((var_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { || zend_uchar type = concrete_type((var_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)); || if (type == IS_STRING && !ZEND_DEBUG) { @@ -917,6 +937,7 @@ static void* dasm_labels[zend_lb_MAX]; || } || } || if (ZEND_DEBUG) { +|| uint32_t lineno = opline ? ((zend_op*)opline)->lineno : 0; || if (filename) { | LOAD_ADDR FCARG2a, ZSTR_VAL((zend_string*)filename) || } else { @@ -932,7 +953,7 @@ static void* dasm_labels[zend_lb_MAX]; ||} while(0); |.endmacro -|.macro ZVAL_PTR_DTOR, zv, op_info, gc, cold, safe, filename, lineno +|.macro ZVAL_PTR_DTOR, zv, op_info, gc, cold, safe, filename, opline || if ((op_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { || if ((op_info) & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { | // if (Z_REFCOUNTED_P(cv)) { @@ -960,7 +981,7 @@ static void* dasm_labels[zend_lb_MAX]; | SET_Z_TYPE_INFO zv, IS_NULL || } | // zval_dtor_func(r); -| ZVAL_DTOR_FUNC op_info, filename, lineno +| ZVAL_DTOR_FUNC op_info, filename, opline || if (gc && RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT))) { | jmp >4 || } @@ -991,9 +1012,9 @@ static void* dasm_labels[zend_lb_MAX]; || } |.endmacro -|.macro FREE_OP, op_type, op, op_info, cold, op_array, lineno +|.macro FREE_OP, op_type, op, op_info, cold, op_array, opline ||if (op_type & (IS_VAR|IS_TMP_VAR)) { -| ZVAL_PTR_DTOR FP + op.var, op_info, 0, cold, 0, op_array->filename, lineno +| ZVAL_PTR_DTOR FP + op.var, op_info, 0, cold, 0, op_array->filename, opline ||} |.endmacro @@ -1185,23 +1206,6 @@ static void* dasm_labels[zend_lb_MAX]; |1: |.endmacro -|.macro SAVE_VALID_OPLINE, opline -|| if (opline == last_valid_opline) { -| mov aword EX->opline, IP -|| } else { -|.if X64 -|| if (IS_32BIT(opline)) { -| mov aword EX->opline, ((ptrdiff_t)opline) -|| } else { -| mov64 r0, ((ptrdiff_t)opline) -| mov aword EX->opline, r0 -|| } -|.else -| mov aword EX->opline, opline -|.endif -|| } -|.endmacro - |.macro UNDEFINED_OFFSET, opline || if (opline == last_valid_opline) { | call ->undefined_offset_ex @@ -1335,6 +1339,15 @@ static int zend_jit_exception_handler_stub(dasm_State **Dst) return 1; } +static int zend_jit_exception_handler_undef_stub(dasm_State **Dst) +{ + |->exception_handler_undef: + | SET_Z_TYPE_INFO FP + r0, IS_UNDEF + | jmp ->exception_handler + + return 1; +} + static int zend_jit_leave_function_stub(dasm_State **Dst) { |->leave_function_handler: @@ -1356,22 +1369,11 @@ static int zend_jit_leave_throw_stub(dasm_State **Dst) | je >5 | // EG(opline_before_exception) = opline; | mov aword [&EG(opline_before_exception)], IP - | // if (RETURN_VALUE_USED(old_opline)) { - | cmp byte OP:IP->result_type, IS_UNUSED - | je >5 - | // zval_ptr_dtor(EX_VAR(old_opline->result.var)); - |.if X64 - | movsxd r0, dword OP:IP->result.var - |.else - | mov r0, OP:IP->result.var - |.endif - | add r0, FP - | ZVAL_PTR_DTOR r0, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 1, 0, 0, NULL, 0 |5: | // opline = EG(exception_op); | LOAD_ADDR IP, &EG(exception_op) - | add r4, SPAD // stack alignment - | EXT_JMP EG(exception_op)->handler, r0 + | // HANDLE_EXCEPTION() + | jmp ->exception_handler return 1; } @@ -1389,18 +1391,6 @@ static int zend_jit_icall_throw_stub(dasm_State **Dst) | EXT_CALL zend_throw_exception_internal, r0 | add r4, 16 |.endif - | // if (opline->result_type != IS_UNUSED) { - | cmp byte OP:IP->result_type, IS_UNUSED - | je >5 - | // zval_ptr_dtor(EX_VAR(opline->result.var)); - |.if X64 - | movsxd r0, dword OP:IP->result.var - |.else - | mov r0, OP:IP->result.var - |.endif - | add r0, FP - | ZVAL_PTR_DTOR r0, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 1, 0, 0, NULL, 0 - |5: | // HANDLE_EXCEPTION() | jmp ->exception_handler @@ -1440,7 +1430,7 @@ static int zend_jit_throw_cannot_pass_by_ref_stub(dasm_State **Dst) | mov r0, OP:RX->op1.var |.endif | add r0, FP - | ZVAL_PTR_DTOR r0, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, 0, 0, NULL, 0 + | ZVAL_PTR_DTOR r0, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, 0, 0, NULL, NULL |9: | jmp ->exception_handler @@ -1636,6 +1626,7 @@ static int zend_jit_not_obj_stub(dasm_State **Dst) static const zend_jit_stub zend_jit_stubs[] = { JIT_STUB(interrupt_handler), JIT_STUB(exception_handler), + JIT_STUB(exception_handler_undef), JIT_STUB(leave_function), JIT_STUB(leave_throw), JIT_STUB(icall_throw), @@ -1724,6 +1715,17 @@ static int zend_jit_check_exception(dasm_State **Dst) return 1; } +static int zend_jit_check_exception_undef_result(dasm_State **Dst, const zend_op *opline) +{ + if (opline->result_type & (IS_TMP_VAR|IS_VAR)) { + | cmp aword [&EG(exception)], 0 + | mov r0, opline->result.var + | jne ->exception_handler_undef + return 1; + } + return zend_jit_check_exception(Dst); +} + static int zend_jit_handler(dasm_State **Dst, const zend_op *opline, int may_throw) { if (!zend_jit_set_valid_ip(Dst, opline)) { @@ -2338,7 +2340,7 @@ static int zend_jit_shift(dasm_State **Dst, const zend_op *opline, zend_op_array |1: } - | FREE_OP opline->op1_type, opline->op1, op1_info, 1, op_array, opline->lineno + | FREE_OP opline->op1_type, opline->op1, op1_info, 1, op_array, opline return 1; @@ -2548,8 +2550,8 @@ static int zend_jit_math_helper(dasm_State **Dst, || } else { || ZEND_ASSERT(0); || } - | FREE_OP op1_type, op1, op1_info, 0, op_array, opline->lineno - | FREE_OP op2_type, op2, op2_info, 0, op_array, opline->lineno + | FREE_OP op1_type, op1, op1_info, 0, op_array, opline + | FREE_OP op2_type, op2, op2_info, 0, op_array, opline || if (zend_may_throw(opline, op_array, ssa)) { || zend_jit_check_exception(Dst); || } @@ -2665,8 +2667,8 @@ static int zend_jit_concat_helper(dasm_State **Dst, } | EXT_CALL zend_jit_fast_concat_helper, r0 } - | FREE_OP op1_type, op1, op1_info, 0, op_array, opline->lineno - | FREE_OP op2_type, op2, op2_info, 0, op_array, opline->lineno + | FREE_OP op1_type, op1, op1_info, 0, op_array, opline + | FREE_OP op2_type, op2, op2_info, 0, op_array, opline |5: } if ((op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) || @@ -2700,8 +2702,8 @@ static int zend_jit_concat_helper(dasm_State **Dst, |.endif } | EXT_CALL concat_function, r0 - | FREE_OP op1_type, op1, op1_info, 0, op_array, opline->lineno - | FREE_OP op2_type, op2, op2_info, 0, op_array, opline->lineno + | FREE_OP op1_type, op1, op1_info, 0, op_array, opline + | FREE_OP op2_type, op2, op2_info, 0, op_array, opline || if (zend_may_throw(opline, op_array, ssa)) { || zend_jit_check_exception(Dst); || } @@ -3239,7 +3241,7 @@ static int zend_jit_assign_to_variable(dasm_State **Dst, return 0; } | mov FCARG1a, aword [r4] // restore - | ZVAL_DTOR_FUNC var_info, op_array->filename, opline->lineno + | ZVAL_DTOR_FUNC var_info, op_array->filename, opline | jmp >3 |4: } @@ -3518,7 +3520,7 @@ static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, zend_op_ } #endif - | FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, 0, op_array, opline->lineno + | FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, 0, op_array, opline } if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { @@ -3537,7 +3539,7 @@ static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, zend_op_ #endif |9: - | FREE_OP opline->op2_type, opline->op2, op2_info, 0, op_array, opline->lineno + | FREE_OP opline->op2_type, opline->op2, op2_info, 0, op_array, opline if (zend_may_throw(opline, op_array, ssa)) { zend_jit_check_exception(Dst); @@ -4648,11 +4650,11 @@ static int zend_jit_cmp(dasm_State **Dst, const zend_op *opline, int b, int *opn } | EXT_CALL compare_function, r0 || if (opline->opcode != ZEND_CASE) { - | FREE_OP opline->op1_type, opline->op1, op1_info, 0, op_array, opline->lineno + | FREE_OP opline->op1_type, opline->op1, op1_info, 0, op_array, opline || } - | FREE_OP opline->op2_type, opline->op2, op2_info, 0, op_array, opline->lineno + | FREE_OP opline->op2_type, opline->op2, op2_info, 0, op_array, opline || if (zend_may_throw(opline, op_array, ssa)) { - || zend_jit_check_exception(Dst); + || zend_jit_check_exception_undef_result(Dst, opline); || } if (!zend_jit_cmp_slow(Dst, opline, b, op_array, ssa)) { return 0; @@ -4735,7 +4737,7 @@ static int zend_jit_identical(dasm_State **Dst, const zend_op *opline, int b, in | mov FCARG1d, opline->op1.var | EXT_CALL zend_jit_undefined_op_helper, r0 || if (zend_may_throw(opline, op_array, ssa)) { - || zend_jit_check_exception(Dst); + || zend_jit_check_exception_undef_result(Dst, opline); || } | LOAD_ADDR FCARG1a, &EG(uninitialized_zval) | jmp >1 @@ -4751,7 +4753,7 @@ static int zend_jit_identical(dasm_State **Dst, const zend_op *opline, int b, in | mov FCARG1d, opline->op2.var | EXT_CALL zend_jit_undefined_op_helper, r0 || if (zend_may_throw(opline, op_array, ssa)) { - || zend_jit_check_exception(Dst); + || zend_jit_check_exception_undef_result(Dst, opline); || } | mov FCARG1a, aword [r4] // restore | LOAD_ADDR FCARG2a, &EG(uninitialized_zval) @@ -4769,7 +4771,7 @@ static int zend_jit_identical(dasm_State **Dst, const zend_op *opline, int b, in | mov FCARG1d, opline->op1.var | EXT_CALL zend_jit_undefined_op_helper, r0 || if (zend_may_throw(opline, op_array, ssa)) { - || zend_jit_check_exception(Dst); + || zend_jit_check_exception_undef_result(Dst, opline); || } | LOAD_ADDR FCARG1a, &EG(uninitialized_zval) | jmp >1 @@ -4789,7 +4791,7 @@ static int zend_jit_identical(dasm_State **Dst, const zend_op *opline, int b, in | mov FCARG1d, opline->op2.var | EXT_CALL zend_jit_undefined_op_helper, r0 || if (zend_may_throw(opline, op_array, ssa)) { - || zend_jit_check_exception(Dst); + || zend_jit_check_exception_undef_result(Dst, opline); || } | LOAD_ADDR FCARG2a, &EG(uninitialized_zval) | jmp >1 @@ -4819,16 +4821,17 @@ static int zend_jit_identical(dasm_State **Dst, const zend_op *opline, int b, in ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) && (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)))) { | SAVE_VALID_OPLINE opline - | FREE_OP opline->op1_type, opline->op1, op1_info, 1, op_array, opline->lineno - | FREE_OP opline->op2_type, opline->op2, op2_info, 1, op_array, opline->lineno - zend_jit_check_exception(Dst); + | FREE_OP opline->op1_type, opline->op1, op1_info, 1, op_array, opline + | FREE_OP opline->op2_type, opline->op2, op2_info, 1, op_array, opline } if (smart_branch) { + zend_jit_check_exception_undef_result(Dst, opline); if (not_identical_label != (uint32_t)-1) { | jmp =>not_identical_label } } else { | SET_Z_TYPE_INFO FP + opline->result.var, (opline->opcode == ZEND_IS_IDENTICAL ? IS_FALSE : IS_TRUE) + zend_jit_check_exception(Dst); } } else if (has_concrete_type(op1_info) && has_concrete_type(op2_info) && @@ -4867,8 +4870,8 @@ static int zend_jit_identical(dasm_State **Dst, const zend_op *opline, int b, in if (opline->op2_type == IS_VAR && (op2_info & MAY_BE_REF)) { | jne >8 | SAVE_VALID_OPLINE opline - | FREE_OP opline->op2_type, opline->op2, op2_info, 1, op_array, opline->lineno - zend_jit_check_exception(Dst); + | FREE_OP opline->op2_type, opline->op2, op2_info, 1, op_array, opline + zend_jit_check_exception_undef_result(Dst, opline); if (identical_label != (uint32_t)-1) { | jmp =>identical_label } else { @@ -4893,8 +4896,8 @@ static int zend_jit_identical(dasm_State **Dst, const zend_op *opline, int b, in if ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) && (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) { | SAVE_VALID_OPLINE opline - | FREE_OP opline->op2_type, opline->op2, op2_info, 1, op_array, opline->lineno - zend_jit_check_exception(Dst); + | FREE_OP opline->op2_type, opline->op2, op2_info, 1, op_array, opline + zend_jit_check_exception_undef_result(Dst, opline); } if (smart_branch && not_identical_label != (uint32_t)-1) { | jmp =>not_identical_label @@ -4907,8 +4910,8 @@ static int zend_jit_identical(dasm_State **Dst, const zend_op *opline, int b, in if (opline->op1_type == IS_VAR && (op1_info & MAY_BE_REF)) { | jne >8 | SAVE_VALID_OPLINE opline - | FREE_OP opline->op1_type, opline->op1, op1_info, 1, op_array, opline->lineno - zend_jit_check_exception(Dst); + | FREE_OP opline->op1_type, opline->op1, op1_info, 1, op_array, opline + zend_jit_check_exception_undef_result(Dst, opline); if (identical_label != (uint32_t)-1) { | jmp =>identical_label } else { @@ -4933,8 +4936,8 @@ static int zend_jit_identical(dasm_State **Dst, const zend_op *opline, int b, in if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) { | SAVE_VALID_OPLINE opline - | FREE_OP opline->op1_type, opline->op1, op1_info, 1, op_array, opline->lineno - zend_jit_check_exception(Dst); + | FREE_OP opline->op1_type, opline->op1, op1_info, 1, op_array, opline + zend_jit_check_exception_undef_result(Dst, opline); } if (smart_branch && not_identical_label != (uint32_t)-1) { | jmp =>not_identical_label @@ -4953,9 +4956,9 @@ static int zend_jit_identical(dasm_State **Dst, const zend_op *opline, int b, in (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)))) { | mov aword [r4], r0 // save | SAVE_VALID_OPLINE opline - | FREE_OP opline->op1_type, opline->op1, op1_info, 1, op_array, opline->lineno - | FREE_OP opline->op2_type, opline->op2, op2_info, 1, op_array, opline->lineno - zend_jit_check_exception(Dst); + | FREE_OP opline->op1_type, opline->op1, op1_info, 1, op_array, opline + | FREE_OP opline->op2_type, opline->op2, op2_info, 1, op_array, opline + zend_jit_check_exception_undef_result(Dst, opline); | mov r0, aword [r4] // restore } | test r0, r0 @@ -5125,7 +5128,7 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, int b, | EXT_CALL zend_jit_undefined_op_helper, r0 if (zend_may_throw(opline, op_array, ssa)) { - if (!zend_jit_check_exception(Dst)) { + if (!zend_jit_check_exception_undef_result(Dst, opline)) { return 0; } } @@ -5207,9 +5210,9 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, int b, | add eax, 2 } | SET_Z_TYPE_INFO FP + opline->result.var, eax - | FREE_OP opline->op1_type, opline->op1, op1_info, !(op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)), op_array, opline->lineno + | FREE_OP opline->op1_type, opline->op1, op1_info, !(op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)), op_array, opline if (zend_may_throw(opline, op_array, ssa)) { - if (!zend_jit_check_exception(Dst)) { + if (!zend_jit_check_exception_undef_result(Dst, opline)) { return 0; } } @@ -5240,12 +5243,12 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, int b, | dec dword [FCARG1a] | jnz >3 | mov aword [r4], r0 // save - | ZVAL_DTOR_FUNC op1_info, op_array->filename, opline->lineno + | ZVAL_DTOR_FUNC op1_info, op_array->filename, opline | mov r0, aword [r4] // restore |3: } if (zend_may_throw(opline, op_array, ssa)) { - if (!zend_jit_check_exception(Dst)) { + if (!zend_jit_check_exception_undef_result(Dst, opline)) { return 0; } } @@ -5942,7 +5945,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar if (func) { for (i = 0; i < call_info->num_args; i++ ) { uint32_t offset = (uint32_t)(uintptr_t)ZEND_CALL_VAR_NUM(NULL, i); - | ZVAL_PTR_DTOR RX + offset, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN, 0, 1, 1, op_array->filename, opline->lineno + | ZVAL_PTR_DTOR RX + offset, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN, 0, 1, 1, op_array->filename, opline } } else { | mov FCARG1a, RX @@ -5994,7 +5997,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar (MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN); if (func_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { - | ZVAL_PTR_DTOR r4 + 8, func_info, 1, 1, 0, op_array->filename, opline->lineno + | ZVAL_PTR_DTOR r4 + 8, func_info, 1, 1, 0, op_array->filename, opline } | add r4, 16 /* revert alloca() */ } @@ -6189,7 +6192,7 @@ static int zend_jit_send_ref(dasm_State **Dst, const zend_op *opline, zend_op_ar } |6: - | FREE_OP opline->op1_type, opline->op1, op1_info, !cold, op_array, opline->lineno + | FREE_OP opline->op1_type, opline->op1, op1_info, !cold, op_array, opline |7: return 1; @@ -6542,12 +6545,12 @@ static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, int b, i } if (!(op1_info & (MAY_BE_ANY - mask))) { - | FREE_OP opline->op1_type, opline->op1, op1_info, 1, op_array, opline->lineno + | FREE_OP opline->op1_type, opline->op1, op1_info, 1, op_array, opline if (!zend_jit_smart_true(Dst, opline, b, op_array, ssa, 0)) { return 0; } } else if (!(op1_info & mask)) { - | FREE_OP opline->op1_type, opline->op1, op1_info, 1, op_array, opline->lineno + | FREE_OP opline->op1_type, opline->op1, op1_info, 1, op_array, opline if (!zend_jit_smart_false(Dst, opline, b, op_array, ssa, 0)) { return 0; } @@ -6580,7 +6583,7 @@ static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, int b, i } | mov byte [r4], al // save | // zval_dtor_func(r); - | ZVAL_DTOR_FUNC op1_info, op_array->filename, opline->lineno + | ZVAL_DTOR_FUNC op1_info, op_array->filename, opline | mov cl, byte [r4] // restore |jmp >2 } @@ -6629,7 +6632,7 @@ static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, int b, i | movzx eax, al | add eax, 2 | SET_Z_TYPE_INFO FP + opline->result.var, eax - | FREE_OP opline->op1_type, opline->op1, op1_info, 1, op_array, opline->lineno + | FREE_OP opline->op1_type, opline->op1, op1_info, 1, op_array, opline } } else if (opline->extended_value == IS_RESOURCE) { ZEND_ASSERT(0); @@ -6657,7 +6660,7 @@ static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, int b, i } | mov byte [r4], al // save | // zval_dtor_func(r); - | ZVAL_DTOR_FUNC op1_info, op_array->filename, opline->lineno + | ZVAL_DTOR_FUNC op1_info, op_array->filename, opline | mov cl, byte [r4] // restore |jmp >2 } @@ -6704,7 +6707,7 @@ static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, int b, i | movzx eax, al | add eax, 2 | SET_Z_TYPE_INFO FP + opline->result.var, eax - | FREE_OP opline->op1_type, opline->op1, op1_info, 1, op_array, opline->lineno + | FREE_OP opline->op1_type, opline->op1, op1_info, 1, op_array, opline } } } @@ -6758,7 +6761,7 @@ static int zend_jit_free_compiled_variables(dasm_State **Dst, const zend_op *opl if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { uint32_t offset = (uint32_t)(uintptr_t)ZEND_CALL_VAR_NUM(NULL, i); - | ZVAL_PTR_DTOR FP + offset, info, 1, 1, 1, op_array->filename, opline->lineno + | ZVAL_PTR_DTOR FP + offset, info, 1, 1, 1, op_array->filename, opline } } return 1; @@ -6913,7 +6916,7 @@ static int zend_jit_return(dasm_State **Dst, const zend_op *opline, zend_op_arra || } | //SAVE_OPLINE() | SAVE_VALID_OPLINE opline - | ZVAL_DTOR_FUNC op1_info, op_array->filename, opline->lineno + | ZVAL_DTOR_FUNC op1_info, op_array->filename, opline | //????mov r1, EX->return_value // reload ??? || } || if (jit_return_label >= 0) { @@ -7173,8 +7176,8 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, const zend_op *opline, zend } #endif - | FREE_OP opline->op2_type, opline->op2, op2_info, 0, op_array, opline->lineno - | FREE_OP opline->op1_type, opline->op1, op1_info, 0, op_array, opline->lineno + | FREE_OP opline->op2_type, opline->op2, op2_info, 0, op_array, opline + | FREE_OP opline->op1_type, opline->op1, op1_info, 0, op_array, opline if (zend_may_throw(opline, op_array, ssa)) { if (!zend_jit_check_exception(Dst)) { @@ -7250,10 +7253,10 @@ static int zend_jit_isset_isempty_dim(dasm_State **Dst, const zend_op *opline, i #endif |8: - | FREE_OP opline->op2_type, opline->op2, op2_info, 0, op_array, opline->lineno - | FREE_OP opline->op1_type, opline->op1, op1_info, 0, op_array, opline->lineno + | FREE_OP opline->op2_type, opline->op2, op2_info, 0, op_array, opline + | FREE_OP opline->op1_type, opline->op1, op1_info, 0, op_array, opline if (zend_may_throw(opline, op_array, ssa)) { - if (!zend_jit_check_exception(Dst)) { + if (!zend_jit_check_exception_undef_result(Dst, opline)) { return 0; } } @@ -7277,10 +7280,10 @@ static int zend_jit_isset_isempty_dim(dasm_State **Dst, const zend_op *opline, i } |9: // not found - | FREE_OP opline->op2_type, opline->op2, op2_info, 0, op_array, opline->lineno - | FREE_OP opline->op1_type, opline->op1, op1_info, 0, op_array, opline->lineno + | FREE_OP opline->op2_type, opline->op2, op2_info, 0, op_array, opline + | FREE_OP opline->op1_type, opline->op1, op1_info, 0, op_array, opline if (zend_may_throw(opline, op_array, ssa)) { - if (!zend_jit_check_exception(Dst)) { + if (!zend_jit_check_exception_undef_result(Dst, opline)) { return 0; } } @@ -7437,7 +7440,7 @@ static int zend_jit_bind_global(dasm_State **Dst, const zend_op *opline, zend_op | jnz >3 } | mov aword [r4], r0 // save - | ZVAL_DTOR_FUNC op1_info, op_array->filename, opline->lineno + | ZVAL_DTOR_FUNC op1_info, op_array->filename, opline | mov r0, aword [r4] // restore | jmp >5 if (op1_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) { @@ -7560,7 +7563,7 @@ static int zend_jit_recv_init(dasm_State **Dst, const zend_op *opline, zend_op_a |.cold_code if (has_slow & 1) { |7: - | ZVAL_PTR_DTOR FP + opline->result.var, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN, 1, 0, 0, op_array->filename, opline->lineno + | ZVAL_PTR_DTOR FP + opline->result.var, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN, 1, 0, 0, op_array->filename, opline | SET_Z_TYPE_INFO FP + opline->result.var, IS_UNDEF | jmp <5 } @@ -7821,7 +7824,7 @@ static int zend_jit_fetch_obj_read(dasm_State **Dst, zend_op *opline, zend_op_ar |.code; |9: // END - | FREE_OP opline->op1_type, opline->op1, op1_info, 1, op_array, opline->lineno + | FREE_OP opline->op1_type, opline->op1, op1_info, 1, op_array, opline if (zend_may_throw(opline, op_array, ssa)) { if (!zend_jit_check_exception(Dst)) { @@ -7854,7 +7857,7 @@ static int zend_jit_free(dasm_State **Dst, const zend_op *opline, zend_op_array | EXT_CALL zend_hash_iterator_del, r0 |7: } - | ZVAL_PTR_DTOR FP + opline->op1.var, op1_info, 0, 0, 0, op_array->filename, opline->lineno + | ZVAL_PTR_DTOR FP + opline->op1.var, op1_info, 0, 0, 0, op_array->filename, opline if (zend_may_throw(opline, op_array, ssa)) { if (!zend_jit_check_exception(Dst)) { return 0; From 75bb7a5040c4b8e6f4c3de2a9b4dcefc0fd17585 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 8 Dec 2016 14:15:49 +0300 Subject: [PATCH 359/569] Added ZEND_CFG_SPLIT_AT_LIVE_RANGES flag --- ext/opcache/jit/zend_jit.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 6ce841140d7ed..5c203be11c23f 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -880,7 +880,7 @@ static int zend_may_overflow(const zend_op *opline, zend_op_array *op_array, zen static int zend_jit_build_cfg(zend_op_array *op_array, zend_cfg *cfg, uint32_t *flags) { - if (zend_build_cfg(&CG(arena), op_array, ZEND_CFG_STACKLESS | ZEND_CFG_RECV_ENTRY | ZEND_RT_CONSTANTS | ZEND_CFG_NO_ENTRY_PREDECESSORS | ZEND_SSA_RC_INFERENCE_FLAG, cfg, flags) != SUCCESS) { + if (zend_build_cfg(&CG(arena), op_array, ZEND_CFG_STACKLESS | ZEND_CFG_RECV_ENTRY | ZEND_RT_CONSTANTS | ZEND_CFG_SPLIT_AT_LIVE_RANGES | ZEND_CFG_NO_ENTRY_PREDECESSORS | ZEND_SSA_RC_INFERENCE_FLAG, cfg, flags) != SUCCESS) { return FAILURE; } From 5bee30a1f1dcc3b408dcd0a70e533f8dda4f551a Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Fri, 9 Dec 2016 18:31:44 +0800 Subject: [PATCH 360/569] xor is prefered --- ext/opcache/jit/zend_jit_x86.dasc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 1e0481b8c9436..d6b42cb313a13 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -1409,7 +1409,7 @@ static int zend_jit_throw_cannot_pass_by_ref_stub(dasm_State **Dst) | SET_Z_TYPE_INFO RX+r1, IS_UNDEF | mov RX, r0 |.if X64 - | mov CARG1, 0 + | xor CARG1, CARG1 | LOAD_ADDR CARG2, "Cannot pass parameter %d by reference" | mov CARG3d, dword OP:r0->op2.num | EXT_CALL zend_throw_error, r0 @@ -1609,7 +1609,7 @@ static int zend_jit_not_obj_stub(dasm_State **Dst) |->not_obj: |8: |.if X64 - | mov CARG1, 0 + | xor CARG1, CARG1 | LOAD_ADDR CARG2, "Using $this when not in object context" | EXT_CALL zend_throw_error, r0 |.else @@ -6455,7 +6455,7 @@ static int zend_jit_defined(dasm_State **Dst, const zend_op *opline, int b, int |.cold_code |1: | LOAD_ADDR FCARG1a, zv - | mov FCARG2d, 0 + | xor FCARG2d, FCARG2d | EXT_CALL zend_quick_get_constant, r0 | test r0, r0 if (smart_branch) { From 715aa6e7a39dd1ebc98b30f03076b17e6eea2929 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Fri, 9 Dec 2016 18:42:22 +0800 Subject: [PATCH 361/569] ADD/SUB is prefered MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit "For historical reasons, the INC and DEC instructions leave the carry flag unchanged, while the other arithmetic flags are written to. This causes a false dependence on the previous value of the flags and costs an extra μop. To avoid these problems, it is recommended that you always use ADD and SUB instead of INC and DEC. For example, INC EAX should be replaced by ADD EAX,1." http://www.agner.org/optimize/microarchitecture.pdf --- ext/opcache/jit/zend_jit_x86.dasc | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index d6b42cb313a13..e1ae0c6f5ae0c 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -784,11 +784,11 @@ static void* dasm_labels[zend_lb_MAX]; |.endmacro |.macro GC_ADDREF, zv -| inc dword [zv] +| add dword [zv], 1 |.endmacro |.macro GC_DELREF, zv -| dec dword [zv] +| sub dword [zv], 1 |.endmacro |.macro IF_GC_MAY_NOT_LEAK, ptr, tmp_reg, label @@ -802,12 +802,12 @@ static void* dasm_labels[zend_lb_MAX]; |.if X64 || if (!IS_SIGNED_32BIT(Z_LVAL_P(zv))) { | mov64 tmp_reg, ((uintptr_t)Z_LVAL_P(zv)) -| inc dword [tmp_reg] +| add dword [tmp_reg], 1 || } else { -| inc dword [Z_LVAL_P(zv)] +| add dword [Z_LVAL_P(zv)], 1 || } |.else -| inc dword [Z_LVAL_P(zv)] +| add dword [Z_LVAL_P(zv)], 1 |.endif |.endmacro @@ -820,7 +820,7 @@ static void* dasm_labels[zend_lb_MAX]; | add dword [Z_LVAL_P(zv)], 2 || } |.else -| inc dword [Z_LVAL_P(zv)] +| add dword [Z_LVAL_P(zv)], 1 |.endif |.endmacro @@ -1913,9 +1913,9 @@ static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, zend_op_arr | ZVAL_COPY_VALUE FP + opline->result.var, FP + opline->op1.var, MAY_BE_LONG, r0, eax, r1 || } if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { - | inc aword [FP + opline->op1.var] + | add aword [FP + opline->op1.var], 1 } else { - | dec aword [FP + opline->op1.var] + | sub aword [FP + opline->op1.var], 1 } op1_def_info = OP1_DEF_INFO(); @@ -5240,7 +5240,7 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, int b, | test byte [FP + opline->op1.var + 9], IS_TYPE_REFCOUNTED | jz >3 | mov FCARG1a, aword [FP + opline->op1.var] - | dec dword [FCARG1a] + | sub dword [FCARG1a], 1 | jnz >3 | mov aword [r4], r0 // save | ZVAL_DTOR_FUNC op1_info, op_array->filename, opline @@ -7334,7 +7334,7 @@ static int zend_jit_bind_global(dasm_State **Dst, const zend_op *opline, zend_op //idx = (uint32_t)(uintptr_t)CACHED_PTR(Z_CACHE_SLOT_P(varname)) - 1; | mov r0, EX->run_time_cache | mov r0, aword [r0 + Z_CACHE_SLOT_P(varname)] - | dec r0 + | sub r0, 1 //if (EXPECTED(idx < EG(symbol_table).nNumUsed)) | cmp r0, [&EG(symbol_table).nNumUsed] | jae >9 From 4e73d293bb7d32b37a7f542cd5965dca32717a4d Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 9 Dec 2016 18:18:26 +0300 Subject: [PATCH 362/569] Fixed crash because of inconsistent "opline" value --- ext/opcache/jit/zend_jit_x86.dasc | 1 + 1 file changed, 1 insertion(+) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index e1ae0c6f5ae0c..2b1c4f3b483ee 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -6018,6 +6018,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar if (opline->opcode != ZEND_DO_ICALL) { uint32_t target_label = ssa->cfg.map[opline - op_array->opcodes] + 1; + | LOAD_ADDR IP, (opline + 1) | jmp =>target_label } } From 435e966bbb582c3b61f20ec1476cd939aa1efa78 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Sat, 10 Dec 2016 10:17:43 +0800 Subject: [PATCH 363/569] Fixed stack alignmenet --- ext/opcache/jit/zend_jit_x86.dasc | 2 -- 1 file changed, 2 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 2b1c4f3b483ee..28955cd4ff92e 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -6257,7 +6257,6 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, zend_op_ar | mov CARG1, E_NOTICE | LOAD_ADDR CARG2, "Only variables should be passed by reference" | EXT_CALL zend_error, r0 - | add r4, 8 |.else | sub r4, 8 | push "Only variables should be passed by reference" @@ -6305,7 +6304,6 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, zend_op_ar | mov CARG1, E_NOTICE | LOAD_ADDR CARG2, "Only variables should be passed by reference" | EXT_CALL zend_error, r0 - | add r4, 8 |.else | sub r4, 8 | push "Only variables should be passed by reference" From 31a40aeb589f4425afdadb11f79f1a832d70b95c Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 12 Dec 2016 15:12:55 +0300 Subject: [PATCH 364/569] Optimized for the most probable case --- ext/opcache/jit/zend_jit_x86.dasc | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 28955cd4ff92e..4fde20e878bb9 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -2936,7 +2936,15 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o switch (type) { case BP_JIT_IS: if (opline->op2_type != IS_CONST) { + | cmp byte [FCARG2a + offsetof(zend_string, val)], '9' + | jle >1 + |.cold_code + |1: | EXT_CALL zend_jit_symtable_find, r0 + | jmp >1 + |.code + | EXT_CALL zend_hash_find, r0 + |1: } else { | EXT_CALL zend_hash_find, r0 } @@ -2951,7 +2959,15 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o case BP_VAR_IS: case BP_VAR_UNSET: if (opline->op2_type != IS_CONST) { + | cmp byte [FCARG2a + offsetof(zend_string, val)], '9' + | jle >1 + |.cold_code + |1: | EXT_CALL zend_jit_symtable_find, r0 + | jmp >1 + |.code + | EXT_CALL zend_hash_find, r0 + |1: } else { | EXT_CALL zend_hash_find, r0 } From c1b60d8d575b715a6d1acefdf47617e17fdbee47 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 1 Feb 2017 11:49:03 +0300 Subject: [PATCH 365/569] Added support for profiling jitted assembler code by Linux perf --- ext/opcache/jit/zend_jit.c | 30 ++++- ext/opcache/jit/zend_jit.h | 1 + ext/opcache/jit/zend_jit_perf_dump.c | 170 ++++++++++++++++++++++++++- 3 files changed, 194 insertions(+), 7 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 144f474e4cbf7..10d9657d542fc 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -222,7 +222,7 @@ static void *dasm_link_and_encode(dasm_State **dasm_state, #if defined(HAVE_DISASM) || defined(HAVE_GDB) || defined(HAVE_OPROFILE) || defined(HAVE_PERFTOOLS) || defined(HAVE_VTUNE) if (!name) { - if (ZCG(accel_directives).jit_debug & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_OPROFILE|ZEND_JIT_DEBUG_PERF|ZEND_JIT_DEBUG_VTUNE)) { + if (ZCG(accel_directives).jit_debug & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_OPROFILE|ZEND_JIT_DEBUG_PERF|ZEND_JIT_DEBUG_VTUNE|ZEND_JIT_DEBUG_PERF_DUMP)) { str = zend_jit_func_name(op_array); if (str) { name = ZSTR_VAL(str); @@ -266,12 +266,18 @@ static void *dasm_link_and_encode(dasm_State **dasm_state, #endif #ifdef HAVE_PERFTOOLS - if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_PERF) { + if (ZCG(accel_directives).jit_debug & (ZEND_JIT_DEBUG_PERF|ZEND_JIT_DEBUG_PERF_DUMP)) { if (name) { - zend_jit_perf_dump( + zend_jit_perf_map_register( name, entry, size); + if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_PERF_DUMP) { + zend_jit_perf_jitdump_register( + name, + entry, + size); + } } } #endif @@ -319,7 +325,7 @@ static void *jit_alloc(size_t size, int shared) # endif # ifdef HAVE_MPROTECT - if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_GDB) { + if (ZCG(accel_directives).jit_debug & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP)) { prot = PROT_EXEC | PROT_READ | PROT_WRITE; } else { prot = PROT_NONE; @@ -1815,7 +1821,7 @@ ZEND_API int zend_jit_script(zend_script *script) ZEND_API void zend_jit_unprotect(void) { #ifdef HAVE_MPROTECT - if (!(ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_GDB)) { + if (!(ZCG(accel_directives).jit_debug & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP))) { if (mprotect(dasm_buf, ((char*)dasm_end) - ((char*)dasm_buf), PROT_READ | PROT_WRITE) != 0) { fprintf(stderr, "mprotect() failed [%d] %s\n", errno, strerror(errno)); } @@ -1826,7 +1832,7 @@ ZEND_API void zend_jit_unprotect(void) ZEND_API void zend_jit_protect(void) { #ifdef HAVE_MPROTECT - if (!(ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_GDB)) { + if (!(ZCG(accel_directives).jit_debug & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP))) { if (mprotect(dasm_buf, ((char*)dasm_end) - ((char*)dasm_buf), PROT_READ | PROT_EXEC) != 0) { fprintf(stderr, "mprotect() failed [%d] %s\n", errno, strerror(errno)); } @@ -1913,6 +1919,12 @@ ZEND_API int zend_jit_startup(zend_long jit, size_t size) } #endif +#ifdef HAVE_PERFTOOLS + if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_PERF_DUMP) { + zend_jit_perf_jitdump_open(); + } +#endif + zend_jit_unprotect(); ret = zend_jit_make_stubs(); zend_jit_protect(); @@ -1945,6 +1957,12 @@ ZEND_API void zend_jit_shutdown(void) } #endif +#ifdef HAVE_PERFTOOLS + if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_PERF_DUMP) { + zend_jit_perf_jitdump_close(); + } +#endif + if (dasm_buf) { jit_free(dasm_buf, ((char*)dasm_end) - ((char*)dasm_buf)); } diff --git a/ext/opcache/jit/zend_jit.h b/ext/opcache/jit/zend_jit.h index 5568dc880f287..2108908aad5c1 100644 --- a/ext/opcache/jit/zend_jit.h +++ b/ext/opcache/jit/zend_jit.h @@ -60,6 +60,7 @@ #define ZEND_JIT_DEBUG_PERF (1<<5) #define ZEND_JIT_DEBUG_OPROFILE (1<<6) #define ZEND_JIT_DEBUG_VTUNE (1<<7) +#define ZEND_JIT_DEBUG_PERF_DUMP (1<<8) ZEND_API int zend_jit_op_array(zend_op_array *op_array, zend_script *script); ZEND_API int zend_jit_script(zend_script *script); diff --git a/ext/opcache/jit/zend_jit_perf_dump.c b/ext/opcache/jit/zend_jit_perf_dump.c index 5feb93dc662d1..097a1bee67faa 100644 --- a/ext/opcache/jit/zend_jit_perf_dump.c +++ b/ext/opcache/jit/zend_jit_perf_dump.c @@ -20,8 +20,176 @@ #include #include +#include +#include +#include -static void zend_jit_perf_dump(const char *name, void *start, size_t size) +#include "zend_elf.h" + +/* + * 1) Profile using perf-.map + * + * perf record php -d opcache.huge_code_pages=0 -d opcache.jit_debug=0x20 bench.php + * perf report + * + * 2) Profile using jit-.dump + * + * perf record php -d opcache.huge_code_pages=0 -d opcache.jit_debug=0x100 bench.php + * perf inject -j -i perf.data -o perf.data.jitted + * perf report -i perf.data.jitted + * + */ + + +#define ZEND_PERF_JITDUMP_HEADER_MAGIC 0x4A695444 +#define ZEND_PERF_JITDUMP_HEADER_VERSION 1 + +#define ZEND_PERF_JITDUMP_RECORD_LOAD 0 +#define ZEND_PERF_JITDUMP_RECORD_MOVE 1 +#define ZEND_PERF_JITDUMP_RECORD_DEBUG_INFO 2 +#define ZEND_PERF_JITDUMP_RECORD_CLOSE 3 +#define ZEND_PERF_JITDUMP_UNWINDING_UNFO 4 + +#define ALIGN8(size) (((size) + 7) & ~7) +#define PADDING8(size) (ALIGN8(size) - (size)) + +typedef struct zend_perf_jitdump_header { + uint32_t magic; + uint32_t version; + uint32_t size; + uint32_t elf_mach_target; + uint32_t reserved; + uint32_t process_id; + uint64_t time_stamp; + uint64_t flags; +} zend_perf_jitdump_header; + +typedef struct _zend_perf_jitdump_record { + uint32_t event; + uint32_t size; + uint64_t time_stamp; +} zend_perf_jitdump_record; + +typedef struct _zend_perf_jitdump_load_record { + zend_perf_jitdump_record hdr; + uint32_t process_id; + uint32_t thread_id; + uint64_t vma; + uint64_t code_address; + uint64_t code_size; + uint64_t code_id; +} zend_perf_jitdump_load_record; + +static int jitdump_fd = -1; +static void *jitdump_mem = MAP_FAILED; + +static uint64_t zend_perf_timestamp(void) +{ + struct timespec ts; + + if (clock_gettime(CLOCK_MONOTONIC, &ts) != 0) { + return 0; + } + return ((uint64_t)ts.tv_sec * 1000000000) + ts.tv_nsec; +} + +static void zend_jit_perf_jitdump_open(void) +{ + char filename[64]; + int fd, ret; + zend_elf_header elf_hdr; + zend_perf_jitdump_header jit_hdr; + + sprintf(filename, "/tmp/jit-%d.dump", getpid()); + if (!zend_perf_timestamp()) { + return; + } + + fd = open("/proc/self/exe", O_RDONLY); + if (fd < 0) { + return; + } + + ret = read(fd, &elf_hdr, sizeof(elf_hdr)); + close(fd); + + if (ret != sizeof(elf_hdr) || + elf_hdr.emagic[0] != 0x7f || + elf_hdr.emagic[1] != 'E' || + elf_hdr.emagic[2] != 'L' || + elf_hdr.emagic[3] != 'F') { + return; + } + + jitdump_fd = open(filename, O_CREAT | O_TRUNC | O_RDWR, 0666); + if (jitdump_fd < 0) { + return; + } + + jitdump_mem = mmap(NULL, + sysconf(_SC_PAGESIZE), + PROT_READ|PROT_EXEC, + MAP_PRIVATE, jitdump_fd, 0); + + if (jitdump_mem == MAP_FAILED) { + close(jitdump_fd); + jitdump_fd = -1; + return; + } + + memset(&jit_hdr, 0, sizeof(jit_hdr)); + jit_hdr.magic = ZEND_PERF_JITDUMP_HEADER_MAGIC; + jit_hdr.version = ZEND_PERF_JITDUMP_HEADER_VERSION; + jit_hdr.size = sizeof(jit_hdr); + jit_hdr.elf_mach_target = elf_hdr.machine; + jit_hdr.process_id = getpid(); + jit_hdr.time_stamp = zend_perf_timestamp(); + jit_hdr.flags = 0; + write(jitdump_fd, &jit_hdr, sizeof(jit_hdr)); +} + +static void zend_jit_perf_jitdump_close(void) +{ + if (jitdump_fd >= 0) { + zend_perf_jitdump_record rec; + + rec.event = ZEND_PERF_JITDUMP_RECORD_CLOSE; + rec.size = sizeof(rec); + rec.time_stamp = zend_perf_timestamp(); + write(jitdump_fd, &rec, sizeof(rec)); + close(jitdump_fd); + + if (jitdump_mem != MAP_FAILED) { + munmap(jitdump_mem, sysconf(_SC_PAGESIZE)); + } + } +} + +static void zend_jit_perf_jitdump_register(const char *name, void *start, size_t size) +{ + if (jitdump_fd >= 0) { + static uint64_t id = 1; + zend_perf_jitdump_load_record rec; + size_t len = strlen(name); + + memset(&rec, 0, sizeof(rec)); + rec.hdr.event = ZEND_PERF_JITDUMP_RECORD_LOAD; + rec.hdr.size = sizeof(rec) + len + 1 + size; + rec.hdr.time_stamp = zend_perf_timestamp(); + rec.process_id = getpid(); + rec.thread_id = syscall(SYS_gettid); + rec.vma = (uint64_t)(uintptr_t)start; + rec.code_address = (uint64_t)(uintptr_t)start; + rec.code_size = (uint64_t)size; + rec.code_id = id++; + + write(jitdump_fd, &rec, sizeof(rec)); + write(jitdump_fd, name, len + 1); + write(jitdump_fd, start, size); + } +} + +static void zend_jit_perf_map_register(const char *name, void *start, size_t size) { static FILE *fp = NULL; From 97aa7c95605b961cd4357c2beee9d88abafe1f3b Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 1 Feb 2017 17:17:49 +0300 Subject: [PATCH 366/569] Fixed compilation warning --- ext/opcache/jit/zend_jit_x86.dasc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 468b149c40525..b0789149f5886 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -1837,7 +1837,7 @@ static int zend_jit_context_threaded_call(dasm_State **Dst, const zend_op *oplin | call aword [IP] zend_jit_check_exception(Dst); } else { - zend_op *next_opline = opline + 1; + const zend_op *next_opline = opline + 1; | cmp IPl, next_opline | je >1 From 2c21d6b9664f0b3e6370e5c9ab62e8726eee1025 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 6 Feb 2017 17:17:51 +0300 Subject: [PATCH 367/569] Improved support for context-threading (it's still incomplete) --- ext/opcache/jit/zend_jit_x86.dasc | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index b0789149f5886..51f2ec40c8e7e 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -5830,12 +5830,22 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar /* recursive call */ #ifdef CONTEXT_THREADED_JIT | call =>(call_info->num_args+ssa->cfg.blocks_count) + | cmp aword [&EG(exception)], 0 + | jne ->exception_handler + if (!func) { + | jmp >9 + } #else | jmp =>call_info->num_args #endif } else { #ifdef CONTEXT_THREADED_JIT | call aword [IP] + | cmp aword [&EG(exception)], 0 + | jne ->exception_handler + if (!func) { + | jmp >9 + } #else | add r4, SPAD // stack alignment | jmp aword [IP] From 06da76dbe34d6aceee2392adc74d8575775c5c08 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 17 Feb 2017 13:50:14 +0300 Subject: [PATCH 368/569] Prepare code generator for SSE registers allocation --- ext/opcache/jit/zend_jit_x86.dasc | 154 ++++++++++++++++-------------- 1 file changed, 80 insertions(+), 74 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 51f2ec40c8e7e..476fe37b6e67f 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -263,12 +263,12 @@ static void* dasm_labels[zend_lb_MAX]; ||if (op_type == IS_CONST) { | .if X64 | mov r0, aword EX->literals -| sse_ins reg, qword [r0 + op.constant] +| sse_ins xmm(reg), qword [r0 + op.constant] | .else -| sse_ins reg, qword [op.zv] +| sse_ins xmm(reg), qword [op.zv] | .endif ||} else { -| sse_ins reg, qword [FP + op.var] +| sse_ins xmm(reg), qword [FP + op.var] ||} |.endmacro @@ -276,17 +276,17 @@ static void* dasm_labels[zend_lb_MAX]; ||if (op_type == IS_CONST) { | .if X64 | mov r0, aword EX->literals -| cvtsi2sd reg, qword [r0 + op.constant] +| cvtsi2sd xmm(reg), qword [r0 + op.constant] | .else -| cvtsi2sd reg, dword [op.zv] +| cvtsi2sd xmm(reg), dword [op.zv] | .endif ||} else { -| cvtsi2sd reg, aword [FP + op.var] +| cvtsi2sd xmm(reg), aword [FP + op.var] ||} |.endmacro |.macro SSE_GET_Z_LVAL, reg, zv -| cvtsi2sd reg, aword [zv] +| cvtsi2sd xmm(reg), aword [zv] |.endmacro |.macro SSE_LOAD, reg, op_type, op @@ -294,61 +294,45 @@ static void* dasm_labels[zend_lb_MAX]; |.endmacro |.macro SSE_GET_Z_DVAL, reg, zv -| movsd reg, qword [zv] +| movsd xmm(reg), qword [zv] |.endmacro -|.macro SSE_MATH, opcode, reg, op_type, op +|.macro SSE_MATH, opcode, reg, op ||switch (opcode) { || case ZEND_ADD: || case ZEND_ASSIGN_ADD: -| SSE_OP addsd, reg, op_type, op +| addsd xmm(reg), op || break; || case ZEND_SUB: || case ZEND_ASSIGN_SUB: -| SSE_OP subsd, reg, op_type, op +| subsd xmm(reg), op || break; || case ZEND_MUL: || case ZEND_ASSIGN_MUL: -| SSE_OP mulsd, reg, op_type, op +| mulsd xmm(reg), op || break; || case ZEND_DIV: || case ZEND_ASSIGN_DIV: -| SSE_OP divsd, reg, op_type, op +| divsd xmm(reg), op || break; ||} |.endmacro -|.macro SSE_MATH2, opcode, dst_reg, src_reg -||switch (opcode) { -|| case ZEND_ADD: -|| case ZEND_ASSIGN_ADD: -| addsd dst_reg, src_reg -|| break; -|| case ZEND_SUB: -|| case ZEND_ASSIGN_SUB: -| subsd dst_reg, src_reg -|| break; -|| case ZEND_MUL: -|| case ZEND_ASSIGN_MUL: -| mulsd dst_reg, src_reg -|| break; -|| case ZEND_DIV: -|| case ZEND_ASSIGN_DIV: -| divsd dst_reg, src_reg -|| break; -||} +|.macro SSE_MATH_CONST, opcode, reg, op +| .if X64 +| mov r0, aword EX->literals +| SSE_MATH, opcode, reg, qword [r0 + op.constant] +| .else +| SSE_MATH, opcode, reg, qword [op.zv] +| .endif |.endmacro -|.macro SSE_STORE, zv, reg -| movsd qword [zv], reg +|.macro SSE_MATH_REG, opcode, dst_reg, src_reg +| SSE_MATH, opcode, dst_reg, xmm(src_reg) |.endmacro -|.macro DOUBLE_STORE, zv, reg -|.if X64 or SSE -| SSE_STORE zv, reg -|.else -| FPU_STORE zv -|.endif +|.macro SSE_STORE, zv, reg +| movsd qword [zv], xmm(reg) |.endmacro |.macro LONG_OP, long_ins, reg, op_type, op @@ -2085,17 +2069,21 @@ static int zend_jit_math_long_long(dasm_State **Dst, |.cold_code |1: |.if X64 or SSE + int tmp_reg1 = 0; + int tmp_reg2 = 1; + if (op1_type == IS_CONST) { - | SSE_LOAD_LONG xmm0, op1_type, op1 + | SSE_LOAD_LONG tmp_reg1, op1_type, op1 } else { - | SSE_GET_Z_LVAL xmm0, Ra(op1_reg)+op1_offset + | SSE_GET_Z_LVAL tmp_reg1, Ra(op1_reg)+op1_offset } if (op2_type == IS_CONST) { - | SSE_LOAD_LONG xmm1, op2_type, op2 + | SSE_LOAD_LONG tmp_reg2, op2_type, op2 } else { - | SSE_GET_Z_LVAL xmm1, Ra(op2_reg)+op2_offset + | SSE_GET_Z_LVAL tmp_reg2, Ra(op2_reg)+op2_offset } - | SSE_MATH2 opline->opcode, xmm0, xmm1 + | SSE_MATH_REG opline->opcode, tmp_reg1, tmp_reg2 + | SSE_STORE Ra(res_reg)+res_offset, tmp_reg1 |.else if (op2_type == IS_CONST) { | FPU_LONG_LOAD op2_type, op2 @@ -2108,8 +2096,8 @@ static int zend_jit_math_long_long(dasm_State **Dst, | FPU_GET_Z_LVAL Ra(op1_reg)+op1_offset } | FPU_MATH2 opline->opcode, st1 + | FPU_STORE Ra(res_reg)+res_offset |.endif - | DOUBLE_STORE Ra(res_reg)+res_offset, xmm0 | SET_Z_TYPE_INFO Ra(res_reg)+res_offset, IS_DOUBLE | jmp >2 |.code @@ -2142,16 +2130,19 @@ static int zend_jit_math_long_double(dasm_State **Dst, uint32_t res_offset) { |.if X64 or SSE + int result_reg = 0; + if (op1_type == IS_CONST) { - | SSE_LOAD_LONG xmm0, op1_type, op1 + | SSE_LOAD_LONG result_reg, op1_type, op1 } else { - | SSE_GET_Z_LVAL xmm0, Ra(op1_reg)+op1_offset + | SSE_GET_Z_LVAL result_reg, Ra(op1_reg)+op1_offset } if (op2_type == IS_CONST) { - | SSE_MATH opline->opcode, xmm0, op2_type, op2 + | SSE_MATH_CONST opline->opcode, result_reg, op2 } else { - | SSE_MATH2 opline->opcode, xmm0, qword [Ra(op2_reg)+op2_offset] + | SSE_MATH opline->opcode, result_reg, qword [Ra(op2_reg)+op2_offset] } + | SSE_STORE Ra(res_reg)+res_offset, result_reg |.else if (op1_type == IS_CONST) { | FPU_LONG_LOAD op1_type, op1 @@ -2163,8 +2154,8 @@ static int zend_jit_math_long_double(dasm_State **Dst, } else { | FPU_MATH2 opline->opcode, qword [Ra(op2_reg)+op2_offset] } + | FPU_STORE Ra(res_reg)+res_offset |.endif - | DOUBLE_STORE Ra(res_reg)+res_offset, xmm0 | SET_Z_TYPE_INFO Ra(res_reg)+res_offset, IS_DOUBLE return 1; @@ -2184,30 +2175,35 @@ static int zend_jit_math_double_long(dasm_State **Dst, uint32_t res_offset) { |.if X64 or SSE + || int result_reg = 0; + || || if (opline->opcode == ZEND_ADD || opline->opcode == ZEND_MUL || opline->opcode == ZEND_ASSIGN_ADD || opline->opcode == ZEND_ASSIGN_MUL) { || if (op2_type == IS_CONST) { - | SSE_LOAD_LONG xmm0, op2_type, op2 + | SSE_LOAD_LONG result_reg, op2_type, op2 || } else { - | SSE_GET_Z_LVAL xmm0, Ra(op2_reg)+op2_offset + | SSE_GET_Z_LVAL result_reg, Ra(op2_reg)+op2_offset || } || if (op1_type == IS_CONST) { - | SSE_MATH opline->opcode, xmm0, op1_type, op1 + | SSE_MATH_CONST opline->opcode, result_reg, op1 || } else { - | SSE_MATH2 opline->opcode, xmm0, qword [Ra(op1_reg)+op1_offset] + | SSE_MATH opline->opcode, result_reg, qword [Ra(op1_reg)+op1_offset] || } || } else { + || int tmp_reg = 1; + || || if (op1_type == IS_CONST) { - | SSE_LOAD xmm0, op1_type, op1 + | SSE_LOAD result_reg, op1_type, op1 || } else { - | SSE_GET_Z_DVAL xmm0, Ra(op1_reg)+op1_offset + | SSE_GET_Z_DVAL result_reg, Ra(op1_reg)+op1_offset || } || if (op2_type == IS_CONST) { - | SSE_LOAD_LONG xmm1, op2_type, op2 + | SSE_LOAD_LONG tmp_reg, op2_type, op2 || } else { - | SSE_GET_Z_LVAL xmm1, Ra(op2_reg)+op2_offset + | SSE_GET_Z_LVAL tmp_reg, Ra(op2_reg)+op2_offset || } - | SSE_MATH2 opline->opcode, xmm0, xmm1 + | SSE_MATH_REG opline->opcode, result_reg, tmp_reg || } + | SSE_STORE Ra(res_reg)+res_offset, result_reg |.else || if (op2_type == IS_CONST) { | FPU_LONG_LOAD op2_type, op2 @@ -2228,8 +2224,8 @@ static int zend_jit_math_double_long(dasm_State **Dst, || } | FPU_MATH2 opline->opcode, st1 || } + | FPU_STORE Ra(res_reg)+res_offset |.endif - | DOUBLE_STORE Ra(res_reg)+res_offset, xmm0 if (op1_type == IS_CONST || op1_reg != res_reg || op1_offset != res_offset) { | SET_Z_TYPE_INFO Ra(res_reg)+res_offset, IS_DOUBLE } @@ -2252,18 +2248,21 @@ static int zend_jit_math_double_double(dasm_State **Dst, zend_bool same_ops = (op1_type == op2_type) && (op1.var == op2.var); |.if X64 or SSE + int result_reg = 0; + if (op1_type == IS_CONST) { - | SSE_LOAD xmm0, op1_type, op1 + | SSE_LOAD result_reg, op1_type, op1 } else { - | SSE_GET_Z_DVAL xmm0, Ra(op1_reg)+op1_offset + | SSE_GET_Z_DVAL result_reg, Ra(op1_reg)+op1_offset } if (same_ops) { - | SSE_MATH2 opline->opcode, xmm0, xmm0 + | SSE_MATH_REG opline->opcode, result_reg, result_reg } else if (op2_type == IS_CONST) { - | SSE_MATH opline->opcode, xmm0, op2_type, op2 + | SSE_MATH_CONST opline->opcode, result_reg, op2 } else { - | SSE_MATH2 opline->opcode, xmm0, qword [Ra(op2_reg)+op2_offset] + | SSE_MATH opline->opcode, result_reg, qword [Ra(op2_reg)+op2_offset] } + | SSE_STORE Ra(res_reg)+res_offset, result_reg |.else if (op1_type == IS_CONST) { | FPU_LOAD op1_type, op1 @@ -2275,8 +2274,8 @@ static int zend_jit_math_double_double(dasm_State **Dst, } else { | FPU_MATH2 opline->opcode, qword [Ra(op2_reg)+op2_offset] } + | FPU_STORE Ra(res_reg)+res_offset |.endif - | DOUBLE_STORE Ra(res_reg)+res_offset, xmm0 if (op1_type == IS_CONST || op1_reg != res_reg || op1_offset != res_offset) { | SET_Z_TYPE_INFO Ra(res_reg)+res_offset, IS_DOUBLE } @@ -4311,8 +4310,10 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, i static int zend_jit_cmp_long_double(dasm_State **Dst, const zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa) { |.if X64 or SSE - | SSE_LOAD_LONG xmm0, opline->op1_type, opline->op1 - | SSE_OP ucomisd, xmm0, opline->op2_type, opline->op2 + || int tmp_reg = 1; + || + | SSE_LOAD_LONG tmp_reg, opline->op1_type, opline->op1 + | SSE_OP ucomisd, tmp_reg, opline->op2_type, opline->op2 |.else | FPU_LOAD opline->op2_type, opline->op2 | FPU_LONG_LOAD opline->op1_type, opline->op1 @@ -4326,9 +4327,12 @@ static int zend_jit_cmp_long_double(dasm_State **Dst, const zend_op *opline, int static int zend_jit_cmp_double_long(dasm_State **Dst, const zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa) { |.if X64 or SSE - | SSE_LOAD xmm0, opline->op1_type, opline->op1 - | SSE_LOAD_LONG xmm1, opline->op2_type, opline->op2 - | ucomisd xmm0, xmm1 + int tmp_reg1 = 0; + int tmp_reg2 = 1; + + | SSE_LOAD tmp_reg1, opline->op1_type, opline->op1 + | SSE_LOAD_LONG tmp_reg2, opline->op2_type, opline->op2 + | ucomisd xmm(tmp_reg1), xmm(tmp_reg2) |.else | FPU_LONG_LOAD opline->op2_type, opline->op2 | FPU_LOAD opline->op1_type, opline->op1 @@ -4342,8 +4346,10 @@ static int zend_jit_cmp_double_long(dasm_State **Dst, const zend_op *opline, int static int zend_jit_cmp_double_double(dasm_State **Dst, const zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa) { |.if X64 or SSE - | SSE_LOAD xmm0, opline->op1_type, opline->op1 - | SSE_OP ucomisd, xmm0, opline->op2_type, opline->op2 + int tmp_reg = 0; + + | SSE_LOAD tmp_reg, opline->op1_type, opline->op1 + | SSE_OP ucomisd, tmp_reg, opline->op2_type, opline->op2 |.else | FPU_LOAD opline->op2_type, opline->op2 | FPU_LOAD opline->op1_type, opline->op1 From 14a7b51999c2665005fb97b9ee6f8da609359cf0 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 28 Feb 2017 17:12:07 +0300 Subject: [PATCH 369/569] Prototype of linear-scan register allocator (incomplete) --- ext/opcache/jit/zend_jit.c | 623 ++++++++++++++++++++++++++++++ ext/opcache/jit/zend_jit_x86.dasc | 140 +++---- ext/opcache/jit/zend_jit_x86.h | 154 ++++++++ 3 files changed, 840 insertions(+), 77 deletions(-) create mode 100644 ext/opcache/jit/zend_jit_x86.h diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 10d9657d542fc..ad9b1f5e91063 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -34,6 +34,8 @@ #include "Optimizer/zend_call_graph.h" #include "Optimizer/zend_dump.h" +//#define REG_ALLOC +#define DEBUG_REG_ALLOC //#define CONTEXT_THREADED_JIT #define PREFER_MAP_32BIT //#define ZEND_JIT_RECORD @@ -81,6 +83,7 @@ static int zend_may_throw(const zend_op *opline, zend_op_array *op_array, zend_s static int zend_may_overflow(const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa); #include "dynasm/dasm_x86.h" +#include "jit/zend_jit_x86.h" #include "jit/zend_jit_helpers.c" #include "jit/zend_jit_x86.c" #include "jit/zend_jit_disasm_x86.c" @@ -956,6 +959,615 @@ static int zend_jit_op_array_analyze2(zend_op_array *op_array, zend_script *scri return SUCCESS; } +#ifdef REG_ALLOC +typedef struct _zend_lifetime_interval zend_lifetime_interval; + +struct _zend_lifetime_interval { + int ssa_var; + int8_t reg; + zend_bool split; + //???zend_bool fixed; + uint32_t start; + uint32_t end; + //zend_regset regset; + zend_lifetime_interval *next; +}; + +static int zend_jit_add_range(zend_lifetime_interval **intervals, int var, uint32_t from, uint32_t to) +{ + zend_lifetime_interval *ival = intervals[var]; + + if (!ival || ival->start > to + 1 || ival->end < from - 1) { + // TODO: emalloc(); + ival = zend_arena_alloc(&CG(arena), sizeof(zend_lifetime_interval)); + if (!ival) { + return FAILURE; + } + ival->ssa_var = var; + ival->reg = ZREG_NONE; + ival->split = 0; + //???ival->fixed = 0; + ival->start = from; + ival->end = to; + ival->next = intervals[var]; + intervals[var] = ival; + } else { + if (from < ival->start) { + ival->start = from; + } + if (to > ival->end) { + ival->end = to; + } + } + return SUCCESS; +} + +static int zend_jit_begin_range(zend_lifetime_interval **intervals, int var, uint32_t from) +{ + if (!intervals[var] || intervals[var]->start > from) { + // dead store + return zend_jit_add_range(intervals, var, from, from); + } + + intervals[var]->start = from; + + return SUCCESS; +} + +static void zend_jit_free_intervals(zend_lifetime_interval **intervals, int count) +{ + int i; + + if (intervals) { + for (i = 0; i < count; i++) { + if (intervals[i]) { + zend_lifetime_interval *ival = intervals[i]; + + intervals[i] = NULL; + do { + zend_lifetime_interval *next = ival->next; + // TODO: efree(ival); + ival = next; + } while (ival); + } + } + } +} + +static void zend_jit_insert_interval(zend_lifetime_interval **list, zend_lifetime_interval *ival) +{ + while (1) { + if (*list == NULL) { + *list = ival; + ival->next = NULL; + return; + } else if (ival->start < (*list)->start) { + ival->next = *list; + *list = ival; + return; + } + list = &(*list)->next; + } +} + +static int zend_jit_split_interval(zend_lifetime_interval *current, uint32_t pos, zend_lifetime_interval **list) +{ + // TODO: emalloc(); + zend_lifetime_interval *ival = zend_arena_alloc(&CG(arena), sizeof(zend_lifetime_interval)); + + if (!ival) { + return FAILURE; + } + + ival->ssa_var = current->ssa_var; + ival->reg = ZREG_NONE; + ival->split = 1; + //???ival->fixed = 0; + ival->start = pos; + ival->end = current->end; + + current->end = pos; + + zend_jit_insert_interval(list, ival); + + return SUCCESS; +} + +static zend_lifetime_interval *zend_jit_sort_intervals(zend_lifetime_interval **intervals, int count) +{ + zend_lifetime_interval *list = NULL; + zend_lifetime_interval *last = NULL; + int i; + + for (i = 0; i < count; i++) { + zend_lifetime_interval *ival = intervals[i]; + + while (ival != NULL) { + zend_lifetime_interval *next = ival->next; + + /* Optimized version of zend_jit_insert_interval() */ + if (!last) { + list = last = ival; + ival->next = NULL; + } else if (ival->start >= last->start) { + last->next = ival; + last = ival; + ival->next = NULL; + } else { + zend_lifetime_interval **p = &list; + + while (1) { + /*if (*p == NULL) { + *p = last = ival; + ival->next = NULL; + break; + } else*/ + if (ival->start < (*p)->start) { + ival->next = *p; + *p = ival; + break; + } + p = &(*p)->next; + } + } + + ival = next; + } + } + + return list; +} + +/* See "Linear Scan Register Allocation on SSA Form", Christian Wimmer and + Michael Franz, CGO'10 (2010), Figure 4. */ +static int zend_jit_compute_liveness(zend_op_array *op_array, zend_ssa *ssa, zend_bitset candidates, zend_lifetime_interval **list) +{ + int set_size, i, j, k; + uint32_t n; + zend_bitset live, live_in, pi_vars; + uint32_t *loop_end; + zend_ssa_phi *phi; + zend_lifetime_interval **intervals; + + set_size = zend_bitset_len(ssa->vars_count); + intervals = zend_arena_calloc(&CG(arena), ssa->vars_count, sizeof(zend_lifetime_interval*)); + live_in = zend_arena_calloc(&CG(arena), set_size * ssa->cfg.blocks_count, ZEND_BITSET_ELM_SIZE); + live = zend_arena_alloc(&CG(arena), set_size * ZEND_BITSET_ELM_SIZE); + pi_vars = zend_arena_alloc(&CG(arena), set_size * ZEND_BITSET_ELM_SIZE); + loop_end = zend_arena_calloc(&CG(arena), ssa->cfg.blocks_count, sizeof(uint32_t)); + + if (!intervals || !live || !live_in || !pi_vars) { + goto failure; + } + + for (i = ssa->cfg.blocks_count - 1; i >= 0; i--) { + zend_basic_block *b = ssa->cfg.blocks + i; + + /* live = UNION of successor.liveIn for each successor of b */ + /* live.add(phi.inputOf(b)) for each phi of successors of b */ + zend_bitset_clear(live, set_size); + for (j = 0; j < 2 && b->successors[j] >= 0; j++) { + int succ = b->successors[j]; + + zend_bitset_union(live, live_in + set_size * succ, set_size); + zend_bitset_clear(pi_vars, set_size); + for (phi = ssa->blocks[succ].phis; phi; phi = phi->next) { + if (ssa->vars[phi->ssa_var].no_val) { + /* skip */ + } else if (phi->pi >= 0) { + if (phi->pi == i && phi->sources[0] >= 0) { + if (zend_bitset_in(candidates, phi->sources[0])) { + zend_bitset_incl(live, phi->sources[0]); + } + zend_bitset_incl(pi_vars, phi->var); + } + } else if (!zend_bitset_in(pi_vars, phi->var)) { + for (k = 0; k < ssa->cfg.blocks[succ].predecessors_count; k++) { + if (ssa->cfg.predecessors[ssa->cfg.blocks[succ].predecessor_offset + k] == i) { + if (phi->sources[k] >= 0 && zend_bitset_in(candidates, phi->sources[k])) { + zend_bitset_incl(live, phi->sources[k]); + } + break; + } + } + } + } + } + + /* addRange(var, b.from, b.to) for each var in live */ + ZEND_BITSET_FOREACH(live, set_size, j) { + if (!ssa->vars[j].no_val) { + if (zend_jit_add_range(intervals, j, b->start, b->start + b->len - 1) != SUCCESS) { + goto failure; + } + } + } ZEND_BITSET_FOREACH_END(); + + /* for each operation op of b in reverse order */ + for (n = b->start + b->len; n > b->start;) { + zend_ssa_op *op; + uint32_t num; + + n--; + op = ssa->ops + n; + + if (UNEXPECTED(op_array->opcodes[n].opcode == ZEND_OP_DATA)) { + num = n - 1; + } else { + num = n; + } + + /* for each output operand opd of op do */ + /* setFrom(opd, op) */ + /* live.remove(opd) */ + if (op->op1_def >= 0 && zend_bitset_in(candidates, op->op1_def)) { + if (zend_jit_begin_range(intervals, op->op1_def, num) != SUCCESS) { + goto failure; + } + zend_bitset_excl(live, op->op1_def); + } + if (op->op2_def >= 0 && zend_bitset_in(candidates, op->op2_def)) { + if (zend_jit_begin_range(intervals, op->op2_def, num) != SUCCESS) { + goto failure; + } + zend_bitset_excl(live, op->op2_def); + } + if (op->result_def >= 0 && zend_bitset_in(candidates, op->result_def)) { + if (zend_jit_begin_range(intervals, op->result_def, num) != SUCCESS) { + goto failure; + } + zend_bitset_excl(live, op->result_def); + } + + /* for each input operand opd of op do */ + /* live.add(opd) */ + /* addRange(opd, b.from, op) */ + if (op->op1_use >= 0 && zend_bitset_in(candidates, op->op1_use)) { + zend_bitset_incl(live, op->op1_use); + if (zend_jit_add_range(intervals, op->op1_use, b->start, num) != SUCCESS) { + goto failure; + } + } + if (op->op2_use >= 0 && zend_bitset_in(candidates, op->op2_use)) { + zend_bitset_incl(live, op->op2_use); + if (zend_jit_add_range(intervals, op->op2_use, b->start, num) != SUCCESS) { + goto failure; + } + } + if (op->result_use >= 0 && zend_bitset_in(candidates, op->result_use)) { + zend_bitset_incl(live, op->result_use); + if (zend_jit_add_range(intervals, op->result_use, b->start, num) != SUCCESS) { + goto failure; + } + } + } + + /* live.remove(phi.output) for each phi of b */ + for (phi = ssa->blocks[i].phis; phi; phi = phi->next) { + zend_bitset_excl(live, phi->ssa_var); + } + + if (b->loop_header >= 0 && !loop_end[b->loop_header]) { + loop_end[b->loop_header] = b->start + b->len; + } + + /* if b is loop header */ + if (b->flags & ZEND_BB_LOOP_HEADER) { + if (!loop_end[i]) { + loop_end[i] = b->start + b->len; + } + ZEND_BITSET_FOREACH(live, set_size, j) { + if (!ssa->vars[j].no_val) { + if (zend_jit_add_range(intervals, j, b->start, loop_end[i]) != SUCCESS) { + goto failure; + } + } + } ZEND_BITSET_FOREACH_END(); + } + + /* b.liveIn = live */ + zend_bitset_copy(live_in + set_size * i, live, set_size); + } + +#ifdef DEBUG_REG_ALLOC + fprintf(stderr, "Live Ranges\n"); + for (i = 0; i < ssa->vars_count; i++) { + if (intervals[i]) { + zend_lifetime_interval *ival = intervals[i]; + + fprintf(stderr, "#%d: ", ival->ssa_var); + fprintf(stderr, "%u-%u", ival->start, ival->end); + ival = ival->next; + while (ival) { + fprintf(stderr, ", %u-%u", ival->start, ival->end); + ival = ival->next; + } + fprintf(stderr, "\n"); + } + } +#endif + + *list = zend_jit_sort_intervals(intervals, ssa->vars_count); + return SUCCESS; + +failure: + zend_jit_free_intervals(intervals, ssa->vars_count); + *list = NULL; + return FAILURE; +} + +/* See "Optimized Interval Splitting in a Linear Scan Register Allocator", + Christian Wimmer VEE'05 (2005), Figure 4. Allocation without spilling. + and "Linear Scan Register Allocation on SSA Form", Christian Wimmer and + Michael Franz, CGO'10 (2010), Figure 6. */ +static int zend_jit_try_allocate_free_reg(zend_op_array *op_array, zend_ssa *ssa, zend_lifetime_interval *current, zend_lifetime_interval *active, zend_lifetime_interval *inactive, zend_lifetime_interval **list) +{ + zend_lifetime_interval *it; + uint32_t freeUntilPos[ZREG_NUM]; + uint32_t pos; + zend_reg i, reg; + zend_regset available; + + if ((ssa->var_info[current->ssa_var].type & MAY_BE_ANY) == MAY_BE_DOUBLE) { + available = ZEND_REGSET_FP; + } else { + available = ZEND_REGSET_GP; + } + + /* Set freeUntilPos of all physical registers to maxInt */ + for (i = 0; i < ZREG_NUM; i++) { + freeUntilPos[i] = 0xffffffff; + } + + /* for each interval it in active do */ + /* freeUntilPos[it.reg] = 0 */ + it = active; + while (it) { + freeUntilPos[it->reg] = 0; + it = it->next; + } + + if (current->split) { + /* for each interval it in inactive intersecting with current do */ + /* freeUntilPos[it.reg] = next intersection of it with current */ + it = inactive; + while (it) { + if (current->start < it->end && current->end > it->start) { + freeUntilPos[it->reg] = 0; + } + it = it->next; + } + } + +#if 0 + /* Coalesing */ + if (ssa->vars[current->ssa_var].definition == current->start) { + zend_op *opline = op_array->opcodes + current->start; + int hint = -1; + + switch (opline->opcode) { + case ZEND_ASSIGN: + hint = ssa->ops[current->start].op2_use; + case ZEND_QM_ASSIGN: + hint = ssa->ops[current->start].op1_use; + break; + case ZEND_ADD: + case ZEND_SUB: + case ZEND_MUL: + hint = ssa->ops[current->start].op1_use; + break; + case ZEND_ASSIGN_ADD: + case ZEND_ASSIGN_SUB: + case ZEND_ASSIGN_MUL: + if (opline->extended_value) { + hint = ssa->ops[current->start].op1_use; + } + break; + } + if (hint >= 0) { + } + } +#endif + + pos = 0; reg = ZREG_NONE; + for (i = 0; i < ZREG_NUM; i++) { + if (ZEND_REGSET_IN(available, i) && freeUntilPos[i] > pos) { + reg = i; + pos = freeUntilPos[i]; + } + } + + if (reg == ZREG_NONE) { + /* no register available without spilling */ + return 0; + } else if (current->end < pos) { + /* register available for the whole interval */ + current->reg = reg; + return 1; + } else { + /* register available for the first part of the interval */ + if (zend_jit_split_interval(current, pos, list) != SUCCESS) { + return 0; + } + current->reg = reg; + return 1; + } +} + +/* See "Optimized Interval Splitting in a Linear Scan Register Allocator", + Christian Wimmer VEE'05 (2005), Figure 5. Allocation with spilling. + and "Linear Scan Register Allocation on SSA Form", Christian Wimmer and + Michael Franz, CGO'10 (2010), Figure 6. */ +static int zend_jit_allocate_blocked_reg(void) +{ + /* TODO: ??? */ + return 0; +} + +/* See "Optimized Interval Splitting in a Linear Scan Register Allocator", + Christian Wimmer VEE'10 (2005), Figure 2. */ +static int zend_jit_linear_scan(zend_op_array *op_array, zend_ssa *ssa, zend_lifetime_interval *list) +{ + zend_lifetime_interval *unhandled, *active, *inactive, *handled; + zend_lifetime_interval *current, **p, *q; + uint32_t position; + +#ifdef DEBUG_REG_ALLOC + { + zend_lifetime_interval *ival = list; + + fprintf(stderr, "Before Linear Scan\n"); + while (ival != NULL) { + fprintf(stderr, "%u-%u: #%d\n", ival->start, ival->end, ival->ssa_var); + ival = ival->next; + } + } +#endif + + unhandled = list; + /* active = inactive = handled = {} */ + active = inactive = handled = NULL; + while (unhandled != NULL) { + current = unhandled; + unhandled = unhandled->next; + position = current->start; + + p = &active; + while (*p) { + q = *p; + if (q->end < position) { + /* move ival from active to handled */ + *p = q->next; + q->next = handled; + handled = q; + } else if (q->start > position) { + /* move ival from active to inactive */ + *p = q->next; + q->next = inactive; + inactive = q; + } else { + p = &q->next; + } + } + + p = &inactive; + while (*p) { + q = *p; + if (q->end < position) { + /* move ival from inactive to handled */ + *p = q->next; + q->next = handled; + handled = q; + } else if (q->start <= position) { + /* move ival from inactive to active */ + *p = q->next; + q->next = active; + active = q; + } else { + p = &q->next; + } + } + + if (zend_jit_try_allocate_free_reg(op_array, ssa, current, active, inactive, &unhandled) || + zend_jit_allocate_blocked_reg()) { + current->next = active; + active = current; + } + } + +#ifdef DEBUG_REG_ALLOC + { + zend_lifetime_interval *ival = handled; + + fprintf(stderr, "After Linear Scan\n"); + while (ival != NULL) { + if (ival->reg >= 0) { + fprintf(stderr, "%u-%u: #%d (%s)\n", ival->start, ival->end, ival->ssa_var, zend_reg_name[ival->reg]); + } else { + fprintf(stderr, "%u-%u: #%d (no-reg)\n", ival->start, ival->end, ival->ssa_var); + } + ival = ival->next; + } + + ival = inactive; + while (ival != NULL) { + if (ival->reg >= 0) { + fprintf(stderr, "%u-%u: #%d (%s)\n", ival->start, ival->end, ival->ssa_var, zend_reg_name[ival->reg]); + } else { + fprintf(stderr, "%u-%u: #%d (no-reg)\n", ival->start, ival->end, ival->ssa_var); + } + ival = ival->next; + } + + ival = active; + while (ival != NULL) { + if (ival->reg >= 0) { + fprintf(stderr, "%u-%u: #%d (%s)\n", ival->start, ival->end, ival->ssa_var, zend_reg_name[ival->reg]); + } else { + fprintf(stderr, "%u-%u: #%d (no-reg)\n", ival->start, ival->end, ival->ssa_var); + } + ival = ival->next; + } + } +#endif + + return SUCCESS; +} + +static int zend_jit_allocate_registers(zend_op_array *op_array, zend_ssa *ssa) +{ + void *checkpoint; + int set_size, candidates_count, i; + zend_bitset candidates; + zend_lifetime_interval *list; + + if (!ssa->var_info) { + return SUCCESS; + } + + checkpoint = zend_arena_checkpoint(CG(arena)); + + /* Identify SSA variables suitable for register allocation */ + set_size = zend_bitset_len(ssa->vars_count); + candidates = zend_arena_alloc(&CG(arena), set_size * ZEND_BITSET_ELM_SIZE); + if (!candidates) { + goto failure; + } + candidates_count = 0; + zend_bitset_clear(candidates, set_size); + for (i = 0; i < ssa->vars_count; i++) { + if (/*!ssa->vars[i].no_val + && */!(ssa->var_info[i].type & MAY_BE_REF) + && has_concrete_type(ssa->var_info[i].type) + && (ssa->var_info[i].type & MAY_BE_ANY) == MAY_BE_DOUBLE) { + zend_bitset_incl(candidates, i); + candidates_count++; + } + } + if (!candidates_count) { + goto success; + } + + /* Find life-time intervals */ + if (zend_jit_compute_liveness(op_array, ssa, candidates, &list) != SUCCESS) { + goto failure; + } + + /* Linear Scan Register Allocation */ + if (zend_jit_linear_scan(op_array, ssa, list) != SUCCESS) { + goto failure; + } + +success: + zend_arena_release(&CG(arena), checkpoint); + return SUCCESS; + +failure: + zend_arena_release(&CG(arena), checkpoint); + return FAILURE; +} +#endif + static int zend_jit(zend_op_array *op_array, zend_ssa *ssa, const zend_op *rt_opline) { int b, i, end; @@ -1481,6 +2093,12 @@ static int zend_real_jit_func(zend_op_array *op_array, zend_script *script, cons goto jit_failure; } +#ifdef REG_ALLOC + if (zend_jit_allocate_registers(op_array, &ssa) != SUCCESS) { + goto jit_failure; + } +#endif + if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_SSA) { zend_dump_op_array(op_array, ZEND_DUMP_HIDE_UNREACHABLE|ZEND_DUMP_RC_INFERENCE|ZEND_DUMP_SSA|ZEND_DUMP_RT_CONSTANTS, "JIT", &ssa); } @@ -1771,6 +2389,11 @@ ZEND_API int zend_jit_script(zend_script *script) if (zend_jit_op_array_analyze2(call_graph.op_arrays[i], script, &info->ssa, &info->flags) != SUCCESS) { goto jit_failure; } +#ifdef REG_ALLOC + if (zend_jit_allocate_registers(call_graph.op_arrays[i], &info->ssa) != SUCCESS) { + goto jit_failure; + } +#endif } } diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 476fe37b6e67f..0d59d592dea31 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -61,33 +61,19 @@ |.define SSE, 1 |.endif -#define r0 0 -#define r1 1 -#define r2 2 -#define r3 3 -#define r4 4 -#define r5 5 -#define r6 6 -#define r7 7 -#if SIZEOF_SIZE_T == 8 -# define r8 8 -# define r9 9 -# define r10 10 -# define r11 11 -# define r12 12 -# define r13 13 -# define r14 14 -# define r15 15 -# define FP r14 -# define IP r15 -# define RX IP -# define FCARG1a r7 +#include "jit/zend_jit_x86.h" + +const char* zend_reg_name[] = { +#ifdef __x86_64__ + "rax", "rcx", "rdx", "rbx", "rsp", "rbp", "rsi", "rdi", + "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7", + "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13", "xmm14", "xm15" #else -# define FP r6 -# define IP r7 -# define RX IP -# define FCARG1a r1 + "rax", "rcx", "rdx", "rbx", "rsp", "rbp", "rsi", "rdi", + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7" #endif +}; |.type EX, zend_execute_data, FP |.type OP, zend_op, IP @@ -263,12 +249,12 @@ static void* dasm_labels[zend_lb_MAX]; ||if (op_type == IS_CONST) { | .if X64 | mov r0, aword EX->literals -| sse_ins xmm(reg), qword [r0 + op.constant] +| sse_ins xmm(reg-ZREG_XMM0), qword [r0 + op.constant] | .else -| sse_ins xmm(reg), qword [op.zv] +| sse_ins xmm(reg-ZREG_XMM0), qword [op.zv] | .endif ||} else { -| sse_ins xmm(reg), qword [FP + op.var] +| sse_ins xmm(reg-ZREG_XMM0), qword [FP + op.var] ||} |.endmacro @@ -276,17 +262,17 @@ static void* dasm_labels[zend_lb_MAX]; ||if (op_type == IS_CONST) { | .if X64 | mov r0, aword EX->literals -| cvtsi2sd xmm(reg), qword [r0 + op.constant] +| cvtsi2sd xmm(reg-ZREG_XMM0), qword [r0 + op.constant] | .else -| cvtsi2sd xmm(reg), dword [op.zv] +| cvtsi2sd xmm(reg-ZREG_XMM0), dword [op.zv] | .endif ||} else { -| cvtsi2sd xmm(reg), aword [FP + op.var] +| cvtsi2sd xmm(reg-ZREG_XMM0), aword [FP + op.var] ||} |.endmacro |.macro SSE_GET_Z_LVAL, reg, zv -| cvtsi2sd xmm(reg), aword [zv] +| cvtsi2sd xmm(reg-ZREG_XMM0), aword [zv] |.endmacro |.macro SSE_LOAD, reg, op_type, op @@ -294,26 +280,26 @@ static void* dasm_labels[zend_lb_MAX]; |.endmacro |.macro SSE_GET_Z_DVAL, reg, zv -| movsd xmm(reg), qword [zv] +| movsd xmm(reg-ZREG_XMM0), qword [zv] |.endmacro |.macro SSE_MATH, opcode, reg, op ||switch (opcode) { || case ZEND_ADD: || case ZEND_ASSIGN_ADD: -| addsd xmm(reg), op +| addsd xmm(reg-ZREG_XMM0), op || break; || case ZEND_SUB: || case ZEND_ASSIGN_SUB: -| subsd xmm(reg), op +| subsd xmm(reg-ZREG_XMM0), op || break; || case ZEND_MUL: || case ZEND_ASSIGN_MUL: -| mulsd xmm(reg), op +| mulsd xmm(reg-ZREG_XMM0), op || break; || case ZEND_DIV: || case ZEND_ASSIGN_DIV: -| divsd xmm(reg), op +| divsd xmm(reg-ZREG_XMM0), op || break; ||} |.endmacro @@ -328,11 +314,11 @@ static void* dasm_labels[zend_lb_MAX]; |.endmacro |.macro SSE_MATH_REG, opcode, dst_reg, src_reg -| SSE_MATH, opcode, dst_reg, xmm(src_reg) +| SSE_MATH, opcode, dst_reg, xmm(src_reg-ZREG_XMM0) |.endmacro |.macro SSE_STORE, zv, reg -| movsd qword [zv], xmm(reg) +| movsd qword [zv], xmm(reg-ZREG_XMM0) |.endmacro |.macro LONG_OP, long_ins, reg, op_type, op @@ -2069,8 +2055,8 @@ static int zend_jit_math_long_long(dasm_State **Dst, |.cold_code |1: |.if X64 or SSE - int tmp_reg1 = 0; - int tmp_reg2 = 1; + zend_reg tmp_reg1 = ZREG_XMM0; + zend_reg tmp_reg2 = ZREG_XMM1; if (op1_type == IS_CONST) { | SSE_LOAD_LONG tmp_reg1, op1_type, op1 @@ -2130,7 +2116,7 @@ static int zend_jit_math_long_double(dasm_State **Dst, uint32_t res_offset) { |.if X64 or SSE - int result_reg = 0; + zend_reg result_reg = ZREG_XMM0; if (op1_type == IS_CONST) { | SSE_LOAD_LONG result_reg, op1_type, op1 @@ -2175,7 +2161,7 @@ static int zend_jit_math_double_long(dasm_State **Dst, uint32_t res_offset) { |.if X64 or SSE - || int result_reg = 0; + || zend_reg result_reg = ZREG_XMM0; || || if (opline->opcode == ZEND_ADD || opline->opcode == ZEND_MUL || opline->opcode == ZEND_ASSIGN_ADD || opline->opcode == ZEND_ASSIGN_MUL) { || if (op2_type == IS_CONST) { @@ -2248,7 +2234,7 @@ static int zend_jit_math_double_double(dasm_State **Dst, zend_bool same_ops = (op1_type == op2_type) && (op1.var == op2.var); |.if X64 or SSE - int result_reg = 0; + zend_reg result_reg = ZREG_XMM0; if (op1_type == IS_CONST) { | SSE_LOAD result_reg, op1_type, op1 @@ -2510,13 +2496,13 @@ static int zend_jit_math_helper(dasm_State **Dst, |6: | SAVE_VALID_OPLINE opline if (separate_op1) { - if (op1_reg != FCARG1a || op1_offset != 0) { + if (op1_reg != ZREG_FCARG1a || op1_offset != 0) { | SEPARATE_ZVAL_NOREF Ra(op1_reg)+op1_offset, op1_info, 0, op_array->filename, opline->lineno } else { | SEPARATE_ZVAL_NOREF_REG op1_info, 0, op_array->filename, opline->lineno } } - if (res_reg != FCARG1a || res_offset != 0) { + if (res_reg != ZREG_FCARG1a || res_offset != 0) { | lea FCARG1a, [Ra(res_reg)+res_offset] } if (op1_type == IS_CONST) { @@ -2595,9 +2581,9 @@ static int zend_jit_math(dasm_State **Dst, const zend_op *opline, int *opnum, ze | // call = EX(call); | mov RX, EX->call } - return zend_jit_math_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, FP, opline->op1.var, op1_info, opline->op2_type, opline->op2, FP, opline->op2.var, op2_info, RX, (opline+1)->result.var, RES_INFO(), 0); + return zend_jit_math_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, ZREG_FP, opline->op1.var, op1_info, opline->op2_type, opline->op2, ZREG_FP, opline->op2.var, op2_info, ZREG_RX, (opline+1)->result.var, RES_INFO(), 0); } else { - return zend_jit_math_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, FP, opline->op1.var, op1_info, opline->op2_type, opline->op2, FP, opline->op2.var, op2_info, FP, opline->result.var, RES_INFO(), 0); + return zend_jit_math_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, ZREG_FP, opline->op1.var, op1_info, opline->op2_type, opline->op2, ZREG_FP, opline->op2.var, op2_info, ZREG_FP, opline->result.var, RES_INFO(), 0); } fallback: @@ -2632,7 +2618,7 @@ static int zend_jit_concat_helper(dasm_State **Dst, | IF_NOT_Z_TYPE Ra(op2_reg)+op2_offset, IS_STRING, >6 } if (op1_type != IS_CONST && op1_reg == res_reg && op1_offset == res_offset) { - if (res_reg != FCARG1a || res_offset != 0) { + if (res_reg != ZREG_FCARG1a || res_offset != 0) { | lea FCARG1a, [Ra(res_reg)+res_offset] } if (op2_type == IS_CONST) { @@ -2642,7 +2628,7 @@ static int zend_jit_concat_helper(dasm_State **Dst, } | EXT_CALL zend_jit_fast_assign_concat_helper, r0 } else { - if (res_reg != FCARG1a || res_offset != 0) { + if (res_reg != ZREG_FCARG1a || res_offset != 0) { | lea FCARG1a, [Ra(res_reg)+res_offset] } if (op1_type == IS_CONST) { @@ -2678,7 +2664,7 @@ static int zend_jit_concat_helper(dasm_State **Dst, } #endif | SAVE_VALID_OPLINE opline - if (res_reg != FCARG1a || res_offset != 0) { + if (res_reg != ZREG_FCARG1a || res_offset != 0) { | lea FCARG1a, [Ra(res_reg)+res_offset] } if (op1_type == IS_CONST) { @@ -2748,9 +2734,9 @@ static int zend_jit_concat(dasm_State **Dst, const zend_op *opline, int *opnum, | // call = EX(call); | mov RX, EX->call } - return zend_jit_concat_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, FP, opline->op1.var, op1_info, opline->op2_type, opline->op2, FP, opline->op2.var, op2_info, RX, (opline+1)->result.var, RES_INFO()); + return zend_jit_concat_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, ZREG_FP, opline->op1.var, op1_info, opline->op2_type, opline->op2, ZREG_FP, opline->op2.var, op2_info, ZREG_RX, (opline+1)->result.var, RES_INFO()); } else { - return zend_jit_concat_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, FP, opline->op1.var, op1_info, opline->op2_type, opline->op2, FP, opline->op2.var, op2_info, FP, opline->result.var, RES_INFO()); + return zend_jit_concat_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, ZREG_FP, opline->op1.var, op1_info, opline->op2_type, opline->op2, ZREG_FP, opline->op2.var, op2_info, ZREG_FP, opline->result.var, RES_INFO()); } fallback: @@ -3098,7 +3084,7 @@ static int zend_jit_simple_assign(dasm_State **Dst, int in_cold) /* Labels: 1,2,3 */ { - ZEND_ASSERT(var_reg != r0); + ZEND_ASSERT(var_reg != ZREG_R0); if (val_type == IS_CONST) { zval *zv = RT_CONSTANT(op_array, val); if (var2 == (uint32_t)-1) { @@ -3123,13 +3109,13 @@ static int zend_jit_simple_assign(dasm_State **Dst, |1: } | // zend_error(E_NOTICE, "Undefined variable: %s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); - if (var_reg != FP) { + if (var_reg != ZREG_FP) { | mov aword [r4], Ra(var_reg) // save } | SAVE_VALID_OPLINE opline | mov FCARG1d, val.var | EXT_CALL zend_jit_undefined_op_helper, r0 - if (var_reg != FP) { + if (var_reg != ZREG_FP) { | mov Ra(var_reg), aword [r4] // restore } | SET_Z_TYPE_INFO Ra(var_reg)+var_offset, IS_NULL @@ -3145,7 +3131,7 @@ static int zend_jit_simple_assign(dasm_State **Dst, } if (val_info & MAY_BE_REF) { if (val_type == IS_CV) { - ZEND_ASSERT(var_reg != r2); + ZEND_ASSERT(var_reg != ZREG_R2); | lea r2, [FP + val.var] | ZVAL_DEREF r2, val_info if (var2 == (uint32_t)-1) { @@ -3236,7 +3222,7 @@ static int zend_jit_assign_to_variable(dasm_State **Dst, if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { int in_cold = 0; - ZEND_ASSERT(var_reg != r0); + ZEND_ASSERT(var_reg != ZREG_R0); if (var_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { | IF_Z_REFCOUNTED Ra(var_reg)+var_offset, >1 |.cold_code @@ -3261,10 +3247,10 @@ static int zend_jit_assign_to_variable(dasm_State **Dst, |4: } if (RC_MAY_BE_N(var_info)) { - if (var_reg == FP) { + if (var_reg == ZREG_FP) { | GET_Z_PTR FCARG1a, Ra(var_reg)+var_offset | IF_GC_MAY_NOT_LEAK FCARG1a, eax, >5 - } else if (var_reg != FCARG1a) { + } else if (var_reg != ZREG_FCARG1a) { | GET_Z_PTR FCARG1a, Ra(var_reg)+var_offset | IF_GC_MAY_NOT_LEAK FCARG1a, eax, >5 | mov [r4], Ra(var_reg) // save @@ -3275,7 +3261,7 @@ static int zend_jit_assign_to_variable(dasm_State **Dst, | GET_Z_PTR FCARG1a, Ra(var_reg)+var_offset } | EXT_CALL gc_possible_root, r0 - if (var_reg != FP) { + if (var_reg != ZREG_FP) { | mov Ra(var_reg), [r4] // restore } if (in_cold) { @@ -3418,7 +3404,7 @@ static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, zend_op_ if (opline->op2_type == IS_UNUSED) { uint32_t var_info = zend_array_element_type(op1_info, 0, 0); - if (!zend_jit_simple_assign(Dst, opline, op_array, ssa, FCARG1a, 0, var_info, (opline+1)->op1_type, (opline+1)->op1, val_info, + if (!zend_jit_simple_assign(Dst, opline, op_array, ssa, ZREG_FCARG1a, 0, var_info, (opline+1)->op1_type, (opline+1)->op1, val_info, opline->result_type == IS_UNUSED ? -1 : opline->result.var, 0)) { return 0; } @@ -3427,7 +3413,7 @@ static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, zend_op_ | // value = zend_assign_to_variable(variable_ptr, value, OP_DATA_TYPE); | ZVAL_DEREF FCARG1a, var_info - if (!zend_jit_assign_to_variable(Dst, opline, op_array, ssa, FCARG1a, 0, var_info, (opline+1)->op1_type, (opline+1)->op1, val_info, + if (!zend_jit_assign_to_variable(Dst, opline, op_array, ssa, ZREG_FCARG1a, 0, var_info, (opline+1)->op1_type, (opline+1)->op1, val_info, opline->result_type == IS_UNUSED ? -1 : opline->result.var)) { return 0; } @@ -3703,12 +3689,12 @@ static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, zend_ case ZEND_ASSIGN_SUB: case ZEND_ASSIGN_MUL: case ZEND_ASSIGN_DIV: - if (!zend_jit_math_helper(Dst, opline, op_array, ssa, IS_CV, opline->op1, FCARG1a, 0, var_info, (opline+1)->op1_type, (opline+1)->op1, FP, (opline+1)->op1.var, OP1_DATA_INFO(), FCARG1a, 0, OP1_DEF_INFO(), 1)) { + if (!zend_jit_math_helper(Dst, opline, op_array, ssa, IS_CV, opline->op1, ZREG_FCARG1a, 0, var_info, (opline+1)->op1_type, (opline+1)->op1, ZREG_FP, (opline+1)->op1.var, OP1_DATA_INFO(), ZREG_FCARG1a, 0, OP1_DEF_INFO(), 1)) { return 0; } break; case ZEND_ASSIGN_CONCAT: - if (!zend_jit_concat_helper(Dst, opline, op_array, ssa, IS_CV, opline->op1, FCARG1a, 0, var_info, (opline+1)->op1_type, (opline+1)->op1, FP, (opline+1)->op1.var, OP1_DATA_INFO(), FCARG1a, 0, OP1_DEF_INFO())) { + if (!zend_jit_concat_helper(Dst, opline, op_array, ssa, IS_CV, opline->op1, ZREG_FCARG1a, 0, var_info, (opline+1)->op1_type, (opline+1)->op1, ZREG_FP, (opline+1)->op1.var, OP1_DATA_INFO(), ZREG_FCARG1a, 0, OP1_DEF_INFO())) { return 0; } break; @@ -3895,9 +3881,9 @@ static int zend_jit_assign_op(dasm_State **Dst, const zend_op *opline, zend_op_a case ZEND_ASSIGN_SUB: case ZEND_ASSIGN_MUL: case ZEND_ASSIGN_DIV: - return zend_jit_math_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, FCARG1a, 0, op1_info, opline->op2_type, opline->op2, FP, opline->op2.var, op2_info, FCARG1a, 0, OP1_DEF_INFO(), 1); + return zend_jit_math_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, ZREG_FCARG1a, 0, op1_info, opline->op2_type, opline->op2, ZREG_FP, opline->op2.var, op2_info, ZREG_FCARG1a, 0, OP1_DEF_INFO(), 1); case ZEND_ASSIGN_CONCAT: - return zend_jit_concat_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, FCARG1a, 0, op1_info, opline->op2_type, opline->op2, FP, opline->op2.var, op2_info, FCARG1a, 0, OP1_DEF_INFO()); + return zend_jit_concat_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, ZREG_FCARG1a, 0, op1_info, opline->op2_type, opline->op2, ZREG_FP, opline->op2.var, op2_info, ZREG_FCARG1a, 0, OP1_DEF_INFO()); default: ZEND_ASSERT(0); } @@ -3907,9 +3893,9 @@ static int zend_jit_assign_op(dasm_State **Dst, const zend_op *opline, zend_op_a case ZEND_ASSIGN_SUB: case ZEND_ASSIGN_MUL: case ZEND_ASSIGN_DIV: - return zend_jit_math_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, FP, opline->op1.var, op1_info, opline->op2_type, opline->op2, FP, opline->op2.var, op2_info, FP, opline->op1.var, OP1_DEF_INFO(), 1); + return zend_jit_math_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, ZREG_FP, opline->op1.var, op1_info, opline->op2_type, opline->op2, ZREG_FP, opline->op2.var, op2_info, ZREG_FP, opline->op1.var, OP1_DEF_INFO(), 1); case ZEND_ASSIGN_CONCAT: - return zend_jit_concat_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, FP, opline->op1.var, op1_info, opline->op2_type, opline->op2, FP, opline->op2.var, op2_info, FP, opline->op1.var, OP1_DEF_INFO()); + return zend_jit_concat_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, ZREG_FP, opline->op1.var, op1_info, opline->op2_type, opline->op2, ZREG_FP, opline->op2.var, op2_info, ZREG_FP, opline->op1.var, OP1_DEF_INFO()); default: ZEND_ASSERT(0); } @@ -4310,7 +4296,7 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, i static int zend_jit_cmp_long_double(dasm_State **Dst, const zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa) { |.if X64 or SSE - || int tmp_reg = 1; + || zend_reg tmp_reg = ZREG_XMM1; || | SSE_LOAD_LONG tmp_reg, opline->op1_type, opline->op1 | SSE_OP ucomisd, tmp_reg, opline->op2_type, opline->op2 @@ -4327,12 +4313,12 @@ static int zend_jit_cmp_long_double(dasm_State **Dst, const zend_op *opline, int static int zend_jit_cmp_double_long(dasm_State **Dst, const zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa) { |.if X64 or SSE - int tmp_reg1 = 0; - int tmp_reg2 = 1; + zend_reg tmp_reg1 = ZREG_XMM0; + zend_reg tmp_reg2 = ZREG_XMM1; | SSE_LOAD tmp_reg1, opline->op1_type, opline->op1 | SSE_LOAD_LONG tmp_reg2, opline->op2_type, opline->op2 - | ucomisd xmm(tmp_reg1), xmm(tmp_reg2) + | ucomisd xmm(tmp_reg1-ZREG_XMM0), xmm(tmp_reg2-ZREG_XMM0) |.else | FPU_LONG_LOAD opline->op2_type, opline->op2 | FPU_LOAD opline->op1_type, opline->op1 @@ -4346,7 +4332,7 @@ static int zend_jit_cmp_double_long(dasm_State **Dst, const zend_op *opline, int static int zend_jit_cmp_double_double(dasm_State **Dst, const zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa) { |.if X64 or SSE - int tmp_reg = 0; + zend_reg tmp_reg = ZREG_XMM0; | SSE_LOAD tmp_reg, opline->op1_type, opline->op1 | SSE_OP ucomisd, tmp_reg, opline->op2_type, opline->op2 @@ -5305,7 +5291,7 @@ static int zend_jit_qm_assign(dasm_State **Dst, const zend_op *opline, zend_op_a { uint32_t op1_info = OP1_INFO(); - return zend_jit_simple_assign(Dst, opline, op_array, ssa, FP, opline->result.var, -1, opline->op1_type, opline->op1, op1_info, -1, 0); + return zend_jit_simple_assign(Dst, opline, op_array, ssa, ZREG_FP, opline->result.var, -1, opline->op1_type, opline->op1, op1_info, -1, 0); } static int zend_jit_assign(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) @@ -5322,12 +5308,12 @@ static int zend_jit_assign(dasm_State **Dst, const zend_op *opline, zend_op_arra if (op1_info & MAY_BE_REF) { | lea FCARG1a, [FP + opline->op1.var] | ZVAL_DEREF FCARG1a, op1_info - if (!zend_jit_assign_to_variable(Dst, opline, op_array, ssa, FCARG1a, 0, op1_info, opline->op2_type, opline->op2, op2_info, + if (!zend_jit_assign_to_variable(Dst, opline, op_array, ssa, ZREG_FCARG1a, 0, op1_info, opline->op2_type, opline->op2, op2_info, opline->result_type == IS_UNUSED ? -1 : opline->result.var)) { return 0; } } else { - if (!zend_jit_assign_to_variable(Dst, opline, op_array, ssa, FP, opline->op1.var, op1_info, opline->op2_type, opline->op2, op2_info, + if (!zend_jit_assign_to_variable(Dst, opline, op_array, ssa, ZREG_FP, opline->op1.var, op1_info, opline->op2_type, opline->op2, op2_info, opline->result_type == IS_UNUSED ? -1 : opline->result.var)) { return 0; } diff --git a/ext/opcache/jit/zend_jit_x86.h b/ext/opcache/jit/zend_jit_x86.h new file mode 100644 index 0000000000000..d6dc95dc3f0b3 --- /dev/null +++ b/ext/opcache/jit/zend_jit_x86.h @@ -0,0 +1,154 @@ +/* + +----------------------------------------------------------------------+ + | Zend JIT | + +----------------------------------------------------------------------+ + | Copyright (c) 1998-2016 The PHP Group | + +----------------------------------------------------------------------+ + | This source file is subject to version 3.01 of the PHP license, | + | that is bundled with this package in the file LICENSE, and is | + | available through the world-wide-web at the following url: | + | http://www.php.net/license/3_01.txt | + | If you did not receive a copy of the PHP license and are unable to | + | obtain it through the world-wide-web, please send a note to | + | license@php.net so we can mail you a copy immediately. | + +----------------------------------------------------------------------+ + | Authors: Dmitry Stogov | + +----------------------------------------------------------------------+ +*/ + +#ifndef HAVE_JIT_X86_H +#define HAVE_JIT_X86_H + +typedef enum _zend_reg { + ZREG_NONE = -1, + + ZREG_R0, + ZREG_R1, + ZREG_R2, + ZREG_R3, + ZREG_R4, + ZREG_R5, + ZREG_R6, + ZREG_R7, + + ZREG_RAX = ZREG_R0, + ZREG_RCX = ZREG_R1, + ZREG_RDX = ZREG_R2, + ZREG_RBX = ZREG_R3, + ZREG_RSP = ZREG_R4, + ZREG_RBP = ZREG_R5, + ZREG_RSI = ZREG_R6, + ZREG_RDI = ZREG_R7, + +#ifdef __x86_64__ + ZREG_R8, + ZREG_R9, + ZREG_R10, + ZREG_R11, + ZREG_R12, + ZREG_R13, + ZREG_R14, + ZREG_R15, +#endif + + ZREG_XMM0, + ZREG_XMM1, + ZREG_XMM2, + ZREG_XMM3, + ZREG_XMM4, + ZREG_XMM5, + ZREG_XMM6, + ZREG_XMM7, + +#ifdef __x86_64__ + ZREG_XMM8, + ZREG_XMM9, + ZREG_XMM10, + ZREG_XMM11, + ZREG_XMM12, + ZREG_XMM13, + ZREG_XMM14, + ZREG_XMM15, +#endif + + ZREG_NUM +} zend_reg; + +#ifdef __x86_64__ +# define ZREG_FP ZREG_R14 +# define ZREG_IP ZREG_R15 +# define ZREG_RX ZREG_IP +# define ZREG_FCARG1a ZREG_RDI +# define ZREG_FCARG2a ZREG_RSI +#else +# define ZREG_FP ZREG_RSI +# define ZREG_IP ZREG_RDI +# define ZREG_RX ZREG_IP +# define ZREG_FCARG1a ZREG_RCX +# define ZREG_FCARG2a ZREG_RDX +#endif + +extern const char *zend_reg_name[]; + +typedef uint32_t zend_regset; + +#define ZEND_REGSET_EMPTY 0 + +#define ZEND_REGSET_IS_EMPTY(regset) \ + (regset == ZEND_REGSET_EMPTY) + +#define ZEND_REGSET(reg) \ + (1 << (reg)) + +#define ZEND_REGSET_INTERVAL(reg1, reg2) \ + (((1 << ((reg2) - (reg1))) - 1) << (reg1)) + +#define ZEND_REGSET_IN(regset, reg) \ + (((regset) & ZEND_REGSET(reg)) != 0) + +#define ZEND_REGSET_INCL(regset, reg) \ + (regset) |= ZEND_REGSET(reg) + +#define ZEND_REGSET_EXCL(regset, reg) \ + (regset) &= ~ZEND_REGSET(reg) + +#define ZEND_REGSET_UNION(set1, set2) \ + ((set1) | (set2)) + +#define ZEND_REGSET_UNION(set1, set2) \ + ((set1) | (set2)) + +#define ZEND_REGSET_INTERSECTION(set1, set2) \ + ((set1) & (set2)) + +#define ZEND_REGSET_DIFFERENCE(set1, set2) \ + ((set1) & ~(set2)) + +#ifdef __x86_64__ +# define ZEND_REGSET_FIXED \ + (ZEND_REGSET(ZREG_RSP) | ZEND_REGSET(ZREG_R14) | ZEND_REGSET(ZREG_R15)) +# define ZEND_REGSET_GP \ + ZEND_REGSET_DIFFERENCE(ZEND_REGSET_INTERVAL(ZREG_R0, ZREG_R15), ZEND_REGSET_FIXED) +# define ZEND_REGSET_FP \ + ZEND_REGSET_DIFFERENCE(ZEND_REGSET_INTERVAL(ZREG_XMM0, ZREG_XMM15), ZEND_REGSET_FIXED) +#define ZEND_REGSET_SCRATCH \ + (ZEND_REGSET(ZREG_R0) | ZEND_REGSET(ZREG_R1) | ZEND_REGSET(ZREG_R2) | ZEND_REGSET(ZREG_R3) | ZEND_REGSET(ZREG_R12) | ZEND_REGSET_FP) +#else +# define ZEND_REGSET_FIXED \ + (ZEND_REGSET(ZREG_RSP) | ZEND_REGSET(ZREG_RSI) | ZEND_REGSET(ZREG_RDI)) +# define ZEND_REGSET_GP \ + ZEND_REGSET_DIFFERENCE(ZEND_REGSET_INTERVAL(ZREG_R0, ZREG_R7), ZEND_REGSET_FIXED) +# define ZEND_REGSET_FP \ + ZEND_REGSET_DIFFERENCE(ZEND_REGSET_INTERVAL(ZREG_XMM0, ZREG_XMM7), ZEND_REGSET_FIXED) +#define ZEND_REGSET_SCRATCH \ + (ZEND_REGSET(ZREG_EAX) | ZEND_REGSET(ZREG_ECX) | ZEND_REGSET(ZREG_EDX) | ZEND_REGSET_FP) +#endif + +#endif /* ZEND_JIT_X86_H */ +/* + * Local variables: + * tab-width: 4 + * c-basic-offset: 4 + * indent-tabs-mode: t + * End: + */ From c5c1edd4f37fd9076c6e92259b41b1679b13f88e Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 6 Mar 2017 11:29:07 +0300 Subject: [PATCH 370/569] Support for interned strings changes --- ext/opcache/jit/zend_jit_disasm_x86.c | 5 ----- ext/opcache/jit/zend_jit_helpers.c | 18 +++--------------- 2 files changed, 3 insertions(+), 20 deletions(-) diff --git a/ext/opcache/jit/zend_jit_disasm_x86.c b/ext/opcache/jit/zend_jit_disasm_x86.c index f724232420f28..20497a55c535b 100644 --- a/ext/opcache/jit/zend_jit_disasm_x86.c +++ b/ext/opcache/jit/zend_jit_disasm_x86.c @@ -372,11 +372,6 @@ static int zend_jit_disasm_init(void) REGISTER_EG(vm_stack_end); REGISTER_EG(symbol_table); #undef REGISTER_EG -#define REGISTER_CG(n) \ - zend_jit_disasm_add_symbol("CG("#n")", \ - (uint64_t)(uintptr_t)&compiler_globals.n, sizeof(compiler_globals.n)) - REGISTER_CG(known_strings); -#undef REGISTER_CG #endif /* Register JIT helper functions */ diff --git a/ext/opcache/jit/zend_jit_helpers.c b/ext/opcache/jit/zend_jit_helpers.c index 0367b8ae6371f..ef20240aabd79 100644 --- a/ext/opcache/jit/zend_jit_helpers.c +++ b/ext/opcache/jit/zend_jit_helpers.c @@ -604,11 +604,7 @@ static void ZEND_FASTCALL zend_jit_fetch_dim_str_r_helper(zval *container, zval real_offset = (UNEXPECTED(offset < 0)) /* Handle negative offset */ ? (zend_long)Z_STRLEN_P(container) + offset : offset; c = (zend_uchar)Z_STRVAL_P(container)[real_offset]; - if (CG(one_char_string)[c]) { - ZVAL_INTERNED_STR(result, CG(one_char_string)[c]); - } else { - ZVAL_NEW_STR(result, zend_string_init(Z_STRVAL_P(container) + real_offset, 1, 0)); - } + ZVAL_INTERNED_STR(result, ZSTR_CHAR(c)); } } @@ -655,11 +651,7 @@ static void ZEND_FASTCALL zend_jit_fetch_dim_str_is_helper(zval *container, zval real_offset = (UNEXPECTED(offset < 0)) /* Handle negative offset */ ? (zend_long)Z_STRLEN_P(container) + offset : offset; c = (zend_uchar)Z_STRVAL_P(container)[real_offset]; - if (CG(one_char_string)[c]) { - ZVAL_INTERNED_STR(result, CG(one_char_string)[c]); - } else { - ZVAL_NEW_STR(result, zend_string_init(Z_STRVAL_P(container) + real_offset, 1, 0)); - } + ZVAL_INTERNED_STR(result, ZSTR_CHAR(c)); } } @@ -931,11 +923,7 @@ static zend_never_inline void zend_assign_to_string_offset(zval *str, zval *dim, if (result) { /* Return the new character */ - if (CG(one_char_string)[c]) { - ZVAL_INTERNED_STR(result, CG(one_char_string)[c]); - } else { - ZVAL_NEW_STR(result, zend_string_init(Z_STRVAL_P(str) + offset, 1, 0)); - } + ZVAL_INTERNED_STR(result, ZSTR_CHAR(c)); } } From ccb4544619ee4c8b98743db6e9bf01f582f997d3 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 6 Mar 2017 11:32:25 +0300 Subject: [PATCH 371/569] Fixes in register definition macros --- ext/opcache/jit/zend_jit_x86.h | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.h b/ext/opcache/jit/zend_jit_x86.h index d6dc95dc3f0b3..b0c0509147959 100644 --- a/ext/opcache/jit/zend_jit_x86.h +++ b/ext/opcache/jit/zend_jit_x86.h @@ -31,15 +31,6 @@ typedef enum _zend_reg { ZREG_R6, ZREG_R7, - ZREG_RAX = ZREG_R0, - ZREG_RCX = ZREG_R1, - ZREG_RDX = ZREG_R2, - ZREG_RBX = ZREG_R3, - ZREG_RSP = ZREG_R4, - ZREG_RBP = ZREG_R5, - ZREG_RSI = ZREG_R6, - ZREG_RDI = ZREG_R7, - #ifdef __x86_64__ ZREG_R8, ZREG_R9, @@ -74,6 +65,15 @@ typedef enum _zend_reg { ZREG_NUM } zend_reg; +#define ZREG_RAX ZREG_R0 +#define ZREG_RCX ZREG_R1 +#define ZREG_RDX ZREG_R2 +#define ZREG_RBX ZREG_R3 +#define ZREG_RSP ZREG_R4 +#define ZREG_RBP ZREG_R5 +#define ZREG_RSI ZREG_R6 +#define ZREG_RDI ZREG_R7 + #ifdef __x86_64__ # define ZREG_FP ZREG_R14 # define ZREG_IP ZREG_R15 @@ -101,7 +101,7 @@ typedef uint32_t zend_regset; (1 << (reg)) #define ZEND_REGSET_INTERVAL(reg1, reg2) \ - (((1 << ((reg2) - (reg1))) - 1) << (reg1)) + (((1 << ((reg2) - (reg1) + 1)) - 1) << (reg1)) #define ZEND_REGSET_IN(regset, reg) \ (((regset) & ZEND_REGSET(reg)) != 0) @@ -141,7 +141,7 @@ typedef uint32_t zend_regset; # define ZEND_REGSET_FP \ ZEND_REGSET_DIFFERENCE(ZEND_REGSET_INTERVAL(ZREG_XMM0, ZREG_XMM7), ZEND_REGSET_FIXED) #define ZEND_REGSET_SCRATCH \ - (ZEND_REGSET(ZREG_EAX) | ZEND_REGSET(ZREG_ECX) | ZEND_REGSET(ZREG_EDX) | ZEND_REGSET_FP) + (ZEND_REGSET(ZREG_RAX) | ZEND_REGSET(ZREG_RCX) | ZEND_REGSET(ZREG_RDX) | ZEND_REGSET_FP) #endif #endif /* ZEND_JIT_X86_H */ From cb544ccfe86f1d9cd79bb18e8069102ed7e800d5 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 6 Mar 2017 13:26:41 +0300 Subject: [PATCH 372/569] Sort intervals, starting at the same position, acording to the end position --- ext/opcache/jit/zend_jit.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index ad9b1f5e91063..bf49a3af18077 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -1089,7 +1089,9 @@ static zend_lifetime_interval *zend_jit_sort_intervals(zend_lifetime_interval ** if (!last) { list = last = ival; ival->next = NULL; - } else if (ival->start >= last->start) { + } else if ((ival->start > last->start) || + (ival->start == last->start && + ival->end > last->end)) { last->next = ival; last = ival; ival->next = NULL; @@ -1102,7 +1104,9 @@ static zend_lifetime_interval *zend_jit_sort_intervals(zend_lifetime_interval ** ival->next = NULL; break; } else*/ - if (ival->start < (*p)->start) { + if ((ival->start < (*p)->start) || + (ival->start == (*p)->start && + ival->end < (*p)->end)) { ival->next = *p; *p = ival; break; From 23b7d53225c88b5c5b955ef393105c50b401697a Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 6 Mar 2017 17:34:45 +0300 Subject: [PATCH 373/569] Fixed crash (this check can't be avoided anymore) --- ext/opcache/jit/zend_jit.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index bf49a3af18077..cba52f27dbab2 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -1099,12 +1099,11 @@ static zend_lifetime_interval *zend_jit_sort_intervals(zend_lifetime_interval ** zend_lifetime_interval **p = &list; while (1) { - /*if (*p == NULL) { + if (*p == NULL) { *p = last = ival; ival->next = NULL; break; - } else*/ - if ((ival->start < (*p)->start) || + } else if ((ival->start < (*p)->start) || (ival->start == (*p)->start && ival->end < (*p)->end)) { ival->next = *p; From 775bbb1da2c4a2e0a43301c918c4430e3aff8530 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 7 Mar 2017 14:11:54 +0300 Subject: [PATCH 374/569] Linear Scan Register Allocator (incomplete) --- ext/opcache/jit/zend_jit.c | 88 +++++++++++++++++++----- ext/opcache/jit/zend_jit_x86.dasc | 107 ++++++++++++++++++++++++++++++ 2 files changed, 179 insertions(+), 16 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index cba52f27dbab2..66169dd9c0a6e 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -966,10 +966,8 @@ struct _zend_lifetime_interval { int ssa_var; int8_t reg; zend_bool split; - //???zend_bool fixed; uint32_t start; uint32_t end; - //zend_regset regset; zend_lifetime_interval *next; }; @@ -986,7 +984,6 @@ static int zend_jit_add_range(zend_lifetime_interval **intervals, int var, uint3 ival->ssa_var = var; ival->reg = ZREG_NONE; ival->split = 0; - //???ival->fixed = 0; ival->start = from; ival->end = to; ival->next = intervals[var]; @@ -1062,7 +1059,6 @@ static int zend_jit_split_interval(zend_lifetime_interval *current, uint32_t pos ival->ssa_var = current->ssa_var; ival->reg = ZREG_NONE; ival->split = 1; - //???ival->fixed = 0; ival->start = pos; ival->end = current->end; @@ -1121,6 +1117,25 @@ static zend_lifetime_interval *zend_jit_sort_intervals(zend_lifetime_interval ** return list; } +#ifdef DEBUG_REG_ALLOC +static void zend_jit_print_regset(zend_regset regset) +{ + zend_reg reg; + int first = 1; + + for (reg = 0; reg < ZREG_NUM; reg++) { + if (ZEND_REGSET_IN(regset, reg)) { + if (first) { + first = 0; + fprintf(stderr, "%s", zend_reg_name[reg]); + } else { + fprintf(stderr, ", %s", zend_reg_name[reg]); + } + } + } +} +#endif + /* See "Linear Scan Register Allocation on SSA Form", Christian Wimmer and Michael Franz, CGO'10 (2010), Figure 4. */ static int zend_jit_compute_liveness(zend_op_array *op_array, zend_ssa *ssa, zend_bitset candidates, zend_lifetime_interval **list) @@ -1278,8 +1293,7 @@ static int zend_jit_compute_liveness(zend_op_array *op_array, zend_ssa *ssa, zen if (intervals[i]) { zend_lifetime_interval *ival = intervals[i]; - fprintf(stderr, "#%d: ", ival->ssa_var); - fprintf(stderr, "%u-%u", ival->start, ival->end); + fprintf(stderr, "#%d: %u-%u", ival->ssa_var, ival->start, ival->end); ival = ival->next; while (ival) { fprintf(stderr, ", %u-%u", ival->start, ival->end); @@ -1303,18 +1317,21 @@ static int zend_jit_compute_liveness(zend_op_array *op_array, zend_ssa *ssa, zen Christian Wimmer VEE'05 (2005), Figure 4. Allocation without spilling. and "Linear Scan Register Allocation on SSA Form", Christian Wimmer and Michael Franz, CGO'10 (2010), Figure 6. */ -static int zend_jit_try_allocate_free_reg(zend_op_array *op_array, zend_ssa *ssa, zend_lifetime_interval *current, zend_lifetime_interval *active, zend_lifetime_interval *inactive, zend_lifetime_interval **list) +static int zend_jit_try_allocate_free_reg(zend_op_array *op_array, zend_ssa *ssa, zend_lifetime_interval *current, zend_regset available, zend_lifetime_interval *active, zend_lifetime_interval *inactive, zend_lifetime_interval **list) { zend_lifetime_interval *it; uint32_t freeUntilPos[ZREG_NUM]; uint32_t pos; zend_reg i, reg; - zend_regset available; if ((ssa->var_info[current->ssa_var].type & MAY_BE_ANY) == MAY_BE_DOUBLE) { - available = ZEND_REGSET_FP; + available = ZEND_REGSET_INTERSECTION(available, ZEND_REGSET_FP); } else { - available = ZEND_REGSET_GP; + available = ZEND_REGSET_INTERSECTION(available, ZEND_REGSET_GP); + } + + if (ZEND_REGSET_IS_EMPTY(available)) { + return 0; } /* Set freeUntilPos of all physical registers to maxInt */ @@ -1336,12 +1353,38 @@ static int zend_jit_try_allocate_free_reg(zend_op_array *op_array, zend_ssa *ssa it = inactive; while (it) { if (current->start < it->end && current->end > it->start) { - freeUntilPos[it->reg] = 0; + uint32_t next = it->start; + + if (next < freeUntilPos[it->reg]) { + freeUntilPos[it->reg] = next; + } } it = it->next; } } + /* Handle Scratch Registers */ + /* TODO: Optimize ??? */ + if (current->start + 1 < current->end) { + uint32_t line = current->start + 1; + zend_regset regset; + zend_reg reg; + + do { + regset = zend_jit_get_scratch_regset(op_array, ssa, op_array->opcodes + line); + if (!ZEND_REGSET_IS_EMPTY(regset)) { + for (reg = 0; reg < ZREG_NUM; reg++) { + if (ZEND_REGSET_IN(regset, reg)) { + if (line < freeUntilPos[reg]) { + freeUntilPos[reg] = line; + } + } + } + } + line++; + } while (line != current->end); + } + #if 0 /* Coalesing */ if (ssa->vars[current->ssa_var].definition == current->start) { @@ -1388,12 +1431,17 @@ static int zend_jit_try_allocate_free_reg(zend_op_array *op_array, zend_ssa *ssa current->reg = reg; return 1; } else { + /* TODO: enable interval splitting */ +#if 1 + return 0; +#else /* register available for the first part of the interval */ if (zend_jit_split_interval(current, pos, list) != SUCCESS) { return 0; } current->reg = reg; return 1; +#endif } } @@ -1414,6 +1462,7 @@ static int zend_jit_linear_scan(zend_op_array *op_array, zend_ssa *ssa, zend_lif zend_lifetime_interval *unhandled, *active, *inactive, *handled; zend_lifetime_interval *current, **p, *q; uint32_t position; + zend_regset available = ZEND_REGSET_UNION(ZEND_REGSET_GP, ZEND_REGSET_FP); #ifdef DEBUG_REG_ALLOC { @@ -1438,13 +1487,15 @@ static int zend_jit_linear_scan(zend_op_array *op_array, zend_ssa *ssa, zend_lif p = &active; while (*p) { q = *p; - if (q->end < position) { + if (q->end <= position) { /* move ival from active to handled */ + ZEND_REGSET_INCL(available, q->reg); *p = q->next; q->next = handled; handled = q; } else if (q->start > position) { /* move ival from active to inactive */ + ZEND_REGSET_INCL(available, q->reg); *p = q->next; q->next = inactive; inactive = q; @@ -1463,6 +1514,7 @@ static int zend_jit_linear_scan(zend_op_array *op_array, zend_ssa *ssa, zend_lif handled = q; } else if (q->start <= position) { /* move ival from inactive to active */ + ZEND_REGSET_EXCL(available, q->reg); *p = q->next; q->next = active; active = q; @@ -1471,10 +1523,14 @@ static int zend_jit_linear_scan(zend_op_array *op_array, zend_ssa *ssa, zend_lif } } - if (zend_jit_try_allocate_free_reg(op_array, ssa, current, active, inactive, &unhandled) || + if (zend_jit_try_allocate_free_reg(op_array, ssa, current, available, active, inactive, &unhandled) || zend_jit_allocate_blocked_reg()) { + ZEND_REGSET_EXCL(available, current->reg); current->next = active; active = current; + } else { + current->next = handled; + handled = current; } } @@ -1484,7 +1540,7 @@ static int zend_jit_linear_scan(zend_op_array *op_array, zend_ssa *ssa, zend_lif fprintf(stderr, "After Linear Scan\n"); while (ival != NULL) { - if (ival->reg >= 0) { + if (ival->reg > ZREG_NONE) { fprintf(stderr, "%u-%u: #%d (%s)\n", ival->start, ival->end, ival->ssa_var, zend_reg_name[ival->reg]); } else { fprintf(stderr, "%u-%u: #%d (no-reg)\n", ival->start, ival->end, ival->ssa_var); @@ -1494,7 +1550,7 @@ static int zend_jit_linear_scan(zend_op_array *op_array, zend_ssa *ssa, zend_lif ival = inactive; while (ival != NULL) { - if (ival->reg >= 0) { + if (ival->reg > ZREG_NONE) { fprintf(stderr, "%u-%u: #%d (%s)\n", ival->start, ival->end, ival->ssa_var, zend_reg_name[ival->reg]); } else { fprintf(stderr, "%u-%u: #%d (no-reg)\n", ival->start, ival->end, ival->ssa_var); @@ -1504,7 +1560,7 @@ static int zend_jit_linear_scan(zend_op_array *op_array, zend_ssa *ssa, zend_lif ival = active; while (ival != NULL) { - if (ival->reg >= 0) { + if (ival->reg > ZREG_NONE) { fprintf(stderr, "%u-%u: #%d (%s)\n", ival->start, ival->end, ival->ssa_var, zend_reg_name[ival->reg]); } else { fprintf(stderr, "%u-%u: #%d (no-reg)\n", ival->start, ival->end, ival->ssa_var); diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 0d59d592dea31..f013d14419d95 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -7888,6 +7888,113 @@ static int zend_jit_free(dasm_State **Dst, const zend_op *opline, zend_op_array return 1; } +static zend_regset zend_jit_get_scratch_regset(zend_op_array *op_array, zend_ssa *ssa, zend_op *opline) +{ + uint32_t op1_info, op2_info, res_info; + zend_regset regset = ZEND_REGSET_SCRATCH; + + switch (opline->opcode) { + case ZEND_NOP: + case ZEND_OP_DATA: + case ZEND_JMP: + regset = ZEND_REGSET_EMPTY; + break; + case ZEND_QM_ASSIGN: + op1_info = OP1_INFO(); + if (!(op1_info & MAY_BE_UNDEF)) { + res_info = RES_INFO(); + if ((res_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) { + regset = ZEND_REGSET(ZREG_XMM0); + } else if ((res_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) { + regset = ZEND_REGSET(ZREG_R0); + } else { + regset = ZEND_REGSET_UNION(ZEND_REGSET(ZREG_R0), ZEND_REGSET(ZREG_R2)); + } + } + break; + case ZEND_PRE_INC: + case ZEND_PRE_DEC: + case ZEND_POST_INC: + case ZEND_POST_DEC: + op1_info = OP1_INFO(); + if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG))) { + regset = ZEND_REGSET_EMPTY; + if (opline->result_type != IS_UNUSED) { + ZEND_REGSET_INCL(regset, ZREG_R0); + ZEND_REGSET_INCL(regset, ZREG_R1); + } + } + break; + case ZEND_ADD: + case ZEND_SUB: + case ZEND_MUL: + op1_info = OP1_INFO(); + op2_info = OP1_INFO(); + if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) && + !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { + res_info = OP1_INFO(); + if (res_info & MAY_BE_LONG) { + regset = ZEND_REGSET(ZREG_R0); + if (res_info & MAY_BE_DOUBLE) { + ZEND_REGSET_INCL(regset, ZREG_XMM0); + ZEND_REGSET_INCL(regset, ZREG_XMM1); + } + } else { + regset = ZEND_REGSET_UNION(ZEND_REGSET(ZREG_XMM0), ZEND_REGSET(ZREG_XMM1)); + } + } + break; + case ZEND_IS_SMALLER: + case ZEND_IS_SMALLER_OR_EQUAL: + case ZEND_IS_EQUAL: + case ZEND_IS_NOT_EQUAL: + case ZEND_IS_IDENTICAL: + case ZEND_IS_NOT_IDENTICAL: + op1_info = OP1_INFO(); + op2_info = OP1_INFO(); + if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) && + !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { + regset = ZEND_REGSET(ZREG_R0); + if ((op1_info & MAY_BE_DOUBLE) || (op2_info & MAY_BE_DOUBLE)) { + ZEND_REGSET_INCL(regset, ZREG_XMM0); + ZEND_REGSET_INCL(regset, ZREG_XMM1); + } + } + break; + case ZEND_BOOL: + case ZEND_BOOL_NOT: + case ZEND_JMPZ: + case ZEND_JMPNZ: + case ZEND_JMPZNZ: + case ZEND_JMPZ_EX: + case ZEND_JMPNZ_EX: + op1_info = OP1_INFO(); + if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE)))) { + regset = ZEND_REGSET_EMPTY; + if (opline->opcode == ZEND_BOOL || + opline->opcode == ZEND_BOOL_NOT || + opline->opcode == ZEND_JMPZ_EX || + opline->opcode == ZEND_JMPNZ_EX) { + ZEND_REGSET_INCL(regset, ZREG_R0); + } + } + break; + case ZEND_DO_UCALL: + case ZEND_DO_FCALL: + case ZEND_DO_FCALL_BY_NAME: + case ZEND_INCLUDE_OR_EVAL: + case ZEND_GENERATOR_CREATE: + case ZEND_YIELD: + case ZEND_YIELD_FROM: + regset = ZEND_REGSET_UNION(ZEND_REGSET_GP, ZEND_REGSET_FP); + break; + default: + break; + } + + return regset; +} + #ifdef ZEND_JIT_RECORD static int zend_jit_func_header(dasm_State **Dst, zend_op_array *op_array) { From 9f3206a1f207b7b36c7697ce697a78e486599a7c Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Wed, 8 Mar 2017 14:26:23 +0800 Subject: [PATCH 375/569] Duplicated define --- ext/opcache/jit/zend_jit_x86.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.h b/ext/opcache/jit/zend_jit_x86.h index b0c0509147959..713ce6873a1df 100644 --- a/ext/opcache/jit/zend_jit_x86.h +++ b/ext/opcache/jit/zend_jit_x86.h @@ -115,9 +115,6 @@ typedef uint32_t zend_regset; #define ZEND_REGSET_UNION(set1, set2) \ ((set1) | (set2)) -#define ZEND_REGSET_UNION(set1, set2) \ - ((set1) | (set2)) - #define ZEND_REGSET_INTERSECTION(set1, set2) \ ((set1) & (set2)) From 37058a5b06b588b2c6743adc967e2cdd284cfd0c Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 15 Mar 2017 15:21:27 +0300 Subject: [PATCH 376/569] JIT code generator refactoring --- ext/opcache/jit/zend_jit.c | 3 +- ext/opcache/jit/zend_jit_x86.dasc | 4202 ++++++++++++++--------------- ext/opcache/jit/zend_jit_x86.h | 44 + 3 files changed, 2097 insertions(+), 2152 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 66169dd9c0a6e..ee43ffe7bbd12 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -1597,8 +1597,7 @@ static int zend_jit_allocate_registers(zend_op_array *op_array, zend_ssa *ssa) for (i = 0; i < ssa->vars_count; i++) { if (/*!ssa->vars[i].no_val && */!(ssa->var_info[i].type & MAY_BE_REF) - && has_concrete_type(ssa->var_info[i].type) - && (ssa->var_info[i].type & MAY_BE_ANY) == MAY_BE_DOUBLE) { + && zend_jit_may_be_in_reg(op_array, ssa, i)) { zend_bitset_incl(candidates, i); candidates_count++; } diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index f013d14419d95..457d3a9e09c4a 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -93,98 +93,109 @@ static void* dasm_labels[zend_lb_MAX]; #define BP_JIT_IS 6 |.macro LOAD_ADDR, reg, addr -|.if X64 -||if (IS_32BIT(addr)) { -| mov reg, ((ptrdiff_t)addr) // 0x48 0xc7 0xc0 -|| } else { -| mov64 reg, ((ptrdiff_t)addr) // 0x48 0xb8 -||} -|.else -| mov reg, ((ptrdiff_t)addr) -|.endif +| .if X64 +|| if (IS_32BIT(addr)) { +| mov reg, ((ptrdiff_t)addr) // 0x48 0xc7 0xc0 +|| } else { +| mov64 reg, ((ptrdiff_t)addr) // 0x48 0xb8 +|| } +| .else +| mov reg, ((ptrdiff_t)addr) +| .endif |.endmacro |.macro PUSH_ADDR, addr, tmp_reg -|.if X64 -||if (IS_32BIT(addr)) { -| push ((ptrdiff_t)addr) // 0x48 0xc7 0xc0 -|| } else { -| mov64 tmp_reg, ((ptrdiff_t)addr) // 0x48 0xb8 -| push tmp_reg -||} -|.else -| push ((ptrdiff_t)addr) -|.endif +| .if X64 +|| if (IS_32BIT(addr)) { +| push ((ptrdiff_t)addr) // 0x48 0xc7 0xc0 +|| } else { +| mov64 tmp_reg, ((ptrdiff_t)addr) // 0x48 0xb8 +| push tmp_reg +|| } +| .else +| push ((ptrdiff_t)addr) +| .endif +|.endmacro + +|.macro LOAD_BASE_ADDR, reg, base, offset +|| if (offset) { +| lea reg, qword [Ra(base)+offset] +|| } else { +| mov reg, Ra(base) +|| } +|.endmacro + +|.macro PUSH_BASE_ADDR, base, offset, tmp_reg +|| if (offset) { +| lea tmp_reg, qword [Ra(base)+offset] +| push tmp_reg +|| } else { +| push Ra(base) +|| } |.endmacro |.macro EXT_CALL, func, tmp_reg -|.if X64 -||if (IS_32BIT(dasm_end) && IS_32BIT(func)) { -| call qword &func -||} else { -| LOAD_ADDR tmp_reg, func -| call tmp_reg -||} -|.else -| call dword &func -|.endif +| .if X64 +|| if (IS_32BIT(dasm_end) && IS_32BIT(func)) { +| call qword &func +|| } else { +| LOAD_ADDR tmp_reg, func +| call tmp_reg +|| } +| .else +| call dword &func +| .endif |.endmacro |.macro EXT_JMP, func, tmp_reg -|.if X64 -||if (IS_32BIT(dasm_end) && IS_32BIT(func)) { -| jmp qword &func -||} else { -| LOAD_ADDR tmp_reg, func -| jmp tmp_reg -||} -|.else -| jmp dword &func -|.endif +| .if X64 +|| if (IS_32BIT(dasm_end) && IS_32BIT(func)) { +| jmp qword &func +|| } else { +| LOAD_ADDR tmp_reg, func +| jmp tmp_reg +|| } +| .else +| jmp dword &func +| .endif |.endmacro -|.macro LOAD_ZVAL_ADDR, reg, op_type, op -||if (op_type == IS_CONST) { -| .if X64 -| mov reg, aword EX->literals -| add reg, op.constant -| .else -| mov reg, op.zv -| .endif -||} else { -| lea reg, qword [FP + op.var] -||} +|.macro LOAD_ZVAL_ADDR, reg, addr +|| if (Z_MODE(addr) == IS_CONST_ZVAL) { +| LOAD_ADDR reg, Z_ZV(addr) +|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { +| LOAD_BASE_ADDR reg, Z_REG(addr), Z_OFFSET(addr) +|| } else { +|| ZEND_ASSERT(0); +|| } |.endmacro -|.macro PUSH_ZVAL_ADDR, op_type, op, tmp_reg -||if (op_type == IS_CONST) { -| .if X64 -| mov tmp_reg, aword EX->literals -| add tmp_reg, op.constant -| push tmp_reg -| .else -| push op.zv -| .endif -||} else { -| lea tmp_reg, qword [FP + op.var] -| push tmp_reg -||} +|.macro PUSH_ZVAL_ADDR, addr, tmp_reg +|| if (Z_MODE(addr) == IS_CONST_ZVAL) { +| PUSH_ADDR Z_ZV(addr), tmp_reg +|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { +| PUSH_BASE_ADDR Z_REG(addr), Z_OFFSET(addr), tmp_reg +|| } else { +|| ZEND_ASSERT(0); +|| } |.endmacro |.macro GET_Z_TYPE_INFO, reg, zv -| mov reg, dword [zv + 8] +| mov reg, dword [zv+offsetof(zval,u1.type_info)] |.endmacro |.macro SET_Z_TYPE_INFO, zv, type -| mov dword [zv + 8], type +| mov dword [zv+offsetof(zval,u1.type_info)], type |.endmacro -|.macro GET_Z_LVAL, reg, zv -| mov reg, aword [zv] +|.macro GET_ZVAL_TYPE_INFO, reg, addr +|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); +| mov reg, dword [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval,u1.type_info)] |.endmacro -|.macro SET_Z_LVAL, zv, val -| mov aword [zv], val +|.macro SET_ZVAL_TYPE_INFO, addr, type +|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); +| mov dword [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval,u1.type_info)], type |.endmacro |.macro GET_Z_PTR, reg, zv @@ -195,510 +206,497 @@ static void* dasm_labels[zend_lb_MAX]; | mov aword [zv], val |.endmacro -|.macro GET_Z_W2, reg, zv -| mov reg, dword [zv + 4] +|.macro GET_ZVAL_PTR, reg, addr +|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); +| mov reg, aword [Ra(Z_REG(addr))+Z_OFFSET(addr)] |.endmacro -|.macro SET_Z_W2, zv, val -| mov dword [zv + 4], val +|.macro SET_ZVAL_PTR, addr, val +|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); +| mov aword [Ra(Z_REG(addr))+Z_OFFSET(addr)], val |.endmacro -|.macro FPU_OP, fp_ins, op_type, op -||if (op_type == IS_CONST) { -| .if X64 -| mov r0, aword EX->literals -| fp_ins qword [r0 + op.constant] -| .else -| fp_ins qword [op.zv] -| .endif -||} else { -| fp_ins qword [FP + op.var] -||} +|.macro GET_ZVAL_W2, reg, addr +|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); +| mov reg, dword [Ra(Z_REG(addr))+Z_OFFSET(addr)+4] |.endmacro -|.macro FPU_LOAD, op_type, op -| FPU_OP fld, op_type, op +|.macro SET_ZVAL_W2, addr, val +|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); +| mov dword [Ra(Z_REG(addr))+Z_OFFSET(addr)+4], val |.endmacro -|.macro FPU_MATH, opcode, op_type, op -||switch (opcode) { -|| case ZEND_ADD: -|| case ZEND_ASSIGN_ADD: -| FPU_OP fadd, op_type, op -|| break; -|| case ZEND_SUB: -|| case ZEND_ASSIGN_SUB: -| FPU_OP fsub, op_type, op -|| break; -|| case ZEND_MUL: -|| case ZEND_ASSIGN_MUL: -| FPU_OP fmul, op_type, op -|| break; -|| case ZEND_DIV: -|| case ZEND_ASSIGN_DIV: -| FPU_OP fdiv, op_type, op -|| break; -||} -|.endmacro - -|.macro FPU_STORE, zv -| fstp qword [zv] +|.macro FPU_OP, fp_ins, addr +|| if (Z_MODE(addr) == IS_CONST_ZVAL) { +| .if X64 +|| if (IS_32BIT(Z_ZV(addr))) { +| fp_ins qword [Z_ZV(addr)] +|| } else { +| LOAD_ADDR r0, Z_ZV(addr) +| fp_ins qword [r0] +|| } +| .endif +|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { +| fp_ins qword [Ra(Z_REG(addr))+Z_OFFSET(addr)] +|| } else { +|| ZEND_ASSERT(0); +|| } |.endmacro -|.macro SSE_OP, sse_ins, reg, op_type, op -||if (op_type == IS_CONST) { -| .if X64 -| mov r0, aword EX->literals -| sse_ins xmm(reg-ZREG_XMM0), qword [r0 + op.constant] -| .else -| sse_ins xmm(reg-ZREG_XMM0), qword [op.zv] -| .endif -||} else { -| sse_ins xmm(reg-ZREG_XMM0), qword [FP + op.var] -||} +|.macro FPU_GET_ZVAL_DVAL, addr +| FPU_OP fld, addr |.endmacro -|.macro SSE_LOAD_LONG, reg, op_type, op -||if (op_type == IS_CONST) { -| .if X64 -| mov r0, aword EX->literals -| cvtsi2sd xmm(reg-ZREG_XMM0), qword [r0 + op.constant] -| .else -| cvtsi2sd xmm(reg-ZREG_XMM0), dword [op.zv] -| .endif -||} else { -| cvtsi2sd xmm(reg-ZREG_XMM0), aword [FP + op.var] -||} +|.macro FPU_MATH, opcode, addr +|| switch (opcode) { +|| case ZEND_ADD: +|| case ZEND_ASSIGN_ADD: +| FPU_OP fadd, addr +|| break; +|| case ZEND_SUB: +|| case ZEND_ASSIGN_SUB: +| FPU_OP fsub, addr +|| break; +|| case ZEND_MUL: +|| case ZEND_ASSIGN_MUL: +| FPU_OP fmul, addr +|| break; +|| case ZEND_DIV: +|| case ZEND_ASSIGN_DIV: +| FPU_OP fdiv, addr +|| break; +|| } |.endmacro -|.macro SSE_GET_Z_LVAL, reg, zv -| cvtsi2sd xmm(reg-ZREG_XMM0), aword [zv] +|.macro FPU_SET_ZVAL_DVAL, addr +|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); +| fstp qword [Ra(Z_REG(res_addr))+Z_OFFSET(res_addr)] |.endmacro -|.macro SSE_LOAD, reg, op_type, op -| SSE_OP movsd, reg, op_type, op +|.macro SSE_OP, sse_ins, reg, addr +|| if (Z_MODE(addr) == IS_CONST_ZVAL) { +| .if X64 +|| if (IS_32BIT(Z_ZV(addr))) { +| sse_ins xmm(reg-ZREG_XMM0), qword [Z_ZV(addr)] +|| } else { +| LOAD_ADDR r0, Z_ZV(addr) +| sse_ins xmm(reg-ZREG_XMM0), qword [r0] +|| } +| .else +| sse_ins xmm(reg-ZREG_XMM0), qword [Z_ZV(addr)] +| .endif +|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { +| sse_ins xmm(reg-ZREG_XMM0), qword [Ra(Z_REG(addr))+Z_OFFSET(addr)] +|| } else { +|| ZEND_ASSERT(0); +|| } |.endmacro -|.macro SSE_GET_Z_DVAL, reg, zv -| movsd xmm(reg-ZREG_XMM0), qword [zv] +|.macro SSE_GET_ZVAL_LVAL, reg, addr +|| if (Z_MODE(addr) == IS_CONST_ZVAL) { +| .if X64 +|| if (IS_32BIT(Z_ZV(addr))) { +| cvtsi2sd xmm(reg-ZREG_XMM0), aword [Z_ZV(addr)] +|| } else { +| LOAD_ADDR r0, Z_ZV(addr) +| cvtsi2sd xmm(reg-ZREG_XMM0), aword [r0] +|| } +| .else +| cvtsi2sd xmm(reg-ZREG_XMM0), aword [Z_ZV(addr)] +| .endif +|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { +| cvtsi2sd xmm(reg-ZREG_XMM0), aword [Ra(Z_REG(addr))+Z_OFFSET(addr)] +|| } else { +|| ZEND_ASSERT(0); +|| } |.endmacro -|.macro SSE_MATH, opcode, reg, op -||switch (opcode) { -|| case ZEND_ADD: -|| case ZEND_ASSIGN_ADD: -| addsd xmm(reg-ZREG_XMM0), op -|| break; -|| case ZEND_SUB: -|| case ZEND_ASSIGN_SUB: -| subsd xmm(reg-ZREG_XMM0), op -|| break; -|| case ZEND_MUL: -|| case ZEND_ASSIGN_MUL: -| mulsd xmm(reg-ZREG_XMM0), op -|| break; -|| case ZEND_DIV: -|| case ZEND_ASSIGN_DIV: -| divsd xmm(reg-ZREG_XMM0), op -|| break; -||} +|.macro SSE_GET_ZVAL_DVAL, reg, addr +| SSE_OP movsd, reg, addr |.endmacro -|.macro SSE_MATH_CONST, opcode, reg, op -| .if X64 -| mov r0, aword EX->literals -| SSE_MATH, opcode, reg, qword [r0 + op.constant] -| .else -| SSE_MATH, opcode, reg, qword [op.zv] -| .endif +|.macro SSE_MATH, opcode, reg, addr +|| switch (opcode) { +|| case ZEND_ADD: +|| case ZEND_ASSIGN_ADD: +| SSE_OP addsd, reg, addr +|| break; +|| case ZEND_SUB: +|| case ZEND_ASSIGN_SUB: +| SSE_OP subsd, reg, addr +|| break; +|| case ZEND_MUL: +|| case ZEND_ASSIGN_MUL: +| SSE_OP mulsd, reg, addr +|| break; +|| case ZEND_DIV: +|| case ZEND_ASSIGN_DIV: +| SSE_OP divsd, reg, addr +|| break; +|| } |.endmacro |.macro SSE_MATH_REG, opcode, dst_reg, src_reg -| SSE_MATH, opcode, dst_reg, xmm(src_reg-ZREG_XMM0) +|| switch (opcode) { +|| case ZEND_ADD: +|| case ZEND_ASSIGN_ADD: +| addsd xmm(dst_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0) +|| break; +|| case ZEND_SUB: +|| case ZEND_ASSIGN_SUB: +| subsd xmm(dst_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0) +|| break; +|| case ZEND_MUL: +|| case ZEND_ASSIGN_MUL: +| mulsd xmm(dst_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0) +|| break; +|| case ZEND_DIV: +|| case ZEND_ASSIGN_DIV: +| divsd xmm(dst_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0) +|| break; +|| } |.endmacro -|.macro SSE_STORE, zv, reg -| movsd qword [zv], xmm(reg-ZREG_XMM0) +|.macro SSE_SET_ZVAL_DVAL, addr, reg +|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); +| movsd qword [Ra(Z_REG(addr))+Z_OFFSET(addr)], xmm(reg-ZREG_XMM0) |.endmacro -|.macro LONG_OP, long_ins, reg, op_type, op -||if (op_type == IS_CONST) { -| .if X64 -|| if (!IS_SIGNED_32BIT(Z_LVAL_P(RT_CONSTANT(op_array, op)))) { -| mov64 r1, Z_LVAL_P(RT_CONSTANT(op_array, op)) -| long_ins reg, r1 -|| } else { -| long_ins reg, Z_LVAL_P(RT_CONSTANT(op_array, op)) -|| } -| .else -| long_ins reg, Z_LVAL_P(RT_CONSTANT(op_array, op)) -| .endif -||} else { -| long_ins reg, aword [FP + op.var] -||} +|.macro LONG_OP, long_ins, reg, addr +|| if (Z_MODE(addr) == IS_CONST_ZVAL) { +| .if X64 +|| if (!IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(addr)))) { +| mov64 r1, Z_LVAL_P(Z_ZV(addr)) +| long_ins reg, r1 +|| } else { +| long_ins reg, Z_LVAL_P(Z_ZV(addr)) +|| } +| .else +| long_ins reg, Z_LVAL_P(Z_ZV(addr)) +| .endif +|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { +| long_ins reg, aword [Ra(Z_REG(addr))+Z_OFFSET(addr)] +|| } else { +|| ZEND_ASSERT(0); +|| } |.endmacro -|.macro LONG_LOAD, reg, op_type, op -||if (op_type == IS_CONST) { +|.macro LONG_OP_MEM_CONST, long_ins, op1_addr, lval +|| ZEND_ASSERT(Z_MODE(op1_addr) == IS_MEM_ZVAL); | .if X64 -|| if (!IS_SIGNED_32BIT(Z_LVAL_P(RT_CONSTANT(op_array, op)))) { -| mov64 reg, Z_LVAL_P(RT_CONSTANT(op_array, op)) -|| } else { -| mov reg, Z_LVAL_P(RT_CONSTANT(op_array, op)) -|| } +|| if (!IS_SIGNED_32BIT(lval)) { +| mov64 r0, lval +| long_ins aword [Ra(Z_REG(op1_addr))+Z_OFFSET(op1_addr)], r0 +|| } else { +| long_ins aword [Ra(Z_REG(op1_addr))+Z_OFFSET(op1_addr)], lval +|| } | .else -| mov reg, Z_LVAL_P(RT_CONSTANT(op_array, op)) +| long_ins aword [Ra(Z_REG(op1_addr))+Z_OFFSET(op1_addr)], lval | .endif -||} else { -| GET_Z_LVAL reg, FP + op.var -||} |.endmacro -|.macro LONG_MATH, opcode, reg, op_type, op -||switch (opcode) { -|| case ZEND_ADD: -|| case ZEND_ASSIGN_ADD: -| LONG_OP add, reg, op_type, op -|| break; -|| case ZEND_SUB: -|| case ZEND_ASSIGN_SUB: -| LONG_OP sub, reg, op_type, op -|| break; -|| case ZEND_MUL: -|| case ZEND_ASSIGN_MUL: -| LONG_OP imul, reg, op_type, op -|| break; -|| case ZEND_DIV: -|| case ZEND_ASSIGN_DIV: -| idiv aword [FP + op.var] // (reg == r0) -|| break; -||} +|.macro GET_ZVAL_LVAL, reg, addr +|| if (Z_MODE(addr) == IS_CONST_ZVAL) { +| .if X64 +|| if (!IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(addr)))) { +| mov64 reg, Z_LVAL_P(Z_ZV(addr)) +|| } else { +| mov reg, Z_LVAL_P(Z_ZV(addr)) +|| } +| .else +| mov reg, Z_LVAL_P(Z_ZV(addr)) +| .endif +|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { +| mov reg, aword [Ra(Z_REG(addr))+Z_OFFSET(addr)] +|| } else { +|| ZEND_ASSERT(0); +|| } |.endmacro -|.macro LONG_MATH2, opcode, dst_reg, src_reg -||switch (opcode) { -|| case ZEND_ADD: -|| case ZEND_ASSIGN_ADD: -| add dst_reg, src_reg -|| break; -|| case ZEND_SUB: -|| case ZEND_ASSIGN_SUB: -| sub dst_reg, src_reg -|| break; -|| case ZEND_MUL: -|| case ZEND_ASSIGN_MUL: -| imul dst_reg, src_reg -|| break; -|| case ZEND_DIV: -|| case ZEND_ASSIGN_DIV: -| idiv src_reg // (reg1 == r0) -|| break; -||} +|.macro LONG_MATH, opcode, reg, addr +|| switch (opcode) { +|| case ZEND_ADD: +|| case ZEND_ASSIGN_ADD: +| LONG_OP add, reg, addr +|| break; +|| case ZEND_SUB: +|| case ZEND_ASSIGN_SUB: +| LONG_OP sub, reg, addr +|| break; +|| case ZEND_MUL: +|| case ZEND_ASSIGN_MUL: +| LONG_OP imul, reg, addr +|| break; +|| case ZEND_DIV: +|| case ZEND_ASSIGN_DIV: +| idiv aword [Ra(Z_REG(addr))+Z_OFFSET(addr)] // (reg == r0) +|| break; +|| } |.endmacro -|.macro FPU_LONG_OP, fp_ins, op_type, op -||if (op_type == IS_CONST) { -| .if X64 -| mov r0, aword EX->literals -| fp_ins qword [r0 + op.constant] -| .else -| fp_ins dword [op.zv] -| .endif -||} else { -| fp_ins aword [FP + op.var] -||} +|.macro LONG_MATH_REG, opcode, dst_reg, src_reg +|| switch (opcode) { +|| case ZEND_ADD: +|| case ZEND_ASSIGN_ADD: +| add dst_reg, src_reg +|| break; +|| case ZEND_SUB: +|| case ZEND_ASSIGN_SUB: +| sub dst_reg, src_reg +|| break; +|| case ZEND_MUL: +|| case ZEND_ASSIGN_MUL: +| imul dst_reg, src_reg +|| break; +|| case ZEND_DIV: +|| case ZEND_ASSIGN_DIV: +| idiv src_reg // (reg1 == r0) +|| break; +|| } |.endmacro -|.macro FPU_LONG_LOAD, op_type, op -| FPU_LONG_OP fild, op_type, op +|.macro SET_ZVAL_LVAL, addr, lval +|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); +| mov aword [Ra(Z_REG(addr))+Z_OFFSET(addr)], lval |.endmacro -|.macro FPU_GET_Z_LVAL, zv -| fild aword [zv] +|.macro FPU_LONG_OP, fp_ins, addr +|| if (Z_MODE(addr) == IS_CONST_ZVAL) { +| .if X64 +|| if (IS_32BIT(Z_ZV(addr))) { +| fp_ins dword [Z_ZV(addr)] +|| } else { +| LOAD_ADDR r0, Z_ZV(addr) +| fp_ins qword [r0] +|| } +| .else +| fp_ins dword [Z_ZV(addr)] +| .endif +|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { +| fp_ins aword [Ra(Z_REG(addr))+Z_OFFSET(addr)] +|| } else { +|| ZEND_ASSERT(0); +|| } |.endmacro -|.macro FPU_GET_Z_DVAL, zv -| fld qword [zv] +|.macro FPU_GET_ZVAL_LVAL, addr +| FPU_LONG_OP fild, addr |.endmacro -|.macro FPU_MATH2, opcode, reg -||switch (opcode) { -|| case ZEND_ADD: -|| case ZEND_ASSIGN_ADD: -| fadd reg -|| break; -|| case ZEND_SUB: -|| case ZEND_ASSIGN_SUB: -| fsub reg -|| break; -|| case ZEND_MUL: -|| case ZEND_ASSIGN_MUL: -| fmul reg -|| break; -|| case ZEND_DIV: -|| case ZEND_ASSIGN_DIV: -| fdiv reg -|| break; -||} +|.macro FPU_MATH_REG, opcode, reg +|| switch (opcode) { +|| case ZEND_ADD: +|| case ZEND_ASSIGN_ADD: +| fadd reg +|| break; +|| case ZEND_SUB: +|| case ZEND_ASSIGN_SUB: +| fsub reg +|| break; +|| case ZEND_MUL: +|| case ZEND_ASSIGN_MUL: +| fmul reg +|| break; +|| case ZEND_DIV: +|| case ZEND_ASSIGN_DIV: +| fdiv reg +|| break; +|| } |.endmacro -|.macro ZVAL_COPY_CONST, dst, dst_info, zv, tmp_reg -||if (Z_TYPE_P(zv) > IS_TRUE) { -|| if (Z_TYPE_P(zv) == IS_DOUBLE) { -|.if X64 or SSE -|| if (Z_DVAL_P(zv) == 0.0 && !is_signed(Z_DVAL_P(zv))) { -| xorps xmm0, xmm0 -| .if X64 -|| } else if (!IS_32BIT(zv)) { -| mov64 tmp_reg, ((uintptr_t)zv) -| movsd xmm0, qword [tmp_reg] -| .endif -|| } else { -| movsd xmm0, qword [((uint32_t)(uintptr_t)zv)] -|| } -| movsd qword [dst], xmm0 -|.else -|| if (Z_DVAL_P(zv) == 0.0 && !is_signed(Z_DVAL_P(zv))) { -| fldz -|| } else if (Z_DVAL_P(zv) == 1.0) { -| fld1 -|| } else { -| fld qword [zv] -|| } -| fstp qword [dst] -|.endif -|| } else { -|.if X64 -|| if (!IS_SIGNED_32BIT(Z_LVAL_P(zv))) { -| mov64 tmp_reg, ((uintptr_t)Z_LVAL_P(zv)) -| SET_Z_LVAL dst, tmp_reg +|.macro ZVAL_COPY_CONST, dst_addr, dst_info, zv, tmp_reg +|| if (Z_TYPE_P(zv) > IS_TRUE) { +|| if (Z_TYPE_P(zv) == IS_DOUBLE) { +| .if X64 or SSE +|| if (Z_DVAL_P(zv) == 0.0 && !is_signed(Z_DVAL_P(zv))) { +| xorps xmm0, xmm0 +| .if X64 +|| } else if (!IS_32BIT(zv)) { +| mov64 tmp_reg, ((uintptr_t)zv) +| movsd xmm0, qword [tmp_reg] +| .endif +|| } else { +| movsd xmm0, qword [((uint32_t)(uintptr_t)zv)] +|| } +| SSE_SET_ZVAL_DVAL dst_addr, ZREG_XMM0 +| .else +|| if (Z_DVAL_P(zv) == 0.0 && !is_signed(Z_DVAL_P(zv))) { +| fldz +|| } else if (Z_DVAL_P(zv) == 1.0) { +| fld1 +|| } else { +| fld qword [zv] +|| } +| FPU_SET_ZVAL_DVAL dst_addr +| .endif || } else { -| SET_Z_LVAL dst, Z_LVAL_P(zv) +| .if X64 +|| if (!IS_SIGNED_32BIT(Z_LVAL_P(zv))) { +| mov64 tmp_reg, ((uintptr_t)Z_LVAL_P(zv)) +| SET_ZVAL_LVAL dst_addr, tmp_reg +|| } else { +| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv) +|| } +| .else +| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv) +| .endif || } -|.else -| SET_Z_LVAL dst, Z_LVAL_P(zv) -|.endif || } -||} -||if (((dst_info & MAY_BE_ANY) != (1< IS_TRUE) { -|| if (Z_TYPE_P(zv) == IS_DOUBLE) { -|.if X64 or SSE -|| if (Z_DVAL_P(zv) == 0.0 && !is_signed(Z_DVAL_P(zv))) { -| xorps xmm0, xmm0 -| .if X64 -|| } else if (!IS_32BIT(zv)) { -| mov64 tmp_reg, ((uintptr_t)zv) -| movsd xmm0, qword [tmp_reg] -| .endif -|| } else { -| movsd xmm0, qword [((uint32_t)(uintptr_t)zv)] -|| } -| movsd qword [dst], xmm0 -| movsd qword [dst2], xmm0 -|.else -|| if (Z_DVAL_P(zv) == 0.0 && !is_signed(Z_DVAL_P(zv))) { -| fldz -|| } else if (Z_DVAL_P(zv) == 1.0) { -| fld1 -|| } else { -| fld qword [zv] -|| } -| fstp qword [dst] -|| if (Z_DVAL_P(zv) == 0.0 && !is_signed(Z_DVAL_P(zv))) { -| fldz -|| } else if (Z_DVAL_P(zv) == 1.0) { -| fld1 -|| } else { -| fld qword [zv] -|| } -| fstp qword [dst2] -|.endif -|| } else { -|.if X64 -|| if (!IS_SIGNED_32BIT(Z_LVAL_P(zv))) { -| mov64 tmp_reg, ((uintptr_t)Z_LVAL_P(zv)) -| SET_Z_LVAL dst, tmp_reg -| SET_Z_LVAL dst2, tmp_reg +|.macro ZVAL_COPY_CONST_2, dst_addr, res_addr, dst_info, zv, tmp_reg +|| if (Z_TYPE_P(zv) > IS_TRUE) { +|| if (Z_TYPE_P(zv) == IS_DOUBLE) { +| .if X64 or SSE +|| if (Z_DVAL_P(zv) == 0.0 && !is_signed(Z_DVAL_P(zv))) { +| xorps xmm0, xmm0 +| .if X64 +|| } else if (!IS_32BIT(zv)) { +| mov64 tmp_reg, ((uintptr_t)zv) +| movsd xmm0, qword [tmp_reg] +| .endif +|| } else { +| movsd xmm0, qword [((uint32_t)(uintptr_t)zv)] +|| } +| SSE_SET_ZVAL_DVAL dst_addr, ZREG_XMM0 +| SSE_SET_ZVAL_DVAL res_addr, ZREG_XMM0 +| .else +|| if (Z_DVAL_P(zv) == 0.0 && !is_signed(Z_DVAL_P(zv))) { +| fldz +|| } else if (Z_DVAL_P(zv) == 1.0) { +| fld1 +|| } else { +| fld qword [zv] +|| } +| FPU_SET_ZVAL_DVAL dst_addr +|| if (Z_DVAL_P(zv) == 0.0 && !is_signed(Z_DVAL_P(zv))) { +| fldz +|| } else if (Z_DVAL_P(zv) == 1.0) { +| fld1 +|| } else { +| fld qword [zv] +|| } +| FPU_SET_ZVAL_DVAL res_addr +| .endif || } else { -| SET_Z_LVAL dst, Z_LVAL_P(zv) -| SET_Z_LVAL dst2, Z_LVAL_P(zv) +| .if X64 +|| if (!IS_SIGNED_32BIT(Z_LVAL_P(zv))) { +| mov64 tmp_reg, ((uintptr_t)Z_LVAL_P(zv)) +| SET_ZVAL_LVAL dst_addr, tmp_reg +| SET_ZVAL_LVAL res_addr, tmp_reg +|| } else { +| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv) +| SET_ZVAL_LVAL res_addr, Z_LVAL_P(zv) +|| } +| .else +| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv) +| SET_ZVAL_LVAL res_addr, Z_LVAL_P(zv) +| .endif || } -|.else -| SET_Z_LVAL dst, Z_LVAL_P(zv) -| SET_Z_LVAL dst2, Z_LVAL_P(zv) -|.endif || } -||} -||if (((dst_info & MAY_BE_ANY) != (1<1 -|| } -| GC_ADDREF value_ptr_reg +|| if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { +|| if (val_info & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { +| IF_NOT_REFCOUNTED type_flags_reg, >1 +|| } +| GC_ADDREF value_ptr_reg |1: -||} +|| } |.endmacro |.macro TRY_ADDREF_2, val_info, type_flags_reg, value_ptr_reg -||if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { -|| if (val_info & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { -| IF_NOT_REFCOUNTED type_flags_reg, >1 -|| } -| add dword [value_ptr_reg], 2 +|| if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { +|| if (val_info & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { +| IF_NOT_REFCOUNTED type_flags_reg, >1 +|| } +| add dword [value_ptr_reg], 2 |1: -||} +|| } |.endmacro // zval should be in FCARG1a @@ -822,30 +837,30 @@ static void* dasm_labels[zend_lb_MAX]; || } else { | mov FCARG2a, 0 || } -|.if X64 -| mov CARG3d, lineno -|.else -| push lineno -|.endif +| .if X64 +| mov CARG3d, lineno +| .else +| push lineno +| .endif || } -| EXT_CALL _zval_copy_ctor_func, r0 +| EXT_CALL _zval_copy_ctor_func, r0 |.endmacro // zval should be in FCARG1a |.macro ZVAL_COPY_CTOR, val_info, type_flags_reg, value_ptr_reg, filename, lineno -||if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { -|| if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY)) { -| IF_NOT_FLAGS type_flags_reg, IS_TYPE_REFCOUNTED + IS_TYPE_COPYABLE, >2 -| IF_NOT_FLAGS type_flags_reg, IS_TYPE_COPYABLE, >1 -| GC_ADDREF value_ptr_reg -| jmp >2 +|| if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { +|| if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY)) { +| IF_NOT_FLAGS type_flags_reg, IS_TYPE_REFCOUNTED + IS_TYPE_COPYABLE, >2 +| IF_NOT_FLAGS type_flags_reg, IS_TYPE_COPYABLE, >1 +| GC_ADDREF value_ptr_reg +| jmp >2 |1: -| ZVAL_COPY_CTOR_FUNC filename, lineno +| ZVAL_COPY_CTOR_FUNC filename, lineno |2: -|| } else { -| TRY_ADDREF val_info, type_flags_reg, value_ptr_reg +|| } else { +| TRY_ADDREF val_info, type_flags_reg, value_ptr_reg +|| } || } -||} |.endmacro |.macro ZVAL_DEREF, reg, info @@ -858,85 +873,85 @@ static void* dasm_labels[zend_lb_MAX]; |.endmacro |.macro SAVE_VALID_OPLINE, op -|| if (op == last_valid_opline) { +|| if (op == last_valid_opline) { | mov aword EX->opline, IP -|| } else { -|.if X64 -|| if (IS_32BIT(op)) { -| mov aword EX->opline, ((ptrdiff_t)op) -|| } else { -| mov64 r0, ((ptrdiff_t)op) -| mov aword EX->opline, r0 -|| } -|.else -| mov aword EX->opline, op -|.endif +|| } else { +| .if X64 +|| if (IS_32BIT(op)) { +| mov aword EX->opline, ((ptrdiff_t)op) +|| } else { +| mov64 r0, ((ptrdiff_t)op) +| mov aword EX->opline, r0 +|| } +| .else +| mov aword EX->opline, op +| .endif || } |.endmacro // zval should be in FCARG1a |.macro ZVAL_DTOR_FUNC, var_info, filename, opline // arg1 must be in FCARG1a -||do { -|| if (opline) { -| SAVE_VALID_OPLINE opline -|| } -|| if (has_concrete_type((var_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { -|| zend_uchar type = concrete_type((var_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)); -|| if (type == IS_STRING && !ZEND_DEBUG) { -| test byte [FCARG1a + 5], IS_STR_PERSISTENT -| jnz >1 -| EXT_CALL _efree, r0 -| jmp >2 +|| do { +|| if (opline) { +| SAVE_VALID_OPLINE opline +|| } +|| if (has_concrete_type((var_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { +|| zend_uchar type = concrete_type((var_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)); +|| if (type == IS_STRING && !ZEND_DEBUG) { +| test byte [FCARG1a + 5], IS_STR_PERSISTENT +| jnz >1 +| EXT_CALL _efree, r0 +| jmp >2 |1: -| EXT_CALL free, r0 +| EXT_CALL free, r0 |2: -|| break; -|| } else if (type == IS_ARRAY) { -| EXT_CALL zend_array_destroy, r0 -|| break; -|| } else if (type == IS_OBJECT) { -|.if X64 -| EXT_CALL zend_objects_store_del, r0 -|.else -| sub r4, 12 -| push FCARG1a -| EXT_CALL zend_objects_store_del, r0 -| add r4, 16 -|.endif -|| break; +|| break; +|| } else if (type == IS_ARRAY) { +| EXT_CALL zend_array_destroy, r0 +|| break; +|| } else if (type == IS_OBJECT) { +| .if X64 +| EXT_CALL zend_objects_store_del, r0 +| .else +| sub r4, 12 +| push FCARG1a +| EXT_CALL zend_objects_store_del, r0 +| add r4, 16 +| .endif +|| break; +|| } || } -|| } -|| if (ZEND_DEBUG) { -|| uint32_t lineno = opline ? ((zend_op*)opline)->lineno : 0; -|| if (filename) { -| LOAD_ADDR FCARG2a, ZSTR_VAL((zend_string*)filename) -|| } else { -| mov FCARG2a, 0 +|| if (ZEND_DEBUG) { +|| uint32_t lineno = opline ? ((zend_op*)opline)->lineno : 0; +|| if (filename) { +| LOAD_ADDR FCARG2a, ZSTR_VAL((zend_string*)filename) +|| } else { +| mov FCARG2a, 0 +|| } +| .if X64 +| mov CARG3d, lineno +| .else +| push lineno +| .endif || } -|.if X64 -| mov CARG3d, lineno -|.else -| push lineno -|.endif -|| } -| EXT_CALL _zval_dtor_func, r0 -||} while(0); +| EXT_CALL _zval_dtor_func, r0 +|| } while(0); |.endmacro -|.macro ZVAL_PTR_DTOR, zv, op_info, gc, cold, safe, filename, opline +|.macro ZVAL_PTR_DTOR, addr, op_info, gc, cold, safe, filename, opline || if ((op_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { || if ((op_info) & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { | // if (Z_REFCOUNTED_P(cv)) { || if (cold) { -| IF_Z_REFCOUNTED zv, >1 +| IF_ZVAL_REFCOUNTED addr, >1 |.cold_code |1: || } else { -| IF_NOT_Z_REFCOUNTED zv, >4 +| IF_NOT_ZVAL_REFCOUNTED addr, >4 || } || } | // if (!Z_DELREF_P(cv)) { -| GET_Z_PTR FCARG1a, zv +| GET_ZVAL_PTR FCARG1a, addr | GC_DELREF FCARG1a || if (RC_MAY_BE_1(op_info)) { || if (RC_MAY_BE_N(op_info)) { @@ -948,7 +963,7 @@ static void* dasm_labels[zend_lb_MAX]; || } || if (safe) { | // ZVAL_NULL(cv); -| SET_Z_TYPE_INFO zv, IS_NULL +| SET_ZVAL_TYPE_INFO addr, IS_NULL || } | // zval_dtor_func(r); | ZVAL_DTOR_FUNC op_info, filename, opline @@ -959,12 +974,10 @@ static void* dasm_labels[zend_lb_MAX]; || } || if (gc && RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT))) { || if ((op_info) & MAY_BE_REF) { -| GET_Z_PTR FCARG1a, zv -| cmp byte [FCARG1a + 4], IS_REFERENCE -| jnz >1 -| GET_Z_PTR FCARG1a, FCARG1a + offsetof(zend_reference, val) -| IF_NOT_Z_REFCOUNTED zv, >4 -| GET_Z_PTR FCARG1a, zv +|| zend_jit_addr ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, offsetof(zend_reference, val)); +| IF_NOT_ZVAL_TYPE addr, IS_REFERENCE, >1 +| IF_NOT_ZVAL_REFCOUNTED ref_addr, >4 +| GET_ZVAL_PTR FCARG1a, ref_addr |1: || } | IF_GC_MAY_NOT_LEAK FCARG1a, eax, >4 @@ -983,58 +996,37 @@ static void* dasm_labels[zend_lb_MAX]; |.endmacro |.macro FREE_OP, op_type, op, op_info, cold, op_array, opline -||if (op_type & (IS_VAR|IS_TMP_VAR)) { -| ZVAL_PTR_DTOR FP + op.var, op_info, 0, cold, 0, op_array->filename, opline -||} +|| if (op_type & (IS_VAR|IS_TMP_VAR)) { +| ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_FP, op.var), op_info, 0, cold, 0, op_array->filename, opline +|| } |.endmacro -|.macro SEPARATE_ZVAL_NOREF, zv, op_info, cold, filename, lineno +|.macro SEPARATE_ZVAL_NOREF, addr, op_info, cold, filename, lineno || if ((op_info & (MAY_BE_STRING|MAY_BE_ARRAY)) && RC_MAY_BE_N(op_info)) { || if (cold) { -| IF_Z_FLAGS zv, IS_TYPE_COPYABLE, >1 +| IF_ZVAL_FLAGS addr, IS_TYPE_COPYABLE, >1 |.cold_code |1: || } else { -| IF_NOT_Z_FLAGS zv, IS_TYPE_COPYABLE, >2 +| IF_NOT_ZVAL_FLAGS addr, IS_TYPE_COPYABLE, >2 || } -| GET_Z_PTR r0, zv +| GET_ZVAL_PTR r0, addr || if (RC_MAY_BE_1(op_info)) { | cmp dword [r0], 1 // if (GC_REFCOUNTED() > 1) | jbe >2 || } -| IF_NOT_Z_FLAGS zv, IS_TYPE_REFCOUNTED, >1 +| IF_NOT_ZVAL_FLAGS addr, IS_TYPE_REFCOUNTED, >1 | GC_DELREF r0 |1: -| lea FCARG1a, [zv] -| ZVAL_COPY_CTOR_FUNC filename, lineno -|| if (cold) { -| jmp >2 -|.code -|| } -|2: -|| } -|.endmacro - -|.macro SEPARATE_ZVAL_NOREF_REG, op_info, cold, filename, lineno -|| if ((op_info & (MAY_BE_STRING|MAY_BE_ARRAY)) && RC_MAY_BE_N(op_info)) { -|| if (cold) { -| IF_Z_FLAGS FCARG1a, IS_TYPE_COPYABLE, >1 -|.cold_code -|1: +|| if (Z_REG(addr) == ZREG_FCARG1a) { +| mov aword [r4], FCARG1a // save || } else { -| IF_NOT_Z_FLAGS FCARG1a, IS_TYPE_COPYABLE, >2 +| LOAD_ZVAL_ADDR FCARG1a, addr || } -| GET_Z_PTR r0, FCARG1a -|| if (RC_MAY_BE_1(op_info)) { -| cmp dword [r0], 1 // if (GC_REFCOUNTED() > 1) -| jbe >2 -|| } -| IF_NOT_Z_FLAGS FCARG1a, IS_TYPE_REFCOUNTED, >1 -| GC_DELREF r0 -|1: -| mov aword [r4], FCARG1a // save | ZVAL_COPY_CTOR_FUNC filename, lineno -| mov FCARG1a, aword [r4] // restore +|| if (Z_REG(addr) == ZREG_FCARG1a) { +| mov FCARG1a, aword [r4] // restore +|| } || if (cold) { | jmp >2 |.code @@ -1043,11 +1035,14 @@ static void* dasm_labels[zend_lb_MAX]; || } |.endmacro -|.macro SEPARATE_ARRAY, op_type, op, op_info, cold, filename, lineno -| LONG_LOAD FCARG1a, op_type, op +|.macro SEPARATE_ARRAY, addr, op_info, cold, filename, lineno || if (RC_MAY_BE_N(op_info)) { +|| zend_reg tmp_reg; +|| +|| tmp_reg = (Z_REG(addr) == ZREG_FCARG1a) ? ZREG_R0 : ZREG_FCARG1a; +| GET_ZVAL_LVAL Ra(tmp_reg), addr || if (RC_MAY_BE_1(op_info)) { -| cmp dword [FCARG1a], 1 // if (GC_REFCOUNTED() > 1) +| cmp dword [Ra(tmp_reg)], 1 // if (GC_REFCOUNTED() > 1) || if (cold) { | ja >1 |.cold_code @@ -1056,68 +1051,42 @@ static void* dasm_labels[zend_lb_MAX]; | jbe >2 || } || } -| IF_NOT_Z_FLAGS FP + op.var, IS_TYPE_REFCOUNTED, >1 -| GC_DELREF FCARG1a +| IF_NOT_ZVAL_FLAGS addr, IS_TYPE_REFCOUNTED, >1 +| GC_DELREF Ra(tmp_reg) |1: -| LOAD_ZVAL_ADDR FCARG1a, op_type, op -| ZVAL_COPY_CTOR_FUNC filename, lineno -| LONG_LOAD FCARG1a, op_type, op -|| if (RC_MAY_BE_1(op_info)) { -|| if (cold) { -| jmp >2 -|.code -|| } -|| } -|2: -|| } -|.endmacro - -|.macro SEPARATE_ARRAY_REG, op_info, cold, filename, lineno -|| if (RC_MAY_BE_N(op_info)) { -| GET_Z_PTR r0, FCARG1a -|| if (RC_MAY_BE_1(op_info)) { -| cmp dword [r0], 1 // if (GC_REFCOUNTED() > 1) -|| if (cold) { -| ja >1 -|.cold_code -|1: -|| } else { -| jbe >2 -|| } +|| if (Z_REG(addr) == ZREG_FCARG1a) { +| mov aword [r4], FCARG1a // save +|| } else { +| LOAD_ZVAL_ADDR FCARG1a, addr || } -| IF_NOT_Z_FLAGS FCARG1a, IS_TYPE_REFCOUNTED, >1 -| GC_DELREF r0 -|1: -| mov aword [r4], FCARG1a // save | ZVAL_COPY_CTOR_FUNC filename, lineno -| mov FCARG1a, aword [r4] // restore -| GET_Z_PTR FCARG1a, FCARG1a +|| if (Z_REG(addr) == ZREG_FCARG1a) { +| mov FCARG1a, aword [r4] // restore +|| } || if (RC_MAY_BE_1(op_info)) { || if (cold) { | jmp >2 |.code || } || } -| mov FCARG1a, r0 |2: -|| } else { -| GET_Z_PTR FCARG1a, FCARG1a || } +| GET_ZVAL_LVAL FCARG1a, addr |.endmacro |.macro EFREE_REG_24, op_array, opline ||#if ZEND_DEBUG || const char *filename = op_array->filename ? op_array->filename->val : NULL; | LOAD_ADDR FCARG2a, filename -|.if X64 -| mov CARG3d, opline->lineno -| xor CARG4, CARG4 -| xor CARG5, CARG5 -|.else -| push 0 -| push 0 -| push opline->lineno -|.endif +| .if X64 +| mov CARG3d, opline->lineno +| xor CARG4, CARG4 +| xor CARG5, CARG5 +| .else +| push 0 +| push 0 +| push opline->lineno +| .endif | EXT_CALL _efree, r0 ||#else | EXT_CALL _efree_24, r0 @@ -1134,15 +1103,15 @@ static void* dasm_labels[zend_lb_MAX]; || const char *filename = op_array->filename ? op_array->filename->val : NULL; | mov FCARG1a, size | LOAD_ADDR FCARG2a, filename -|.if X64 -| mov CARG3d, opline->lineno -| xor CARG4, CARG4 -| xor CARG5, CARG5 -|.else -| push 0 -| push 0 -| push opline->lineno -|.endif +| .if X64 +| mov CARG3d, opline->lineno +| xor CARG4, CARG4 +| xor CARG5, CARG5 +| .else +| push 0 +| push 0 +| push opline->lineno +| .endif | EXT_CALL _emalloc, r0 ||#else || if (size == 24) { @@ -1158,15 +1127,15 @@ static void* dasm_labels[zend_lb_MAX]; | GC_DELREF reg | jne >1 | // zend_objects_store_del(obj); -|.if X64 -| mov CARG1, reg -| EXT_CALL zend_objects_store_del, r0 -|.else -| sub r4, 12 -| push reg -| EXT_CALL zend_objects_store_del, r0 -| add r4, 16 -|.endif +| .if X64 +| mov CARG1, reg +| EXT_CALL zend_objects_store_del, r0 +| .else +| sub r4, 12 +| push reg +| EXT_CALL zend_objects_store_del, r0 +| add r4, 16 +| .endif | jmp exit_label |1: | IF_GC_MAY_NOT_LEAK reg, tmp_reg, >1 @@ -1177,30 +1146,30 @@ static void* dasm_labels[zend_lb_MAX]; |.endmacro |.macro UNDEFINED_OFFSET, opline -|| if (opline == last_valid_opline) { -| call ->undefined_offset_ex -|| } else { -| SAVE_VALID_OPLINE, opline -| call ->undefined_offset -|| } +|| if (opline == last_valid_opline) { +| call ->undefined_offset_ex +|| } else { +| SAVE_VALID_OPLINE, opline +| call ->undefined_offset +|| } |.endmacro |.macro UNDEFINED_INDEX, opline -|| if (opline == last_valid_opline) { -| call ->undefined_index_ex -|| } else { -| SAVE_VALID_OPLINE, opline -| call ->undefined_index -|| } +|| if (opline == last_valid_opline) { +| call ->undefined_index_ex +|| } else { +| SAVE_VALID_OPLINE, opline +| call ->undefined_index +|| } |.endmacro |.macro CANNOT_ADD_ELEMENT, opline -|| if (opline == last_valid_opline) { -| call ->cannot_add_element_ex -|| } else { -| SAVE_VALID_OPLINE, opline -| call ->cannot_add_element -|| } +|| if (opline == last_valid_opline) { +| call ->cannot_add_element_ex +|| } else { +| SAVE_VALID_OPLINE, opline +| call ->cannot_add_element +|| } |.endmacro static zend_bool reuse_ip; @@ -1266,9 +1235,9 @@ static int zend_jit_interrupt_handler_stub(dasm_State **Dst) | je >1 | //zend_timeout(0); |.if X64 - | xor CARG1d, CARG1d + | xor CARG1d, CARG1d |.else - | push 0 + | push 0 |.endif | EXT_CALL zend_timeout, r0 |1: @@ -1278,13 +1247,13 @@ static int zend_jit_interrupt_handler_stub(dasm_State **Dst) | mov EX->opline, IP | //zend_interrupt_function(execute_data); |.if X64 - | mov CARG1, FP - | EXT_CALL zend_interrupt_function, r0 + | mov CARG1, FP + | EXT_CALL zend_interrupt_function, r0 |.else - | sub r4, 12 - | push FP - | EXT_CALL zend_interrupt_function, r0 - | add r4, 16 + | sub r4, 12 + | push FP + | EXT_CALL zend_interrupt_function, r0 + | add r4, 16 |.endif | //ZEND_VM_ENTER(); | //execute_data = EG(current_execute_data); @@ -1353,13 +1322,13 @@ static int zend_jit_icall_throw_stub(dasm_State **Dst) |->icall_throw_handler: | // zend_throw_exception_internal(NULL); |.if X64 - | xor CARG1, CARG1 - | EXT_CALL zend_throw_exception_internal, r0 + | xor CARG1, CARG1 + | EXT_CALL zend_throw_exception_internal, r0 |.else - | sub r4, 12 - | push 0 - | EXT_CALL zend_throw_exception_internal, r0 - | add r4, 16 + | sub r4, 12 + | push 0 + | EXT_CALL zend_throw_exception_internal, r0 + | add r4, 16 |.endif | // HANDLE_EXCEPTION() | jmp ->exception_handler @@ -1372,35 +1341,35 @@ static int zend_jit_throw_cannot_pass_by_ref_stub(dasm_State **Dst) |->throw_cannot_pass_by_ref: | mov r0, EX->opline |.if X64 - | movsxd r1, dword OP:r0->result.var + | movsxd r1, dword OP:r0->result.var |.else - | mov r1, OP:r0->result.var + | mov r1, OP:r0->result.var |.endif | SET_Z_TYPE_INFO RX+r1, IS_UNDEF | mov RX, r0 |.if X64 - | xor CARG1, CARG1 - | LOAD_ADDR CARG2, "Cannot pass parameter %d by reference" - | mov CARG3d, dword OP:r0->op2.num - | EXT_CALL zend_throw_error, r0 + | xor CARG1, CARG1 + | LOAD_ADDR CARG2, "Cannot pass parameter %d by reference" + | mov CARG3d, dword OP:r0->op2.num + | EXT_CALL zend_throw_error, r0 |.else - | mov r1, dword OP:r0->op2.num - | sub r4, 4 - | push r1 - | push "Cannot pass parameter %d by reference" - | push 0 - | EXT_CALL zend_throw_error, r0 - | add r4, 16 + | mov r1, dword OP:r0->op2.num + | sub r4, 4 + | push r1 + | push "Cannot pass parameter %d by reference" + | push 0 + | EXT_CALL zend_throw_error, r0 + | add r4, 16 |.endif | cmp byte OP:RX->op1_type, IS_TMP_VAR | jne >9 |.if X64 - | movsxd r0, dword OP:RX->op1.var + | movsxd r0, dword OP:RX->op1.var |.else - | mov r0, OP:RX->op1.var + | mov r0, OP:RX->op1.var |.endif | add r0, FP - | ZVAL_PTR_DTOR r0, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, 0, 0, NULL, NULL + | ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0), MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, 0, 0, NULL, NULL |9: | jmp ->exception_handler @@ -1420,47 +1389,47 @@ static int zend_jit_undefined_offset_stub(dasm_State **Dst) { |->undefined_offset: |.if X64 - | sub r4, 8 + | sub r4, 8 |.else - | sub r4, 12 + | sub r4, 12 |.endif | mov r0, EX->opline |.if X64 - | movsxd r1, dword OP:r0->result.var + | movsxd r1, dword OP:r0->result.var |.else - | mov r1, OP:r0->result.var + | mov r1, OP:r0->result.var |.endif | cmp byte OP:r0->op2_type, IS_CONST | SET_Z_TYPE_INFO FP + r1, IS_NULL | jne >2 |.if X64 - | movsxd r0, dword OP:r0->op2.constant - | add r0, aword EX->literals + | movsxd r0, dword OP:r0->op2.constant + | add r0, aword EX->literals |.else - | mov r0, aword OP:r0->op2.zv + | mov r0, aword OP:r0->op2.zv |.endif | jmp >3 |2: |.if X64 - | movsxd r0, dword OP:r0->op2.var + | movsxd r0, dword OP:r0->op2.var |.else - | mov r0, OP:r0->op2.var + | mov r0, OP:r0->op2.var |.endif | add r0, FP |3: |.if X64 - | mov CARG1, E_NOTICE - | LOAD_ADDR CARG2, "Undefined offset: " ZEND_LONG_FMT - | mov CARG3, aword [r0] - | EXT_CALL zend_error, r0 - | add r4, 8 // stack alignment + | mov CARG1, E_NOTICE + | LOAD_ADDR CARG2, "Undefined offset: " ZEND_LONG_FMT + | mov CARG3, aword [r0] + | EXT_CALL zend_error, r0 + | add r4, 8 // stack alignment |.else - | sub r4, 4 - | push aword [r0] - | push "Undefined offset: " ZEND_LONG_FMT - | push E_NOTICE - | EXT_CALL zend_error, r0 - | add r4, 28 + | sub r4, 4 + | push aword [r0] + | push "Undefined offset: " ZEND_LONG_FMT + | push E_NOTICE + | EXT_CALL zend_error, r0 + | add r4, 28 |.endif | ret @@ -1480,50 +1449,50 @@ static int zend_jit_undefined_index_stub(dasm_State **Dst) { |->undefined_index: |.if X64 - | sub r4, 8 + | sub r4, 8 |.else - | sub r4, 12 + | sub r4, 12 |.endif | mov r0, EX->opline |.if X64 - | movsxd r1, dword OP:r0->result.var + | movsxd r1, dword OP:r0->result.var |.else - | mov r1, OP:r0->result.var + | mov r1, OP:r0->result.var |.endif | cmp byte OP:r0->op2_type, IS_CONST | SET_Z_TYPE_INFO FP + r1, IS_NULL | jne >2 |.if X64 - | movsxd r0, dword OP:r0->op2.constant - | add r0, aword EX->literals + | movsxd r0, dword OP:r0->op2.constant + | add r0, aword EX->literals |.else - | mov r0, aword OP:r0->op2.zv + | mov r0, aword OP:r0->op2.zv |.endif | jmp >3 |2: |.if X64 - | movsxd r0, dword OP:r0->op2.var + | movsxd r0, dword OP:r0->op2.var |.else - | mov r0, OP:r0->op2.var + | mov r0, OP:r0->op2.var |.endif | add r0, FP |3: |.if X64 - | mov CARG1, E_NOTICE - | LOAD_ADDR CARG2, "Undefined index: %s" - | mov CARG3, aword [r0] - | add CARG3, offsetof(zend_string, val) - | EXT_CALL zend_error, r0 - | add r4, 8 + | mov CARG1, E_NOTICE + | LOAD_ADDR CARG2, "Undefined index: %s" + | mov CARG3, aword [r0] + | add CARG3, offsetof(zend_string, val) + | EXT_CALL zend_error, r0 + | add r4, 8 |.else - | sub r4, 4 - | mov r0, aword [r0] - | add r0, offsetof(zend_string, val) - | push r0 - | push "Undefined index: %s" - | push E_NOTICE - | EXT_CALL zend_error, r0 - | add r4, 28 + | sub r4, 4 + | mov r0, aword [r0] + | add r0, offsetof(zend_string, val) + | push r0 + | push "Undefined index: %s" + | push E_NOTICE + | EXT_CALL zend_error, r0 + | add r4, 28 |.endif | ret @@ -1543,31 +1512,31 @@ static int zend_jit_cannot_add_element_stub(dasm_State **Dst) { |->cannot_add_element: |.if X64 - | sub r4, 8 + | sub r4, 8 |.else - | sub r4, 12 + | sub r4, 12 |.endif | mov r0, EX->opline | cmp byte OP:r0->result_type, IS_UNUSED | jz >1 |.if X64 - | movsxd r0, dword OP:r0->result.var + | movsxd r0, dword OP:r0->result.var |.else - | mov r0, OP:r0->result.var + | mov r0, OP:r0->result.var |.endif | SET_Z_TYPE_INFO FP + r0, IS_NULL |1: |.if X64 - | mov CARG1, E_WARNING - | LOAD_ADDR CARG2, "Cannot add element to the array as the next element is already occupied" - | EXT_CALL zend_error, r0 - | add r4, 8 + | mov CARG1, E_WARNING + | LOAD_ADDR CARG2, "Cannot add element to the array as the next element is already occupied" + | EXT_CALL zend_error, r0 + | add r4, 8 |.else - | sub r4, 8 - | push "Cannot add element to the array as the next element is already occupied" - | push E_WARNING - | EXT_CALL zend_error, r0 - | add r4, 28 + | sub r4, 8 + | push "Cannot add element to the array as the next element is already occupied" + | push E_WARNING + | EXT_CALL zend_error, r0 + | add r4, 28 |.endif | ret @@ -1579,15 +1548,15 @@ static int zend_jit_not_obj_stub(dasm_State **Dst) |->not_obj: |8: |.if X64 - | xor CARG1, CARG1 - | LOAD_ADDR CARG2, "Using $this when not in object context" - | EXT_CALL zend_throw_error, r0 + | xor CARG1, CARG1 + | LOAD_ADDR CARG2, "Using $this when not in object context" + | EXT_CALL zend_throw_error, r0 |.else - | sub r4, 8 - | push "Using $this when not in object context" - | push 0 - | EXT_CALL zend_throw_error, r0 - | add r4, 16 + | sub r4, 8 + | push "Using $this when not in object context" + | push 0 + | EXT_CALL zend_throw_error, r0 + | add r4, 16 |.endif | jmp ->exception_handler return 1; @@ -1869,113 +1838,123 @@ static int zend_jit_new(dasm_State **Dst, const zend_op *opline, int *opnum, zen static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) { uint32_t op1_info, op1_def_info; + zend_jit_addr op1_addr, res_addr; op1_info = OP1_INFO(); if (opline->op1_type != IS_CV || !(op1_info & MAY_BE_LONG)) { goto fallback; } + op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1); + if (opline->result_type != IS_UNUSED) { + res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result); + } + if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)-MAY_BE_LONG)) { - | IF_NOT_Z_TYPE FP + opline->op1.var, IS_LONG, >2 + | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >2 + } + if ((opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) && + opline->result_type != IS_UNUSED) { + | ZVAL_COPY_VALUE res_addr, op1_addr, MAY_BE_LONG, ZREG_R0, ZREG_R1 } - || if ((opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) && - || opline->result_type != IS_UNUSED) { - | ZVAL_COPY_VALUE FP + opline->result.var, FP + opline->op1.var, MAY_BE_LONG, r0, eax, r1 - || } if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { - | add aword [FP + opline->op1.var], 1 + | LONG_OP_MEM_CONST add, op1_addr, Z_L(1) } else { - | sub aword [FP + opline->op1.var], 1 + | LONG_OP_MEM_CONST sub, op1_addr, Z_L(1) } op1_def_info = OP1_DEF_INFO(); if ((op1_def_info & MAY_BE_DOUBLE) && zend_may_overflow(opline, op_array, ssa)) { | jo >1 - || if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && - || opline->result_type != IS_UNUSED) { - | ZVAL_COPY_VALUE FP + opline->result.var, FP + opline->op1.var, MAY_BE_LONG, r0, eax, r1 - || } + if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && + opline->result_type != IS_UNUSED) { + | ZVAL_COPY_VALUE res_addr, op1_addr, MAY_BE_LONG, ZREG_R0, ZREG_R1 + } |.cold_code |1: if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { |.if X64 - | mov64 rax, 0x43e0000000000000 - | SET_Z_LVAL FP + opline->op1.var, rax + | mov64 rax, 0x43e0000000000000 + | SET_ZVAL_LVAL op1_addr, rax |.else - | SET_Z_LVAL FP + opline->op1.var, 0 - | SET_Z_W2 FP + opline->op1.var, 0x41e00000 + | SET_ZVAL_LVAL op1_addr, 0 + | SET_ZVAL_W2 op1_addr, 0x41e00000 |.endif } else { |.if X64 - | mov64 rax, 0xc3e0000000000000 - | SET_Z_LVAL FP + opline->op1.var, rax + | mov64 rax, 0xc3e0000000000000 + | SET_ZVAL_LVAL op1_addr, rax |.else - | SET_Z_LVAL FP + opline->op1.var, 0x00200000 - | SET_Z_W2 FP + opline->op1.var, 0xc1e00000 + | SET_ZVAL_LVAL op1_addr, 0x00200000 + | SET_ZVAL_W2 op1_addr, 0xc1e00000 |.endif } - | SET_Z_TYPE_INFO FP + opline->op1.var, IS_DOUBLE - || if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && - || opline->result_type != IS_UNUSED) { - | ZVAL_COPY_VALUE FP + opline->result.var, FP + opline->op1.var, MAY_BE_DOUBLE, r0, eax, r1 - || } + | SET_ZVAL_TYPE_INFO op1_addr, IS_DOUBLE + if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && + opline->result_type != IS_UNUSED) { + | ZVAL_COPY_VALUE res_addr, op1_addr, MAY_BE_DOUBLE, ZREG_R0, ZREG_R1 + } | jmp >3 |.code } else { - || if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && - || opline->result_type != IS_UNUSED) { - | ZVAL_COPY_VALUE FP + opline->result.var, FP + opline->op1.var, MAY_BE_LONG, r0, eax, r1 - || } + if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && + opline->result_type != IS_UNUSED) { + | ZVAL_COPY_VALUE res_addr, op1_addr, MAY_BE_LONG, ZREG_R0, ZREG_R1 + } } if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { |.cold_code |2: | SAVE_VALID_OPLINE opline - || if (op1_info & MAY_BE_UNDEF) { - | IF_NOT_Z_TYPE FP + opline->op1.var, IS_UNDEF, >2 - | // zend_error(E_NOTICE, "Undefined variable: %s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); - | mov FCARG1d, opline->op1.var - | EXT_CALL zend_jit_undefined_op_helper, r0 - | SET_Z_TYPE_INFO FP + opline->op1.var, IS_NULL - || op1_info |= MAY_BE_NULL; - || } + if (op1_info & MAY_BE_UNDEF) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >2 + | // zend_error(E_NOTICE, "Undefined variable: %s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); + | mov FCARG1d, opline->op1.var + | EXT_CALL zend_jit_undefined_op_helper, r0 + | SET_ZVAL_TYPE_INFO op1_addr, IS_NULL + op1_info |= MAY_BE_NULL; + } |2: - | lea FCARG1a, [FP + opline->op1.var] + | LOAD_ZVAL_ADDR FCARG1a, op1_addr | // ZVAL_DEREF(var_ptr); | ZVAL_DEREF FCARG1a, op1_info - || if ((opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC)) { - || if (opline->result_type != IS_UNUSED) { - | ZVAL_COPY_VALUE FP + opline->result.var, FCARG1a, op1_info, r0, eax, r2 - | //ZVAL_COPY_CTOR op1_info, ah, r2, op_array->filename, opline->lineno - || if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { - || if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY)) { - | IF_NOT_FLAGS ah, IS_TYPE_REFCOUNTED + IS_TYPE_COPYABLE, >2 - | IF_FLAGS ah, IS_TYPE_COPYABLE, >1 - | GC_ADDREF r2 - | jmp >2 - |1: - | mov aword [r4], FCARG1a // save - | ZVAL_COPY_CTOR_FUNC op_array->filename, opline->lineno - | mov FCARG1a, aword [r4] // restore - |2: - || } else { - | TRY_ADDREF op1_info, ah, r2 - || } - || } - || } - || } else { - | SEPARATE_ZVAL_NOREF_REG op1_info, 0, op_array->filename, opline->lineno - || } - || if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { - | EXT_CALL increment_function, r0 - || } else { - | EXT_CALL decrement_function, r0 - || } - || if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && - || opline->result_type != IS_UNUSED) { - | ZVAL_COPY_VALUE FP + opline->result.var, FP + opline->op1.var, op1_def_info, r0, eax, r1 - | TRY_ADDREF op1_def_info, ah, r1 - || } + if ((opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC)) { + if (opline->result_type != IS_UNUSED) { + zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); + + | ZVAL_COPY_VALUE res_addr, val_addr, op1_info, ZREG_R0, ZREG_R2 + | //ZVAL_COPY_CTOR op1_info, ah, r2, op_array->filename, opline->lineno + if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { + if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY)) { + | IF_NOT_FLAGS ah, IS_TYPE_REFCOUNTED + IS_TYPE_COPYABLE, >2 + | IF_FLAGS ah, IS_TYPE_COPYABLE, >1 + | GC_ADDREF r2 + | jmp >2 + |1: + | mov aword [r4], FCARG1a // save + | ZVAL_COPY_CTOR_FUNC op_array->filename, opline->lineno + | mov FCARG1a, aword [r4] // restore + |2: + } else { + | TRY_ADDREF op1_info, ah, r2 + } + } + } + } else { + zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); + + | SEPARATE_ZVAL_NOREF var_addr, op1_info, 0, op_array->filename, opline->lineno + } + if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { + | EXT_CALL increment_function, r0 + } else { + | EXT_CALL decrement_function, r0 + } + if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && + opline->result_type != IS_UNUSED) { + | ZVAL_COPY_VALUE res_addr, op1_addr, op1_def_info, ZREG_R0, ZREG_R1 + | TRY_ADDREF op1_def_info, ah, r1 + } | jmp >3 |.code } @@ -1991,63 +1970,38 @@ static int zend_jit_math_long_long(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, - zend_uchar op1_type, - znode_op op1, - uint32_t op1_reg, - uint32_t op1_offset, - zend_uchar op2_type, - znode_op op2, - uint32_t op2_reg, - uint32_t op2_offset, - uint32_t res_reg, - uint32_t res_offset, - uint32_t res_info) + zend_jit_addr op1_addr, + zend_jit_addr op2_addr, + zend_jit_addr res_addr, + uint32_t res_info) { - zend_bool same_ops = (op1_type == op2_type) && (op1.var == op2.var); + zend_bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); if (opline->opcode == ZEND_MUL && - ((op2_type == IS_CONST && - IS_SIGNED_32BIT(Z_LVAL_P(RT_CONSTANT(op_array, op2))) && - is_power_of_two(Z_LVAL_P(RT_CONSTANT(op_array, op2)))) || - (op1_type == IS_CONST && - IS_SIGNED_32BIT(Z_LVAL_P(RT_CONSTANT(op_array, op1))) && - is_power_of_two(Z_LVAL_P(RT_CONSTANT(op_array, op1)))))) { - if (op2_type == IS_CONST) { - if (op1_type == IS_CONST) { - | LONG_LOAD r0, op1_type, op1 - } else { - | GET_Z_LVAL r0, Ra(op1_reg)+op1_offset - } - | shl r0, floor_log2(Z_LVAL_P(RT_CONSTANT(op_array, op2))) + ((Z_MODE(op2_addr) == IS_CONST_ZVAL && + IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(op2_addr))) && + is_power_of_two(Z_LVAL_P(Z_ZV(op2_addr)))) || + (Z_MODE(op1_addr) == IS_CONST_ZVAL && + IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(op1_addr))) && + is_power_of_two(Z_LVAL_P(Z_ZV(op1_addr)))))) { + if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { + | GET_ZVAL_LVAL r0, op1_addr + | shl r0, floor_log2(Z_LVAL_P(Z_ZV(op2_addr))) } else { - if (op2_type == IS_CONST) { - | LONG_LOAD r0, op2_type, op2 - } else { - | GET_Z_LVAL r0, Ra(op2_reg)+op2_offset - } - | shl r0, floor_log2(Z_LVAL_P(RT_CONSTANT(op_array, op1))) + | GET_ZVAL_LVAL r0, op2_addr + | shl r0, floor_log2(Z_LVAL_P(Z_ZV(op1_addr))) } } else if (opline->opcode == ZEND_DIV && - (op2_type == IS_CONST && - is_power_of_two(Z_LVAL_P(RT_CONSTANT(op_array, op2))))) { - if (op1_type == IS_CONST) { - | LONG_LOAD r0, op1_type, op1 - } else { - | GET_Z_LVAL r0, Ra(op1_reg)+op1_offset - } - | shr r0, floor_log2(Z_LVAL_P(RT_CONSTANT(op_array, op2))) + (Z_MODE(op2_addr) == IS_CONST_ZVAL && + is_power_of_two(Z_LVAL_P(Z_ZV(op2_addr))))) { + | GET_ZVAL_LVAL r0, op1_addr + | shr r0, floor_log2(Z_LVAL_P(Z_ZV(op2_addr))) } else { - if (op1_type == IS_CONST) { - | LONG_LOAD r0, op1_type, op1 - } else { - | GET_Z_LVAL r0, Ra(op1_reg)+op1_offset - } + | GET_ZVAL_LVAL r0, op1_addr if (same_ops && opline->opcode != ZEND_DIV) { - | LONG_MATH2 opline->opcode, r0, r0 - } else if (op2_type == IS_CONST) { - | LONG_MATH opline->opcode, r0, op2_type, op2 + | LONG_MATH_REG opline->opcode, r0, r0 } else { - | LONG_MATH2 opline->opcode, r0, aword [Ra(op2_reg)+op2_offset] + | LONG_MATH opline->opcode, r0, op2_addr } } if ((res_info & MAY_BE_DOUBLE) && zend_may_overflow(opline, op_array, ssa)) { @@ -2055,47 +2009,31 @@ static int zend_jit_math_long_long(dasm_State **Dst, |.cold_code |1: |.if X64 or SSE - zend_reg tmp_reg1 = ZREG_XMM0; - zend_reg tmp_reg2 = ZREG_XMM1; + zend_reg tmp_reg1 = ZREG_XMM0; + zend_reg tmp_reg2 = ZREG_XMM1; - if (op1_type == IS_CONST) { - | SSE_LOAD_LONG tmp_reg1, op1_type, op1 - } else { - | SSE_GET_Z_LVAL tmp_reg1, Ra(op1_reg)+op1_offset - } - if (op2_type == IS_CONST) { - | SSE_LOAD_LONG tmp_reg2, op2_type, op2 - } else { - | SSE_GET_Z_LVAL tmp_reg2, Ra(op2_reg)+op2_offset - } - | SSE_MATH_REG opline->opcode, tmp_reg1, tmp_reg2 - | SSE_STORE Ra(res_reg)+res_offset, tmp_reg1 + | SSE_GET_ZVAL_LVAL tmp_reg1, op1_addr + | SSE_GET_ZVAL_LVAL tmp_reg2, op2_addr + | SSE_MATH_REG opline->opcode, tmp_reg1, tmp_reg2 + | SSE_SET_ZVAL_DVAL res_addr, tmp_reg1 |.else - if (op2_type == IS_CONST) { - | FPU_LONG_LOAD op2_type, op2 - } else { - | FPU_GET_Z_LVAL Ra(op2_reg)+op2_offset - } - if (op1_type == IS_CONST) { - | FPU_LONG_LOAD op1_type, op1 - } else { - | FPU_GET_Z_LVAL Ra(op1_reg)+op1_offset - } - | FPU_MATH2 opline->opcode, st1 - | FPU_STORE Ra(res_reg)+res_offset + | FPU_GET_ZVAL_LVAL op2_addr + | FPU_GET_ZVAL_LVAL op1_addr + | FPU_MATH_REG opline->opcode, st1 + | FPU_SET_ZVAL_DVAL res_addr |.endif - | SET_Z_TYPE_INFO Ra(res_reg)+res_offset, IS_DOUBLE + | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE | jmp >2 |.code - | SET_Z_LVAL Ra(res_reg)+res_offset, r0 - if (op1_type == IS_CONST || op1_reg != res_reg || op1_offset != res_offset) { - | SET_Z_TYPE_INFO Ra(res_reg)+res_offset, IS_LONG + | SET_ZVAL_LVAL res_addr, r0 + if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { + | SET_ZVAL_TYPE_INFO res_addr, IS_LONG } |2: } else { - | SET_Z_LVAL Ra(res_reg)+res_offset, r0 - if (op1_type == IS_CONST || op1_reg != res_reg || op1_offset != res_offset) { - | SET_Z_TYPE_INFO Ra(res_reg)+res_offset, IS_LONG + | SET_ZVAL_LVAL res_addr, r0 + if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { + | SET_ZVAL_TYPE_INFO res_addr, IS_LONG } } @@ -2104,176 +2042,96 @@ static int zend_jit_math_long_long(dasm_State **Dst, static int zend_jit_math_long_double(dasm_State **Dst, const zend_op *opline, - zend_uchar op1_type, - znode_op op1, - uint32_t op1_reg, - uint32_t op1_offset, - zend_uchar op2_type, - znode_op op2, - uint32_t op2_reg, - uint32_t op2_offset, - uint32_t res_reg, - uint32_t res_offset) + zend_jit_addr op1_addr, + zend_jit_addr op2_addr, + zend_jit_addr res_addr) { |.if X64 or SSE zend_reg result_reg = ZREG_XMM0; - if (op1_type == IS_CONST) { - | SSE_LOAD_LONG result_reg, op1_type, op1 - } else { - | SSE_GET_Z_LVAL result_reg, Ra(op1_reg)+op1_offset - } - if (op2_type == IS_CONST) { - | SSE_MATH_CONST opline->opcode, result_reg, op2 - } else { - | SSE_MATH opline->opcode, result_reg, qword [Ra(op2_reg)+op2_offset] - } - | SSE_STORE Ra(res_reg)+res_offset, result_reg + | SSE_GET_ZVAL_LVAL result_reg, op1_addr + | SSE_MATH opline->opcode, result_reg, op2_addr + | SSE_SET_ZVAL_DVAL res_addr, result_reg |.else - if (op1_type == IS_CONST) { - | FPU_LONG_LOAD op1_type, op1 - } else { - | FPU_GET_Z_LVAL Ra(op1_reg)+op1_offset - } - if (op2_type == IS_CONST) { - | FPU_MATH opline->opcode, op2_type, op2 - } else { - | FPU_MATH2 opline->opcode, qword [Ra(op2_reg)+op2_offset] - } - | FPU_STORE Ra(res_reg)+res_offset + | FPU_GET_ZVAL_LVAL op1_addr + | FPU_MATH opline->opcode, op2_addr + | FPU_SET_ZVAL_DVAL res_addr |.endif - | SET_Z_TYPE_INFO Ra(res_reg)+res_offset, IS_DOUBLE + | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE return 1; } static int zend_jit_math_double_long(dasm_State **Dst, const zend_op *opline, - zend_uchar op1_type, - znode_op op1, - uint32_t op1_reg, - uint32_t op1_offset, - zend_uchar op2_type, - znode_op op2, - uint32_t op2_reg, - uint32_t op2_offset, - uint32_t res_reg, - uint32_t res_offset) + zend_jit_addr op1_addr, + zend_jit_addr op2_addr, + zend_jit_addr res_addr) { |.if X64 or SSE - || zend_reg result_reg = ZREG_XMM0; - || - || if (opline->opcode == ZEND_ADD || opline->opcode == ZEND_MUL || opline->opcode == ZEND_ASSIGN_ADD || opline->opcode == ZEND_ASSIGN_MUL) { - || if (op2_type == IS_CONST) { - | SSE_LOAD_LONG result_reg, op2_type, op2 - || } else { - | SSE_GET_Z_LVAL result_reg, Ra(op2_reg)+op2_offset - || } - || if (op1_type == IS_CONST) { - | SSE_MATH_CONST opline->opcode, result_reg, op1 - || } else { - | SSE_MATH opline->opcode, result_reg, qword [Ra(op1_reg)+op1_offset] - || } - || } else { - || int tmp_reg = 1; - || - || if (op1_type == IS_CONST) { - | SSE_LOAD result_reg, op1_type, op1 - || } else { - | SSE_GET_Z_DVAL result_reg, Ra(op1_reg)+op1_offset - || } - || if (op2_type == IS_CONST) { - | SSE_LOAD_LONG tmp_reg, op2_type, op2 - || } else { - | SSE_GET_Z_LVAL tmp_reg, Ra(op2_reg)+op2_offset - || } - | SSE_MATH_REG opline->opcode, result_reg, tmp_reg - || } - | SSE_STORE Ra(res_reg)+res_offset, result_reg + zend_reg result_reg = ZREG_XMM0; + + if (opline->opcode == ZEND_ADD || opline->opcode == ZEND_MUL || opline->opcode == ZEND_ASSIGN_ADD || opline->opcode == ZEND_ASSIGN_MUL) { + | SSE_GET_ZVAL_LVAL result_reg, op2_addr + | SSE_MATH opline->opcode, result_reg, op1_addr + } else { + zend_reg tmp_reg = ZREG_XMM1; + + | SSE_GET_ZVAL_DVAL result_reg, op1_addr + | SSE_GET_ZVAL_LVAL tmp_reg, op2_addr + | SSE_MATH_REG opline->opcode, result_reg, tmp_reg + } + | SSE_SET_ZVAL_DVAL res_addr, result_reg |.else - || if (op2_type == IS_CONST) { - | FPU_LONG_LOAD op2_type, op2 - || } else { - | FPU_GET_Z_LVAL Ra(op2_reg)+op2_offset - || } - || if (opline->opcode == ZEND_ADD || opline->opcode == ZEND_MUL || opline->opcode == ZEND_ASSIGN_ADD || opline->opcode == ZEND_ASSIGN_MUL) { - || if (op1_type == IS_CONST) { - | FPU_MATH opline->opcode, op1_type, op1 - || } else { - | FPU_MATH2 opline->opcode, qword [Ra(op1_reg)+op1_offset] - || } - || } else { - || if (op1_type == IS_CONST) { - | FPU_LOAD op1_type, op1 - || } else { - | FPU_GET_Z_DVAL Ra(op1_reg)+op1_offset - || } - | FPU_MATH2 opline->opcode, st1 - || } - | FPU_STORE Ra(res_reg)+res_offset + | FPU_GET_ZVAL_LVAL op2_addr + if (opline->opcode == ZEND_ADD || opline->opcode == ZEND_MUL || opline->opcode == ZEND_ASSIGN_ADD || opline->opcode == ZEND_ASSIGN_MUL) { + | FPU_MATH opline->opcode, op1_addr + } else { + | FPU_GET_ZVAL_DVAL op1_addr + | FPU_MATH_REG opline->opcode, st1 + } + | FPU_SET_ZVAL_DVAL res_addr |.endif - if (op1_type == IS_CONST || op1_reg != res_reg || op1_offset != res_offset) { - | SET_Z_TYPE_INFO Ra(res_reg)+res_offset, IS_DOUBLE + if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { + | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE } return 1; } static int zend_jit_math_double_double(dasm_State **Dst, const zend_op *opline, - zend_uchar op1_type, - znode_op op1, - uint32_t op1_reg, - uint32_t op1_offset, - zend_uchar op2_type, - znode_op op2, - uint32_t op2_reg, - uint32_t op2_offset, - uint32_t res_reg, - uint32_t res_offset) + zend_jit_addr op1_addr, + zend_jit_addr op2_addr, + zend_jit_addr res_addr) { - zend_bool same_ops = (op1_type == op2_type) && (op1.var == op2.var); + zend_bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); |.if X64 or SSE - zend_reg result_reg = ZREG_XMM0; + zend_reg result_reg = ZREG_XMM0; - if (op1_type == IS_CONST) { - | SSE_LOAD result_reg, op1_type, op1 - } else { - | SSE_GET_Z_DVAL result_reg, Ra(op1_reg)+op1_offset - } - if (same_ops) { - | SSE_MATH_REG opline->opcode, result_reg, result_reg - } else if (op2_type == IS_CONST) { - | SSE_MATH_CONST opline->opcode, result_reg, op2 - } else { - | SSE_MATH opline->opcode, result_reg, qword [Ra(op2_reg)+op2_offset] - } - | SSE_STORE Ra(res_reg)+res_offset, result_reg + | SSE_GET_ZVAL_DVAL result_reg, op1_addr + if (same_ops) { + | SSE_MATH_REG opline->opcode, result_reg, result_reg + } else { + | SSE_MATH opline->opcode, result_reg, op2_addr + } + | SSE_SET_ZVAL_DVAL res_addr, result_reg |.else - if (op1_type == IS_CONST) { - | FPU_LOAD op1_type, op1 - } else { - | FPU_GET_Z_DVAL Ra(op1_reg)+op1_offset - } - if (op2_type == IS_CONST) { - | FPU_MATH opline->opcode, op2_type, op2 - } else { - | FPU_MATH2 opline->opcode, qword [Ra(op2_reg)+op2_offset] - } - | FPU_STORE Ra(res_reg)+res_offset + | FPU_GET_ZVAL_DVAL op1_addr + | FPU_MATH opline->opcode, op2_addr + | FPU_SET_ZVAL_DVAL res_addr |.endif - if (op1_type == IS_CONST || op1_reg != res_reg || op1_offset != res_offset) { - | SET_Z_TYPE_INFO Ra(res_reg)+res_offset, IS_DOUBLE + if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { + | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE } return 1; } - static int zend_jit_shift(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) { - zval *op2; uint32_t op1_info; + zend_jit_addr op1_addr, op2_addr, res_addr; zend_bool has_slow; if (!ssa->ops || !ssa->var_info) { @@ -2282,43 +2140,45 @@ static int zend_jit_shift(dasm_State **Dst, const zend_op *opline, zend_op_array op1_info = OP1_INFO(); + op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1); + op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2); + res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result); + if ((op1_info & MAY_BE_UNDEF) || - opline->op2_type != IS_CONST || - Z_TYPE_P(RT_CONSTANT(op_array, opline->op2)) != IS_LONG || - Z_LVAL_P(RT_CONSTANT(op_array, opline->op2)) >= SIZEOF_ZEND_LONG * 8) { + Z_MODE(op2_addr) != IS_CONST_ZVAL || + Z_TYPE_P(Z_ZV(op2_addr)) != IS_LONG || + Z_LVAL_P(Z_ZV(op2_addr)) >= SIZEOF_ZEND_LONG * 8) { goto fallback; } has_slow = (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_LONG)) != 0; - op2 = RT_CONSTANT(op_array, opline->op2); - if (op1_info & (MAY_BE_ANY-MAY_BE_LONG)) { - | IF_NOT_Z_TYPE FP + opline->op1.var, IS_LONG, >9 + | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >9 } - | LONG_LOAD r0, opline->op1_type, opline->op1 + | GET_ZVAL_LVAL r0, op1_addr if (opline->opcode == ZEND_SR) { - | shr, r0, Z_LVAL_P(op2) + | shr, r0, Z_LVAL_P(Z_ZV(op2_addr)) } else { - | shl, r0, Z_LVAL_P(op2) + | shl, r0, Z_LVAL_P(Z_ZV(op2_addr)) } - | SET_Z_LVAL FP + opline->result.var, r0 - | SET_Z_TYPE_INFO FP + opline->result.var, IS_LONG + | SET_ZVAL_LVAL res_addr, r0 + | SET_ZVAL_TYPE_INFO res_addr, IS_LONG if (has_slow) { |.cold_code |9: - | lea FCARG1a, aword [FP + opline->result.var] - | lea FCARG2a, aword [FP + opline->op1.var] + | LOAD_ZVAL_ADDR FCARG1a, res_addr + | LOAD_ZVAL_ADDR FCARG2a, op1_addr |.if X64 - | LOAD_ADDR CARG3, op2 + | LOAD_ZVAL_ADDR CARG3, op2_addr |.else - | push dword op2 + | PUSH_ZVAL_ADDR op2_addr, r0 |.endif if (opline->opcode == ZEND_SR) { - | EXT_CALL shift_right_function, r0 + | EXT_CALL shift_right_function, r0 } else { - | EXT_CALL shift_left_function, r0 + | EXT_CALL shift_left_function, r0 } | jmp >1 |.code @@ -2340,65 +2200,62 @@ static int zend_jit_math_helper(dasm_State **Dst, zend_ssa *ssa, zend_uchar op1_type, znode_op op1, - uint32_t op1_reg, - uint32_t op1_offset, + zend_jit_addr op1_addr, uint32_t op1_info, zend_uchar op2_type, znode_op op2, - uint32_t op2_reg, - uint32_t op2_offset, + zend_jit_addr op2_addr, uint32_t op2_info, - uint32_t res_reg, - uint32_t res_offset, + zend_jit_addr res_addr, uint32_t res_info, zend_bool separate_op1) /* Labels: 1,2,3,4,5,6 */ { - zend_bool same_ops = (op1_type == op2_type) && (op1.var == op2.var); + zend_bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)) { if (op1_info & (MAY_BE_ANY-MAY_BE_LONG)) { if (op1_info & MAY_BE_DOUBLE) { - | IF_NOT_Z_TYPE Ra(op1_reg)+op1_offset, IS_LONG, >3 + | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >3 } else { - | IF_NOT_Z_TYPE Ra(op1_reg)+op1_offset, IS_LONG, >6 + | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6 } } if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_LONG))) { if (op2_info & MAY_BE_DOUBLE) { - | IF_NOT_Z_TYPE Ra(op2_reg)+op2_offset, IS_LONG, >1 + | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >1 |.cold_code |1: if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { - | IF_NOT_Z_TYPE Ra(op2_reg)+op2_offset, IS_DOUBLE, >6 + | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >6 } - if (!zend_jit_math_long_double(Dst, opline, op1_type, op1, op1_reg, op1_offset, op2_type, op2, op2_reg, op2_offset, res_reg, res_offset)) { + if (!zend_jit_math_long_double(Dst, opline, op1_addr, op2_addr, res_addr)) { return 0; } | jmp >5 |.code } else { - | IF_NOT_Z_TYPE Ra(op2_reg)+op2_offset, IS_LONG, >6 + | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6 } } - if (!zend_jit_math_long_long(Dst, opline, op_array, ssa, op1_type, op1, op1_reg, op1_offset, op2_type, op2, op2_reg, op2_offset, res_reg, res_offset, res_info)) { + if (!zend_jit_math_long_long(Dst, opline, op_array, ssa, op1_addr, op2_addr, res_addr, res_info)) { return 0; } if (op1_info & MAY_BE_DOUBLE) { |.cold_code |3: if (op1_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { - | IF_NOT_Z_TYPE Ra(op1_reg)+op1_offset, IS_DOUBLE, >6 + | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >6 } if (op2_info & MAY_BE_DOUBLE) { if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { if (!same_ops) { - | IF_NOT_Z_TYPE, Ra(op2_reg)+op2_offset, IS_DOUBLE, >1 + | IF_NOT_ZVAL_TYPE, op2_addr, IS_DOUBLE, >1 } else { - | IF_NOT_Z_TYPE, Ra(op2_reg)+op2_offset, IS_DOUBLE, >6 + | IF_NOT_ZVAL_TYPE, op2_addr, IS_DOUBLE, >6 } } - if (!zend_jit_math_double_double(Dst, opline, op1_type, op1, op1_reg, op1_offset, op2_type, op2, op2_reg, op2_offset, res_reg, res_offset)) { + if (!zend_jit_math_double_double(Dst, opline, op1_addr, op2_addr, res_addr)) { return 0; } | jmp >5 @@ -2406,9 +2263,9 @@ static int zend_jit_math_helper(dasm_State **Dst, if (!same_ops) { |1: if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { - | IF_NOT_Z_TYPE Ra(op2_reg)+op2_offset, IS_LONG, >6 + | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6 } - if (!zend_jit_math_double_long(Dst, opline, op1_type, op1, op1_reg, op1_offset, op2_type, op2, op2_reg, op2_offset, res_reg, res_offset)) { + if (!zend_jit_math_double_long(Dst, opline, op1_addr, op2_addr, res_addr)) { return 0; } | jmp >5 @@ -2419,17 +2276,17 @@ static int zend_jit_math_helper(dasm_State **Dst, !(op1_info & MAY_BE_LONG) && (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { if (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) { - | IF_NOT_Z_TYPE Ra(op1_reg)+op1_offset, IS_DOUBLE, >6 + | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >6 } if (op2_info & MAY_BE_DOUBLE) { if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { if (!same_ops && (op2_info & MAY_BE_LONG)) { - | IF_NOT_Z_TYPE Ra(op2_reg)+op2_offset, IS_DOUBLE, >1 + | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >1 } else { - | IF_NOT_Z_TYPE Ra(op2_reg)+op2_offset, IS_DOUBLE, >6 + | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >6 } } - if (!zend_jit_math_double_double(Dst, opline, op1_type, op1, op1_reg, op1_offset, op2_type, op2, op2_reg, op2_offset, res_reg, res_offset)) { + if (!zend_jit_math_double_double(Dst, opline, op1_addr, op2_addr, res_addr)) { return 0; } } @@ -2439,9 +2296,9 @@ static int zend_jit_math_helper(dasm_State **Dst, } |1: if (op2_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) { - | IF_NOT_Z_TYPE Ra(op2_reg)+op2_offset, IS_LONG, >5 + | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >5 } - if (!zend_jit_math_double_long(Dst, opline, op1_type, op1, op1_reg, op1_offset, op2_type, op2, op2_reg, op2_offset, res_reg, res_offset)) { + if (!zend_jit_math_double_long(Dst, opline, op1_addr, op2_addr, res_addr)) { return 0; } if (op2_info & MAY_BE_DOUBLE) { @@ -2453,17 +2310,17 @@ static int zend_jit_math_helper(dasm_State **Dst, !(op2_info & MAY_BE_LONG) && (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { if (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) { - | IF_NOT_Z_TYPE Ra(op2_reg)+op2_offset, IS_DOUBLE, >6 + | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >6 } if (op1_info & MAY_BE_DOUBLE) { if (!same_ops && (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { if (!same_ops && (op1_info & MAY_BE_LONG)) { - | IF_NOT_Z_TYPE Ra(op1_reg)+op1_offset, IS_DOUBLE, >1 + | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >1 } else { - | IF_NOT_Z_TYPE Ra(op1_reg)+op1_offset, IS_DOUBLE, >6 + | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >6 } } - if (!zend_jit_math_double_double(Dst, opline, op1_type, op1, op1_reg, op1_offset, op2_type, op2, op2_reg, op2_offset, res_reg, res_offset)) { + if (!zend_jit_math_double_double(Dst, opline, op1_addr, op2_addr, res_addr)) { return 0; } } @@ -2473,9 +2330,9 @@ static int zend_jit_math_helper(dasm_State **Dst, } |1: if (op1_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) { - | IF_NOT_Z_TYPE Ra(op1_reg)+op1_offset, IS_LONG, >6 + | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6 } - if (!zend_jit_math_long_double(Dst, opline, op1_type, op1, op1_reg, op1_offset, op2_type, op2, op2_reg, op2_offset, res_reg, res_offset)) { + if (!zend_jit_math_long_double(Dst, opline, op1_addr, op2_addr, res_addr)) { return 0; } if (op1_info & MAY_BE_DOUBLE) { @@ -2496,50 +2353,33 @@ static int zend_jit_math_helper(dasm_State **Dst, |6: | SAVE_VALID_OPLINE opline if (separate_op1) { - if (op1_reg != ZREG_FCARG1a || op1_offset != 0) { - | SEPARATE_ZVAL_NOREF Ra(op1_reg)+op1_offset, op1_info, 0, op_array->filename, opline->lineno - } else { - | SEPARATE_ZVAL_NOREF_REG op1_info, 0, op_array->filename, opline->lineno - } - } - if (res_reg != ZREG_FCARG1a || res_offset != 0) { - | lea FCARG1a, [Ra(res_reg)+res_offset] + | SEPARATE_ZVAL_NOREF op1_addr, op1_info, 0, op_array->filename, opline->lineno } - if (op1_type == IS_CONST) { - | LOAD_ZVAL_ADDR FCARG2a, op1_type, op1 - } else { - | lea FCARG2a, [Ra(op1_reg)+op1_offset] + if (Z_REG(res_addr) != ZREG_FCARG1a || Z_OFFSET(res_addr) != 0) { + | LOAD_ZVAL_ADDR FCARG1a, res_addr } - if (op2_type == IS_CONST) { - |.if X64 - | LOAD_ZVAL_ADDR CARG3, op2_type, op2 - |.else - | PUSH_ZVAL_ADDR op2_type, op2, r0 - |.endif + | LOAD_ZVAL_ADDR FCARG2a, op1_addr + |.if X64 + | LOAD_ZVAL_ADDR CARG3, op2_addr + |.else + | PUSH_ZVAL_ADDR op2_addr, r0 + |.endif + if (opline->opcode == ZEND_ADD || opline->opcode == ZEND_ASSIGN_ADD) { + | EXT_CALL add_function, r0 + } else if (opline->opcode == ZEND_SUB || opline->opcode == ZEND_ASSIGN_SUB) { + | EXT_CALL sub_function, r0 + } else if (opline->opcode == ZEND_MUL || opline->opcode == ZEND_ASSIGN_MUL) { + | EXT_CALL mul_function, r0 + } else if (opline->opcode == ZEND_DIV || opline->opcode == ZEND_ASSIGN_DIV) { + | EXT_CALL div_function, r0 } else { - |.if X64 - | lea CARG3, [Ra(op2_reg)+op2_offset] - |.else - | lea r0, [Ra(op2_reg)+op2_offset] - | push r0 - |.endif + ZEND_ASSERT(0); } - || if (opline->opcode == ZEND_ADD || opline->opcode == ZEND_ASSIGN_ADD) { - | EXT_CALL add_function, r0 - || } else if (opline->opcode == ZEND_SUB || opline->opcode == ZEND_ASSIGN_SUB) { - | EXT_CALL sub_function, r0 - || } else if (opline->opcode == ZEND_MUL || opline->opcode == ZEND_ASSIGN_MUL) { - | EXT_CALL mul_function, r0 - || } else if (opline->opcode == ZEND_DIV || opline->opcode == ZEND_ASSIGN_DIV) { - | EXT_CALL div_function, r0 - || } else { - || ZEND_ASSERT(0); - || } | FREE_OP op1_type, op1, op1_info, 0, op_array, opline | FREE_OP op2_type, op2, op2_info, 0, op_array, opline - || if (zend_may_throw(opline, op_array, ssa)) { - || zend_jit_check_exception(Dst); - || } + if (zend_may_throw(opline, op_array, ssa)) { + zend_jit_check_exception(Dst); + } if ((op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { | jmp <5 @@ -2553,6 +2393,7 @@ static int zend_jit_math_helper(dasm_State **Dst, static int zend_jit_math(dasm_State **Dst, const zend_op *opline, int *opnum, zend_op_array *op_array, zend_ssa *ssa) { uint32_t op1_info, op2_info; + zend_jit_addr op1_addr, op2_addr, res_addr; if (!ssa->ops || !ssa->var_info) { goto fallback; @@ -2570,6 +2411,9 @@ static int zend_jit_math(dasm_State **Dst, const zend_op *opline, int *opnum, ze goto fallback; } + op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1); + op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2); + if (opline->result_type == IS_TMP_VAR && (opline+1)->opcode == ZEND_SEND_VAL && (opline+1)->op1_type == IS_TMP_VAR && @@ -2581,11 +2425,13 @@ static int zend_jit_math(dasm_State **Dst, const zend_op *opline, int *opnum, ze | // call = EX(call); | mov RX, EX->call } - return zend_jit_math_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, ZREG_FP, opline->op1.var, op1_info, opline->op2_type, opline->op2, ZREG_FP, opline->op2.var, op2_info, ZREG_RX, (opline+1)->result.var, RES_INFO(), 0); + res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var); } else { - return zend_jit_math_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, ZREG_FP, opline->op1.var, op1_info, opline->op2_type, opline->op2, ZREG_FP, opline->op2.var, op2_info, ZREG_FP, opline->result.var, RES_INFO(), 0); + res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result); } + return zend_jit_math_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, res_addr, RES_INFO(), 0); + fallback: /* fallback to subroutine threading */ return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); @@ -2597,59 +2443,39 @@ static int zend_jit_concat_helper(dasm_State **Dst, zend_ssa *ssa, zend_uchar op1_type, znode_op op1, - uint32_t op1_reg, - uint32_t op1_offset, + zend_jit_addr op1_addr, uint32_t op1_info, zend_uchar op2_type, znode_op op2, - uint32_t op2_reg, - uint32_t op2_offset, + zend_jit_addr op2_addr, uint32_t op2_info, - uint32_t res_reg, - uint32_t res_offset, + zend_jit_addr res_addr, uint32_t res_info) { #if 1 if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) { if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) { - | IF_NOT_Z_TYPE Ra(op1_reg)+op1_offset, IS_STRING, >6 + | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >6 } if (op2_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) { - | IF_NOT_Z_TYPE Ra(op2_reg)+op2_offset, IS_STRING, >6 + | IF_NOT_ZVAL_TYPE op2_addr, IS_STRING, >6 } - if (op1_type != IS_CONST && op1_reg == res_reg && op1_offset == res_offset) { - if (res_reg != ZREG_FCARG1a || res_offset != 0) { - | lea FCARG1a, [Ra(res_reg)+res_offset] - } - if (op2_type == IS_CONST) { - | LOAD_ZVAL_ADDR FCARG2a, op2_type, op2 - } else { - | lea FCARG2a, [Ra(op2_reg)+op2_offset] + if (Z_MODE(op1_addr) == IS_MEM_ZVAL && Z_REG(op1_addr) == Z_REG(res_addr) && Z_OFFSET(op1_addr) == Z_OFFSET(res_addr)) { + if (Z_REG(res_addr) != ZREG_FCARG1a || Z_OFFSET(res_addr) != 0) { + | LOAD_ZVAL_ADDR FCARG1a, res_addr } + | LOAD_ZVAL_ADDR FCARG2a, op2_addr | EXT_CALL zend_jit_fast_assign_concat_helper, r0 } else { - if (res_reg != ZREG_FCARG1a || res_offset != 0) { - | lea FCARG1a, [Ra(res_reg)+res_offset] - } - if (op1_type == IS_CONST) { - | LOAD_ZVAL_ADDR FCARG2a, op1_type, op1 - } else { - | lea FCARG2a, [Ra(op1_reg)+op1_offset] - } - if (op2_type == IS_CONST) { - |.if X64 - | LOAD_ZVAL_ADDR CARG3, op2_type, op2 - |.else - | PUSH_ZVAL_ADDR op2_type, op2, r0 - |.endif - } else { - |.if X64 - | lea CARG3, [Ra(op2_reg)+op2_offset] - |.else - | lea r0, [Ra(op2_reg)+op2_offset] - | push r0 - |.endif + if (Z_REG(res_addr) != ZREG_FCARG1a || Z_OFFSET(res_addr) != 0) { + | LOAD_ZVAL_ADDR FCARG1a, res_addr } + | LOAD_ZVAL_ADDR FCARG2a, op1_addr + |.if X64 + | LOAD_ZVAL_ADDR CARG3, op2_addr + |.else + | PUSH_ZVAL_ADDR op2_addr, r0 + |.endif | EXT_CALL zend_jit_fast_concat_helper, r0 } | FREE_OP op1_type, op1, op1_info, 0, op_array, opline @@ -2664,34 +2490,21 @@ static int zend_jit_concat_helper(dasm_State **Dst, } #endif | SAVE_VALID_OPLINE opline - if (res_reg != ZREG_FCARG1a || res_offset != 0) { - | lea FCARG1a, [Ra(res_reg)+res_offset] - } - if (op1_type == IS_CONST) { - | LOAD_ZVAL_ADDR FCARG2a, op1_type, op1 - } else { - | lea FCARG2a, [Ra(op1_reg)+op1_offset] - } - if (op2_type == IS_CONST) { - |.if X64 - | LOAD_ZVAL_ADDR CARG3, op2_type, op2 - |.else - | PUSH_ZVAL_ADDR op2_type, op2, r0 - |.endif - } else { - |.if X64 - | lea CARG3, [Ra(op2_reg)+op2_offset] - |.else - | lea r0, [Ra(op2_reg)+op2_offset] - | push r0 - |.endif + if (Z_REG(res_addr) != ZREG_FCARG1a || Z_OFFSET(res_addr) != 0) { + | LOAD_ZVAL_ADDR FCARG1a, res_addr } + | LOAD_ZVAL_ADDR FCARG2a, op1_addr + |.if X64 + | LOAD_ZVAL_ADDR CARG3, op2_addr + |.else + | PUSH_ZVAL_ADDR op2_addr, r0 + |.endif | EXT_CALL concat_function, r0 | FREE_OP op1_type, op1, op1_info, 0, op_array, opline | FREE_OP op2_type, op2, op2_info, 0, op_array, opline - || if (zend_may_throw(opline, op_array, ssa)) { - || zend_jit_check_exception(Dst); - || } + if (zend_may_throw(opline, op_array, ssa)) { + zend_jit_check_exception(Dst); + } #if 1 if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) { | jmp <5 @@ -2706,6 +2519,7 @@ static int zend_jit_concat_helper(dasm_State **Dst, static int zend_jit_concat(dasm_State **Dst, const zend_op *opline, int *opnum, zend_op_array *op_array, zend_ssa *ssa) { uint32_t op1_info, op2_info; + zend_jit_addr op1_addr, op2_addr, res_addr; if (!ssa->ops || !ssa->var_info) { goto fallback; @@ -2723,6 +2537,9 @@ static int zend_jit_concat(dasm_State **Dst, const zend_op *opline, int *opnum, goto fallback; } + op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1); + op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2); + if (opline->result_type == IS_TMP_VAR && (opline+1)->opcode == ZEND_SEND_VAL && (opline+1)->op1_type == IS_TMP_VAR && @@ -2734,10 +2551,11 @@ static int zend_jit_concat(dasm_State **Dst, const zend_op *opline, int *opnum, | // call = EX(call); | mov RX, EX->call } - return zend_jit_concat_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, ZREG_FP, opline->op1.var, op1_info, opline->op2_type, opline->op2, ZREG_FP, opline->op2.var, op2_info, ZREG_RX, (opline+1)->result.var, RES_INFO()); + res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var); } else { - return zend_jit_concat_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, ZREG_FP, opline->op1.var, op1_info, opline->op2_type, opline->op2, ZREG_FP, opline->op2.var, op2_info, ZREG_FP, opline->result.var, RES_INFO()); + res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result); } + return zend_jit_concat_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, res_addr, RES_INFO()); fallback: /* fallback to subroutine threading */ @@ -2747,26 +2565,29 @@ fallback: static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, uint32_t type, uint32_t op1_info, uint32_t op2_info, uint32_t found, uint32_t not_found) /* Labels: 1,2,3,4,5 */ { + zend_jit_addr op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2); + zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result); + if (op2_info & MAY_BE_LONG) { if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_LONG)) { | // if (EXPECTED(Z_TYPE_P(dim) == IS_LONG)) - | IF_NOT_Z_TYPE FP + opline->op2.var, IS_LONG, >3 + | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >3 } | // hval = Z_LVAL_P(dim); - | LONG_LOAD FCARG2a, opline->op2_type, opline->op2 + | GET_ZVAL_LVAL FCARG2a, op2_addr if (op1_info & MAY_BE_ARRAY_KEY_LONG) { - if (opline->op2_type == IS_CONST) { - zend_long val = Z_LVAL_P(RT_CONSTANT(op_array, opline->op2)); + if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { + zend_long val = Z_LVAL_P(Z_ZV(op2_addr)); if (val >= 0 && val < HT_MAX_SIZE) { | // ZEND_HASH_INDEX_FIND(ht, hval, retval, num_undef); | test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED | jz >4 // HASH_FIND | // if (EXPECTED((zend_ulong)(_h) < (zend_ulong)(_ht)->nNumUsed)) |.if X64 - | movsxd r0, dword [FCARG1a + offsetof(zend_array, nNumUsed)] - | cmp r0, val + | movsxd r0, dword [FCARG1a + offsetof(zend_array, nNumUsed)] + | cmp r0, val |.else - | cmp dword [FCARG1a + offsetof(zend_array, nNumUsed)], val + | cmp dword [FCARG1a + offsetof(zend_array, nNumUsed)], val |.endif if (type == BP_JIT_IS) { | jbe >9 // NOT_FOUND @@ -2775,7 +2596,9 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o } | // _ret = &_ht->arData[_h].val; | mov r0, aword [FCARG1a + offsetof(zend_array, arData)] - | add r0, val * sizeof(Bucket) + if (val != 0) { + | add r0, val * sizeof(Bucket) + } if (type == BP_JIT_IS) { | jmp >5 } else { @@ -2788,10 +2611,10 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o | jz >4 // HASH_FIND | // if (EXPECTED((zend_ulong)(_h) < (zend_ulong)(_ht)->nNumUsed)) |.if X64 - | movsxd r0, dword [FCARG1a + offsetof(zend_array, nNumUsed)] - | cmp r0, FCARG2a + | movsxd r0, dword [FCARG1a + offsetof(zend_array, nNumUsed)] + | cmp r0, FCARG2a |.else - | cmp dword [FCARG1a + offsetof(zend_array, nNumUsed)], FCARG2a + | cmp dword [FCARG1a + offsetof(zend_array, nNumUsed)], FCARG2a |.endif if (type == BP_JIT_IS) { | jbe >9 // NOT_FOUND @@ -2800,10 +2623,10 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o } | // _ret = &_ht->arData[_h].val; |.if X64 - | mov r0, FCARG2a - | shl r0, 5 + | mov r0, FCARG2a + | shl r0, 5 |.else - | imul r0, FCARG2a, sizeof(Bucket) + | imul r0, FCARG2a, sizeof(Bucket) |.endif | add r0, aword [FCARG1a + offsetof(zend_array, arData)] if (type == BP_JIT_IS) { @@ -2829,8 +2652,8 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o case BP_VAR_IS: case BP_VAR_UNSET: if (op1_info & MAY_BE_ARRAY_KEY_LONG) { - if (opline->op2_type == IS_CONST) { - zend_long val = Z_LVAL_P(RT_CONSTANT(op_array, opline->op2)); + if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { + zend_long val = Z_LVAL_P(Z_ZV(op2_addr)); if (val >= 0 && val < HT_MAX_SIZE) { | jmp >2 // NOT_FOUND } @@ -2854,7 +2677,7 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o case BP_VAR_IS: case BP_VAR_UNSET: | // retval = &EG(uninitialized_zval); - | SET_Z_TYPE_INFO FP + opline->result.var, IS_NULL + | SET_ZVAL_TYPE_INFO res_addr, IS_NULL | jmp >9 break; default: @@ -2878,20 +2701,20 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o case BP_VAR_W: |2: | //retval = zend_hash_index_add_new(ht, hval, &EG(uninitialized_zval)); - || if (ZEND_DEBUG) { - || const char *filename = op_array->filename ? op_array->filename->val : NULL; - |.if X64 - | LOAD_ADDR CARG4, filename - | mov CARG5d, opline->lineno - |.else - | push opline->lineno - | push filename - |.endif - || } + if (ZEND_DEBUG) { + const char *filename = op_array->filename ? op_array->filename->val : NULL; + |.if X64 + | LOAD_ADDR CARG4, filename + | mov CARG5d, opline->lineno + |.else + | push opline->lineno + | push filename + |.endif + } |.if X64 - | LOAD_ADDR CARG3, &EG(uninitialized_zval) + | LOAD_ADDR CARG3, &EG(uninitialized_zval) |.else - | PUSH_ADDR &EG(uninitialized_zval), r0 + | PUSH_ADDR &EG(uninitialized_zval), r0 |.endif | EXT_CALL _zend_hash_index_add_new, r0 if (op1_info & MAY_BE_ARRAY_KEY_LONG) { @@ -2913,10 +2736,10 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o |3: if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) { | // if (EXPECTED(Z_TYPE_P(dim) == IS_STRING)) - | IF_NOT_Z_TYPE FP + opline->op2.var, IS_STRING, >3 + | IF_NOT_ZVAL_TYPE op2_addr, IS_STRING, >3 } | // offset_key = Z_STR_P(dim); - | LONG_LOAD FCARG2a, opline->op2_type, opline->op2 + | GET_ZVAL_LVAL FCARG2a, op2_addr | // retval = zend_hash_find(ht, offset_key); switch (type) { case BP_JIT_IS: @@ -2975,7 +2798,7 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o case BP_VAR_IS: case BP_VAR_UNSET: | // retval = &EG(uninitialized_zval); - | SET_Z_TYPE_INFO FP + opline->result.var, IS_NULL + | SET_ZVAL_TYPE_INFO res_addr, IS_NULL | jmp >9 break; default: @@ -3018,14 +2841,13 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o |3: } | SAVE_VALID_OPLINE opline - | LOAD_ZVAL_ADDR FCARG2a, opline->op2_type, opline->op2 + | LOAD_ZVAL_ADDR FCARG2a, op2_addr switch (type) { case BP_VAR_R: |.if X64 - | lea CARG3, [FP + opline->result.var] + | LOAD_ZVAL_ADDR CARG3, res_addr |.else - | lea r0, [FP + opline->result.var] - | push r0 + | PUSH_ZVAL_ADDR res_addr, r0 |.endif | EXT_CALL zend_jit_fetch_dim_r_helper, r0 | jmp >9 @@ -3039,10 +2861,9 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o case BP_VAR_IS: case BP_VAR_UNSET: |.if X64 - | lea CARG3, [FP + opline->result.var] + | LOAD_ZVAL_ADDR CARG3, res_addr |.else - | lea r0, [FP + opline->result.var] - | push r0 + | PUSH_ZVAL_ADDR res_addr, r0 |.endif | EXT_CALL zend_jit_fetch_dim_is_helper, r0 | jmp >9 @@ -3074,26 +2895,28 @@ static int zend_jit_simple_assign(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, - uint32_t var_reg, - uint32_t var_offset, + zend_jit_addr var_addr, uint32_t var_info, zend_uchar val_type, znode_op val, + zend_jit_addr val_addr, uint32_t val_info, - uint32_t var2, + zend_jit_addr res_addr, int in_cold) /* Labels: 1,2,3 */ { - ZEND_ASSERT(var_reg != ZREG_R0); - if (val_type == IS_CONST) { - zval *zv = RT_CONSTANT(op_array, val); - if (var2 == (uint32_t)-1) { - | ZVAL_COPY_CONST Ra(var_reg)+var_offset, var_info, zv, r0 + ZEND_ASSERT(Z_MODE(var_addr) == IS_MEM_ZVAL); + ZEND_ASSERT(Z_REG(var_addr) != ZREG_R0); + if (Z_MODE(val_addr) == IS_CONST_ZVAL) { + zval *zv = Z_ZV(val_addr); + + if (!res_addr) { + | ZVAL_COPY_CONST var_addr, var_info, zv, r0 } else { - | ZVAL_COPY_CONST_2 Ra(var_reg)+var_offset, FP + var2, var_info, zv, r0 + | ZVAL_COPY_CONST_2 var_addr, res_addr, var_info, zv, r0 } if (Z_REFCOUNTED_P(zv)) { - if (var2 == (uint32_t)-1) { + if (!res_addr) { | ADDREF_CONST zv, r0 } else { | ADDREF_CONST_2 zv, r0 @@ -3102,25 +2925,25 @@ static int zend_jit_simple_assign(dasm_State **Dst, } else { if (val_info & MAY_BE_UNDEF) { if (in_cold) { - | IF_NOT_Z_TYPE FP + val.var, IS_UNDEF, >2 + | IF_NOT_ZVAL_TYPE val_addr, IS_UNDEF, >2 } else { - | IF_Z_TYPE FP + val.var, IS_UNDEF, >1 + | IF_ZVAL_TYPE val_addr, IS_UNDEF, >1 |.cold_code |1: } | // zend_error(E_NOTICE, "Undefined variable: %s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); - if (var_reg != ZREG_FP) { - | mov aword [r4], Ra(var_reg) // save + if (Z_REG(var_addr) != ZREG_FP) { + | mov aword [r4], Ra(Z_REG(var_addr)) // save } | SAVE_VALID_OPLINE opline | mov FCARG1d, val.var | EXT_CALL zend_jit_undefined_op_helper, r0 - if (var_reg != ZREG_FP) { - | mov Ra(var_reg), aword [r4] // restore + if (Z_REG(var_addr) != ZREG_FP) { + | mov Ra(Z_REG(var_addr)), aword [r4] // restore } - | SET_Z_TYPE_INFO Ra(var_reg)+var_offset, IS_NULL - if (var2 != (uint32_t)-1) { - | SET_Z_TYPE_INFO FP + var2, IS_NULL + | SET_ZVAL_TYPE_INFO var_addr, IS_NULL + if (res_addr) { + | SET_ZVAL_TYPE_INFO res_addr, IS_NULL } | jmp >3 if (in_cold) { @@ -3131,73 +2954,68 @@ static int zend_jit_simple_assign(dasm_State **Dst, } if (val_info & MAY_BE_REF) { if (val_type == IS_CV) { - ZEND_ASSERT(var_reg != ZREG_R2); - | lea r2, [FP + val.var] + ZEND_ASSERT(Z_REG(var_addr) != ZREG_R2); + | LOAD_ZVAL_ADDR r2, val_addr | ZVAL_DEREF r2, val_info - if (var2 == (uint32_t)-1) { - | ZVAL_COPY_VALUE_clobber_src Ra(var_reg)+var_offset, r2, val_info, r2, edx, r0 - } else { - | ZVAL_COPY_VALUE_clobber_src_2 Ra(var_reg)+var_offset, FP + var2, r2, val_info, r2, edx, r0 - } + val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R2, 0); } else { + zend_jit_addr ref_addr; + if (in_cold) { - | IF_NOT_Z_TYPE FP + val.var, IS_REFERENCE, >1 + | IF_NOT_ZVAL_TYPE val_addr, IS_REFERENCE, >1 } else { - | IF_Z_TYPE FP + val.var, IS_REFERENCE, >1 + | IF_ZVAL_TYPE val_addr, IS_REFERENCE, >1 |.cold_code |1: } | // zend_refcounted *ref = Z_COUNTED_P(retval_ptr); - | GET_Z_PTR r2, FP + val.var + | GET_ZVAL_PTR r2, val_addr | GC_DELREF r2 | // ZVAL_COPY_VALUE(return_value, &ref->value); - if (var2 == (uint32_t)-1) { - | ZVAL_COPY_VALUE_clobber_src Ra(var_reg)+var_offset, r2 + 8, val_info, r2, edx, r0 + ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R2, 8); + if (!res_addr) { + | ZVAL_COPY_VALUE var_addr, ref_addr, val_info, ZREG_R2, ZREG_R0 } else { - | ZVAL_COPY_VALUE_clobber_src_2 Ra(var_reg)+var_offset, FP + var2, r2 + 8, val_info, r2, edx, r0 + | ZVAL_COPY_VALUE_2 var_addr, res_addr, ref_addr, val_info, ZREG_R2, ZREG_R0 } | je >2 | IF_NOT_REFCOUNTED dh, >3 - if (var2 == (uint32_t)-1) { + if (!res_addr) { | GC_ADDREF r0 } else { | add dword [r0], 2 } | jmp >3 |2: - if (var2 != (uint32_t)-1) { + if (res_addr) { | IF_NOT_REFCOUNTED dh, >2 | GC_ADDREF r0 |2: } - | EFREE_24 aword [FP + val.var], op_array, opline + | EFREE_24 aword [Ra(Z_REG(val_addr))+Z_OFFSET(val_addr)], op_array, opline | jmp >3 if (in_cold) { |1: } else { |.code } - if (var2 == (uint32_t)-1) { - | ZVAL_COPY_VALUE Ra(var_reg)+var_offset, FP + val.var, val_info, r2, edx, r0 - } else { - | ZVAL_COPY_VALUE_2 Ra(var_reg)+var_offset, FP + var2, FP + val.var, val_info, r2, edx, r0 - } } + } + + if (!res_addr) { + | ZVAL_COPY_VALUE var_addr, val_addr, val_info, ZREG_R2, ZREG_R0 } else { - if (var2 == (uint32_t)-1) { - | ZVAL_COPY_VALUE Ra(var_reg)+var_offset, FP + val.var, val_info, r2, edx, r0 - } else { - | ZVAL_COPY_VALUE_2 Ra(var_reg)+var_offset, FP + var2, FP + val.var, val_info, r2, edx, r0 - } + | ZVAL_COPY_VALUE_2 var_addr, res_addr, val_addr, val_info, ZREG_R2, ZREG_R0 } + if (val_type == IS_CV) { - if (var2 == (uint32_t)-1) { + if (!res_addr) { | TRY_ADDREF val_info, dh, r0 } else { | TRY_ADDREF_2 val_info, dh, r0 } } else { - if (var2 != (uint32_t)-1) { + if (res_addr) { | TRY_ADDREF val_info, dh, r0 } } @@ -3210,35 +3028,36 @@ static int zend_jit_assign_to_variable(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, - uint32_t var_reg, - uint32_t var_offset, + zend_jit_addr var_addr, uint32_t var_info, zend_uchar val_type, znode_op val, + zend_jit_addr val_addr, uint32_t val_info, - uint32_t var2) + zend_jit_addr res_addr) /* Labels: 1,2,3,4,5 */ { + ZEND_ASSERT(Z_MODE(var_addr) == IS_MEM_ZVAL); if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { int in_cold = 0; - ZEND_ASSERT(var_reg != ZREG_R0); + ZEND_ASSERT(Z_REG(var_addr) != ZREG_R0); if (var_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { - | IF_Z_REFCOUNTED Ra(var_reg)+var_offset, >1 + | IF_ZVAL_REFCOUNTED var_addr, >1 |.cold_code |1: in_cold = 1; } | // TODO: support for object->set | // TODO: support for assignment to itself - | GET_Z_PTR r0, Ra(var_reg)+var_offset + | GET_ZVAL_PTR r0, var_addr | GC_DELREF r0 if (RC_MAY_BE_1(var_info)) { if (RC_MAY_BE_N(var_info)) { | jnz >4 } | mov aword [r4], r0 // save - if (!zend_jit_simple_assign(Dst, opline, op_array, ssa, var_reg, var_offset, var_info, val_type, val, val_info, var2, in_cold)) { + if (!zend_jit_simple_assign(Dst, opline, op_array, ssa, var_addr, var_info, val_type, val, val_addr, val_info, res_addr, in_cold)) { return 0; } | mov FCARG1a, aword [r4] // restore @@ -3247,22 +3066,22 @@ static int zend_jit_assign_to_variable(dasm_State **Dst, |4: } if (RC_MAY_BE_N(var_info)) { - if (var_reg == ZREG_FP) { - | GET_Z_PTR FCARG1a, Ra(var_reg)+var_offset + if (Z_REG(var_addr) == ZREG_FP) { + | GET_ZVAL_PTR FCARG1a, var_addr | IF_GC_MAY_NOT_LEAK FCARG1a, eax, >5 - } else if (var_reg != ZREG_FCARG1a) { - | GET_Z_PTR FCARG1a, Ra(var_reg)+var_offset + } else if (Z_REG(var_addr) != ZREG_FCARG1a) { + | GET_ZVAL_PTR FCARG1a, var_addr | IF_GC_MAY_NOT_LEAK FCARG1a, eax, >5 - | mov [r4], Ra(var_reg) // save + | mov [r4], Ra(Z_REG(var_addr)) // save } else { - | GET_Z_PTR r0, Ra(var_reg)+var_offset + | GET_ZVAL_PTR r0, var_addr | IF_GC_MAY_NOT_LEAK r0, eax, >5 - | mov [r4], Ra(var_reg) // save - | GET_Z_PTR FCARG1a, Ra(var_reg)+var_offset + | mov [r4], Ra(Z_REG(var_addr)) // save + | GET_ZVAL_PTR FCARG1a, var_addr } | EXT_CALL gc_possible_root, r0 - if (var_reg != ZREG_FP) { - | mov Ra(var_reg), [r4] // restore + if (Z_REG(var_addr) != ZREG_FP) { + | mov Ra(Z_REG(var_addr)), [r4] // restore } if (in_cold) { | jmp >5 @@ -3274,7 +3093,7 @@ static int zend_jit_assign_to_variable(dasm_State **Dst, |5: } - if (!zend_jit_simple_assign(Dst, opline, op_array, ssa, var_reg, var_offset, var_info, val_type, val, val_info, var2, 0)) { + if (!zend_jit_simple_assign(Dst, opline, op_array, ssa, var_addr, var_info, val_type, val, val_addr, val_info, res_addr, 0)) { return 0; } |3: @@ -3285,6 +3104,7 @@ static int zend_jit_assign_to_variable(dasm_State **Dst, static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) { uint32_t op1_info, op2_info, val_info; + zend_jit_addr op1_addr, op2_addr, op3_addr, res_addr; if (opline->op1_type != IS_CV) { goto fallback; @@ -3298,71 +3118,67 @@ static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, zend_op_ op2_info = OP2_INFO(); val_info = OP1_DATA_INFO(); + op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1); + op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2); + op3_addr = zend_jit_decode_op(op_array, (opline+1)->op1_type, (opline+1)->op1); + if (opline->result_type == IS_UNUSED) { + res_addr = 0; + } else { + res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result); + } + if (op1_info & MAY_BE_REF) { - | LOAD_ZVAL_ADDR FCARG1a, opline->op1_type, opline->op1 + | LOAD_ZVAL_ADDR FCARG1a, op1_addr | ZVAL_DEREF FCARG1a, op1_info + op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); } if (op1_info & MAY_BE_ARRAY) { - if (op1_info & MAY_BE_REF) { - | IF_NOT_Z_TYPE FCARG1a, IS_ARRAY, >7 - | SEPARATE_ARRAY_REG op1_info, 1, op_array->filename, opline->lineno - } else { - if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { - | IF_NOT_Z_TYPE FP + opline->op1.var, IS_ARRAY, >7 - } - | SEPARATE_ARRAY opline->op1_type, opline->op1, op1_info, 1, op_array->filename, opline->lineno + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7 } + | SEPARATE_ARRAY op1_addr, op1_info, 1, op_array->filename, opline->lineno } else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE)) { if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { - if (op1_info & MAY_BE_REF) { - | cmp byte [FCARG1a + 8], IS_FALSE - } else { - | cmp byte [FP + opline->op1.var + 8], IS_FALSE - } + | CMP_ZVAL_TYPE op1_addr, IS_FALSE | jg >7 } | // ZVAL_NEW_ARR(container); | // zend_hash_init(Z_ARRVAL_P(container), 8, NULL, ZVAL_PTR_DTOR, 0); - if (op1_info & MAY_BE_REF) { - | mov [r4], FCARG1a // save + if (Z_REG(op1_addr) != ZREG_FP) { + | mov [r4], Ra(Z_REG(op1_addr)) // save + } + if (ZEND_DEBUG) { + const char *filename = op_array->filename ? op_array->filename->val : NULL; + |.if X64 + | LOAD_ADDR CARG3, filename + | xor CARG4d, opline->lineno + |.else + | push opline->lineno + | push filename + |.endif + } else { + |.if not X64 + | sub r4, 8 + |.endif } - || if (ZEND_DEBUG) { - || const char *filename = op_array->filename ? op_array->filename->val : NULL; - |.if X64 - | LOAD_ADDR CARG3, filename - | xor CARG4d, opline->lineno - |.else - | push opline->lineno - | push filename - |.endif - || } else { - |.if not X64 - | sub r4, 8 - |.endif - || } |.if X64 - | LOAD_ADDR CARG2, 8 - || if (!(op1_info & MAY_BE_REF)) { - | LOAD_ZVAL_ADDR FCARG1a, opline->op1_type, opline->op1 - || } + | LOAD_ADDR CARG2, 8 + if (Z_REG(op1_addr) != ZREG_FCARG1a) { + | LOAD_ZVAL_ADDR FCARG1a, op1_addr + } |.else - | push 8 - || if (!(op1_info & MAY_BE_REF)) { - | LOAD_ZVAL_ADDR FCARG1a, opline->op1_type, opline->op1 - || } - | push FCARG1a + | push 8 + | PUSH_ZVAL_ADDR op1_addr, r0 |.endif | EXT_CALL _array_init, r0 |.if not X64 - | add r4, 16 + | add r4, 16 |.endif - if (op1_info & MAY_BE_REF) { - | mov FCARG1a, [r4] // restore - | GET_Z_PTR FCARG1a, FCARG1a - } else { - | LONG_LOAD FCARG1a, opline->op1_type, opline->op1 + if (Z_REG(op1_addr) != ZREG_FP) { + | mov Ra(Z_REG(op1_addr)), [r4] // restore } + | GET_ZVAL_LVAL FCARG1a, op1_addr } if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { @@ -3370,16 +3186,16 @@ static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, zend_op_ if (opline->op2_type == IS_UNUSED) { | // var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval)); | LOAD_ADDR FCARG2a, &EG(uninitialized_zval) - || if (ZEND_DEBUG) { - || const char *filename = op_array->filename ? op_array->filename->val : NULL; - |.if X64 - | LOAD_ADDR CARG3, filename - | mov CARG4d, opline->lineno - |.else - | push opline->lineno - | push filename - |.endif - || } + if (ZEND_DEBUG) { + const char *filename = op_array->filename ? op_array->filename->val : NULL; + |.if X64 + | LOAD_ADDR CARG3, filename + | mov CARG4d, opline->lineno + |.else + | push opline->lineno + | push filename + |.endif + } | EXT_CALL _zend_hash_next_index_insert, r0 | // if (UNEXPECTED(!var_ptr)) { | test r0, r0 @@ -3403,18 +3219,18 @@ static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, zend_op_ if (opline->op2_type == IS_UNUSED) { uint32_t var_info = zend_array_element_type(op1_info, 0, 0); + zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); - if (!zend_jit_simple_assign(Dst, opline, op_array, ssa, ZREG_FCARG1a, 0, var_info, (opline+1)->op1_type, (opline+1)->op1, val_info, - opline->result_type == IS_UNUSED ? -1 : opline->result.var, 0)) { + if (!zend_jit_simple_assign(Dst, opline, op_array, ssa, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, op3_addr, val_info, res_addr, 0)) { return 0; } } else { uint32_t var_info = zend_array_element_type(op1_info, 0, 0); + zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); | // value = zend_assign_to_variable(variable_ptr, value, OP_DATA_TYPE); | ZVAL_DEREF FCARG1a, var_info - if (!zend_jit_assign_to_variable(Dst, opline, op_array, ssa, ZREG_FCARG1a, 0, var_info, (opline+1)->op1_type, (opline+1)->op1, val_info, - opline->result_type == IS_UNUSED ? -1 : opline->result.var)) { + if (!zend_jit_assign_to_variable(Dst, opline, op_array, ssa, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, op3_addr, val_info, res_addr)) { return 0; } } @@ -3431,54 +3247,45 @@ static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, zend_op_ if ((op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE)) && (op1_info & MAY_BE_ARRAY)) { if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { - if (op1_info & MAY_BE_REF) { - | cmp byte [FCARG1a + 8], IS_FALSE - } else { - | cmp byte [FP + opline->op1.var + 8], IS_FALSE - } + | CMP_ZVAL_TYPE op1_addr, IS_FALSE | jg >2 } | // ZVAL_NEW_ARR(container); | // zend_hash_init(Z_ARRVAL_P(container), 8, NULL, ZVAL_PTR_DTOR, 0); - if (op1_info & MAY_BE_REF) { - | mov [r4], FCARG1a // save + if (Z_REG(op1_addr) != ZREG_FP) { + | mov [r4], Ra(Z_REG(op1_addr)) // save + } + if (ZEND_DEBUG) { + const char *filename = op_array->filename ? op_array->filename->val : NULL; + |.if X64 + | LOAD_ADDR CARG3, filename + | xor CARG4d, opline->lineno + |.else + | push opline->lineno + | push filename + |.endif + } else { + |.if not X64 + | sub r4, 8 + |.endif } - || if (ZEND_DEBUG) { - || const char *filename = op_array->filename ? op_array->filename->val : NULL; - |.if X64 - | LOAD_ADDR CARG3, filename - | xor CARG4d, opline->lineno - |.else - | push opline->lineno - | push filename - |.endif - || } else { - |.if not X64 - | sub r4, 8 - |.endif - || } |.if X64 - | LOAD_ADDR CARG2, 8 - || if (!(op1_info & MAY_BE_REF)) { - | LOAD_ZVAL_ADDR FCARG1a, opline->op1_type, opline->op1 - || } + | LOAD_ADDR CARG2, 8 + if (Z_REG(op1_addr) != ZREG_FCARG1a) { + | LOAD_ZVAL_ADDR FCARG1a, op1_addr + } |.else - | push 8 - || if (!(op1_info & MAY_BE_REF)) { - | LOAD_ZVAL_ADDR FCARG1a, opline->op1_type, opline->op1 - || } - | push FCARG1a + | push 8 + | PUSH_ZVAL_ADDR op1_addr, r0 |.endif | EXT_CALL _array_init, r0 |.if not X64 - | add r4, 16 + | add r4, 16 |.endif - if (op1_info & MAY_BE_REF) { - | mov FCARG1a, [r4] // restore - | GET_Z_PTR FCARG1a, FCARG1a - } else { - | LONG_LOAD FCARG1a, opline->op1_type, opline->op1 + if (Z_REG(op1_addr) != ZREG_FP) { + | mov Ra(Z_REG(op1_addr)), [r4] // restore } + | GET_ZVAL_LVAL FCARG1a, op1_addr | // ZEND_VM_C_GOTO(assign_dim_op_new_array); | jmp <6 |2: @@ -3486,31 +3293,34 @@ static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, zend_op_ if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { | SAVE_VALID_OPLINE opline - if (!(op1_info & MAY_BE_REF)) { - | LOAD_ZVAL_ADDR FCARG1a, opline->op1_type, opline->op1 + if (Z_REG(op1_addr) != ZREG_FCARG1a) { + | LOAD_ZVAL_ADDR FCARG1a, op1_addr } if (opline->op2_type == IS_UNUSED) { | xor FCARG2a, FCARG2a } else { - | LOAD_ZVAL_ADDR FCARG2a, opline->op2_type, opline->op2 + | LOAD_ZVAL_ADDR FCARG2a, op2_addr } if (opline->result_type == IS_UNUSED) { |.if X64 - | xor CARG4, CARG4 + | xor CARG4, CARG4 |.else - | push 0 + | push 0 |.endif } else { + zend_jit_addr res_addr; + + res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result); |.if X64 - | LOAD_ZVAL_ADDR CARG4, opline->result_type, opline->result + | LOAD_ZVAL_ADDR CARG4, res_addr |.else - | PUSH_ZVAL_ADDR opline->result_type, opline->result, r0 + | PUSH_ZVAL_ADDR res_addr, r0 |.endif } |.if X64 - | LOAD_ZVAL_ADDR CARG3, (opline+1)->op1_type, (opline+1)->op1 + | LOAD_ZVAL_ADDR CARG3, op3_addr |.else - | PUSH_ZVAL_ADDR (opline+1)->op1_type, (opline+1)->op1, r0 + | PUSH_ZVAL_ADDR op3_addr, r0 |.endif | EXT_CALL zend_jit_assign_dim_helper, r0 @@ -3555,6 +3365,7 @@ fallback: static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) { uint32_t op1_info, op2_info; + zend_jit_addr op1_addr, op2_addr, op3_addr, var_addr; if (opline->op1_type != IS_CV || opline->result_type != IS_UNUSED) { goto fallback; @@ -3567,33 +3378,29 @@ static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, zend_ op1_info = OP1_INFO(); op2_info = OP2_INFO(); + op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1); + op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2); + op3_addr = zend_jit_decode_op(op_array, (opline+1)->op1_type, (opline+1)->op1); + if (op1_info & MAY_BE_REF) { - | LOAD_ZVAL_ADDR FCARG1a, opline->op1_type, opline->op1 + | LOAD_ZVAL_ADDR FCARG1a, op1_addr | ZVAL_DEREF FCARG1a, op1_info + op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); } if (op1_info & MAY_BE_ARRAY) { - if (op1_info & MAY_BE_REF) { - | IF_NOT_Z_TYPE FCARG1a, IS_ARRAY, >7 - | SEPARATE_ARRAY_REG op1_info, 1, op_array->filename, opline->lineno - } else { - if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { - | IF_NOT_Z_TYPE FP + opline->op1.var, IS_ARRAY, >7 - } - | SEPARATE_ARRAY opline->op1_type, opline->op1, op1_info, 1, op_array->filename, opline->lineno + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7 } + | SEPARATE_ARRAY op1_addr, op1_info, 1, op_array->filename, opline->lineno } else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE)) { if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { - if (op1_info & MAY_BE_REF) { - | cmp byte [FCARG1a + 8], IS_FALSE - } else { - | cmp byte [FP + opline->op1.var + 8], IS_FALSE - } + | CMP_ZVAL_TYPE op1_addr, IS_FALSE | jg >7 } if (op1_info & MAY_BE_UNDEF) { if (op1_info & (MAY_BE_NULL|MAY_BE_FALSE)) { - | IF_NOT_Z_TYPE FP + opline->op1.var, IS_UNDEF, >1 + | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1 } | SAVE_VALID_OPLINE opline | mov FCARG1a, opline->op1.var @@ -3602,45 +3409,40 @@ static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, zend_ } | // ZVAL_NEW_ARR(container); | // zend_hash_init(Z_ARRVAL_P(container), 8, NULL, ZVAL_PTR_DTOR, 0); - if (op1_info & MAY_BE_REF) { - | mov [r4], FCARG1a // save + if (Z_REG(op1_addr) != ZREG_FP) { + | mov [r4], Ra(Z_REG(op1_addr)) // save } - || if (ZEND_DEBUG) { - || const char *filename = op_array->filename ? op_array->filename->val : NULL; - |.if X64 - | LOAD_ADDR CARG3, filename - | xor CARG4d, opline->lineno - |.else - | push opline->lineno - | push filename - |.endif - || } else { - |.if not X64 - | sub r4, 8 - |.endif - || } + if (ZEND_DEBUG) { + const char *filename = op_array->filename ? op_array->filename->val : NULL; + |.if X64 + | LOAD_ADDR CARG3, filename + | xor CARG4d, opline->lineno + |.else + | push opline->lineno + | push filename + |.endif + } else { + |.if not X64 + | sub r4, 8 + |.endif + } |.if X64 - | LOAD_ADDR CARG2, 8 - || if (!(op1_info & MAY_BE_REF)) { - | LOAD_ZVAL_ADDR FCARG1a, opline->op1_type, opline->op1 - || } + | LOAD_ADDR CARG2, 8 + if (Z_REG(op1_addr) != ZREG_FCARG1a) { + | LOAD_ZVAL_ADDR FCARG1a, op1_addr + } |.else - | push 8 - || if (!(op1_info & MAY_BE_REF)) { - | LOAD_ZVAL_ADDR FCARG1a, opline->op1_type, opline->op1 - || } - | push FCARG1a + | push 8 + | PUSH_ZVAL_ADDR op1_addr, r0 |.endif | EXT_CALL _array_init, r0 |.if not X64 - | add r4, 16 + | add r4, 16 |.endif - if (op1_info & MAY_BE_REF) { - | mov FCARG1a, [r4] // restore - | GET_Z_PTR FCARG1a, FCARG1a - } else { - | LONG_LOAD FCARG1a, opline->op1_type, opline->op1 + if (Z_REG(op1_addr) != ZREG_FP) { + | mov Ra(Z_REG(op1_addr)), [r4] // restore } + | GET_ZVAL_LVAL FCARG1a, op1_addr } if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { @@ -3650,16 +3452,16 @@ static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, zend_ if (opline->op2_type == IS_UNUSED) { | // var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval)); | LOAD_ADDR FCARG2a, &EG(uninitialized_zval) - || if (ZEND_DEBUG) { - || const char *filename = op_array->filename ? op_array->filename->val : NULL; - |.if X64 - | LOAD_ADDR CARG3, filename - | mov CARG4d, opline->lineno - |.else - | push opline->lineno - | push filename - |.endif - || } + if (ZEND_DEBUG) { + const char *filename = op_array->filename ? op_array->filename->val : NULL; + |.if X64 + | LOAD_ADDR CARG3, filename + | mov CARG4d, opline->lineno + |.else + | push opline->lineno + | push filename + |.endif + } | EXT_CALL _zend_hash_next_index_insert, r0 | // if (UNEXPECTED(!var_ptr)) { | test r0, r0 @@ -3684,17 +3486,18 @@ static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, zend_ } } + var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); switch (opline->opcode) { case ZEND_ASSIGN_ADD: case ZEND_ASSIGN_SUB: case ZEND_ASSIGN_MUL: case ZEND_ASSIGN_DIV: - if (!zend_jit_math_helper(Dst, opline, op_array, ssa, IS_CV, opline->op1, ZREG_FCARG1a, 0, var_info, (opline+1)->op1_type, (opline+1)->op1, ZREG_FP, (opline+1)->op1.var, OP1_DATA_INFO(), ZREG_FCARG1a, 0, OP1_DEF_INFO(), 1)) { + if (!zend_jit_math_helper(Dst, opline, op_array, ssa, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, op3_addr, OP1_DATA_INFO(), var_addr, OP1_DEF_INFO(), 1)) { return 0; } break; case ZEND_ASSIGN_CONCAT: - if (!zend_jit_concat_helper(Dst, opline, op_array, ssa, IS_CV, opline->op1, ZREG_FCARG1a, 0, var_info, (opline+1)->op1_type, (opline+1)->op1, ZREG_FP, (opline+1)->op1.var, OP1_DATA_INFO(), ZREG_FCARG1a, 0, OP1_DEF_INFO())) { + if (!zend_jit_concat_helper(Dst, opline, op_array, ssa, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, op3_addr, OP1_DATA_INFO(), var_addr, OP1_DEF_INFO())) { return 0; } break; @@ -3715,16 +3518,12 @@ static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, zend_ (op1_info & MAY_BE_ARRAY)) { if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { if (op1_info & (MAY_BE_NULL|MAY_BE_FALSE)) { - if (op1_info & MAY_BE_REF) { - | cmp byte [FCARG1a + 8], IS_FALSE - } else { - | cmp byte [FP + opline->op1.var + 8], IS_FALSE - } + | CMP_ZVAL_TYPE op1_addr, IS_FALSE | jg >2 } } if (op1_info & MAY_BE_UNDEF) { - | IF_NOT_Z_TYPE FP + opline->op1.var, IS_UNDEF, >1 + | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1 | SAVE_VALID_OPLINE opline | mov FCARG1a, opline->op1.var | EXT_CALL zend_jit_undefined_op_helper, r0 @@ -3732,45 +3531,40 @@ static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, zend_ } | // ZVAL_NEW_ARR(container); | // zend_hash_init(Z_ARRVAL_P(container), 8, NULL, ZVAL_PTR_DTOR, 0); - if (op1_info & MAY_BE_REF) { - | mov [r4], FCARG1a // save + if (Z_REG(op1_addr) != ZREG_FP) { + | mov [r4], Ra(Z_REG(op1_addr)) // save + } + if (ZEND_DEBUG) { + const char *filename = op_array->filename ? op_array->filename->val : NULL; + |.if X64 + | LOAD_ADDR CARG3, filename + | xor CARG4d, opline->lineno + |.else + | push opline->lineno + | push filename + |.endif + } else { + |.if not X64 + | sub r4, 8 + |.endif } - || if (ZEND_DEBUG) { - || const char *filename = op_array->filename ? op_array->filename->val : NULL; - |.if X64 - | LOAD_ADDR CARG3, filename - | xor CARG4d, opline->lineno - |.else - | push opline->lineno - | push filename - |.endif - || } else { - |.if not X64 - | sub r4, 8 - |.endif - || } |.if X64 - | LOAD_ADDR CARG2, 8 - || if (!(op1_info & MAY_BE_REF)) { - | LOAD_ZVAL_ADDR FCARG1a, opline->op1_type, opline->op1 - || } + | LOAD_ADDR CARG2, 8 + if (Z_REG(op1_addr) != ZREG_FCARG1a) { + | LOAD_ZVAL_ADDR FCARG1a, op1_addr + } |.else - | push 8 - || if (!(op1_info & MAY_BE_REF)) { - | LOAD_ZVAL_ADDR FCARG1a, opline->op1_type, opline->op1 - || } - | push FCARG1a + | push 8 + | PUSH_ZVAL_ADDR op1_addr, r0 |.endif | EXT_CALL _array_init, r0 |.if not X64 - | add r4, 16 + | add r4, 16 |.endif - if (op1_info & MAY_BE_REF) { - | mov FCARG1a, [r4] // restore - | GET_Z_PTR FCARG1a, FCARG1a - } else { - | LONG_LOAD FCARG1a, opline->op1_type, opline->op1 + if (Z_REG(op1_addr) != ZREG_FP) { + | mov Ra(Z_REG(op1_addr)), [r4] // restore } + | GET_ZVAL_LVAL FCARG1a, op1_addr | // ZEND_VM_C_GOTO(assign_dim_op_new_array); | jmp <6 |2: @@ -3778,18 +3572,18 @@ static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, zend_ if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { | SAVE_VALID_OPLINE opline - if (!(op1_info & MAY_BE_REF)) { - | LOAD_ZVAL_ADDR FCARG1a, opline->op1_type, opline->op1 + if (Z_REG(op1_addr) != ZREG_FCARG1a) { + | LOAD_ZVAL_ADDR FCARG1a, op1_addr } if (opline->op2_type == IS_UNUSED) { | xor FCARG2a, FCARG2a } else { - | LOAD_ZVAL_ADDR FCARG2a, opline->op2_type, opline->op2 + | LOAD_ZVAL_ADDR FCARG2a, op2_addr } |.if X64 - | LOAD_ZVAL_ADDR CARG3, (opline+1)->op1_type, (opline+1)->op1 + | LOAD_ZVAL_ADDR CARG3, op3_addr |.else - | PUSH_ZVAL_ADDR (opline+1)->op1_type, (opline+1)->op1, r0 + | PUSH_ZVAL_ADDR op3_addr, r0 |.endif switch (opline->opcode) { case ZEND_ASSIGN_ADD: @@ -3831,6 +3625,7 @@ fallback: static int zend_jit_assign_op(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) { uint32_t op1_info, op2_info; + zend_jit_addr op1_addr, op2_addr; if (opline->extended_value == ZEND_ASSIGN_DIM) { return zend_jit_assign_dim_op(Dst, opline, op_array, ssa); @@ -3849,6 +3644,9 @@ static int zend_jit_assign_op(dasm_State **Dst, const zend_op *opline, zend_op_a op1_info = OP1_INFO(); op2_info = OP2_INFO(); + op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1); + op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2); + if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) { goto fallback; } @@ -3874,31 +3672,21 @@ static int zend_jit_assign_op(dasm_State **Dst, const zend_op *opline, zend_op_a } if (op1_info & MAY_BE_REF) { - | LOAD_ZVAL_ADDR FCARG1a, opline->op1_type, opline->op1 + | LOAD_ZVAL_ADDR FCARG1a, op1_addr | ZVAL_DEREF FCARG1a, op1_info - switch (opline->opcode) { - case ZEND_ASSIGN_ADD: - case ZEND_ASSIGN_SUB: - case ZEND_ASSIGN_MUL: - case ZEND_ASSIGN_DIV: - return zend_jit_math_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, ZREG_FCARG1a, 0, op1_info, opline->op2_type, opline->op2, ZREG_FP, opline->op2.var, op2_info, ZREG_FCARG1a, 0, OP1_DEF_INFO(), 1); - case ZEND_ASSIGN_CONCAT: - return zend_jit_concat_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, ZREG_FCARG1a, 0, op1_info, opline->op2_type, opline->op2, ZREG_FP, opline->op2.var, op2_info, ZREG_FCARG1a, 0, OP1_DEF_INFO()); - default: - ZEND_ASSERT(0); - } - } else { - switch (opline->opcode) { - case ZEND_ASSIGN_ADD: - case ZEND_ASSIGN_SUB: - case ZEND_ASSIGN_MUL: - case ZEND_ASSIGN_DIV: - return zend_jit_math_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, ZREG_FP, opline->op1.var, op1_info, opline->op2_type, opline->op2, ZREG_FP, opline->op2.var, op2_info, ZREG_FP, opline->op1.var, OP1_DEF_INFO(), 1); - case ZEND_ASSIGN_CONCAT: - return zend_jit_concat_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, ZREG_FP, opline->op1.var, op1_info, opline->op2_type, opline->op2, ZREG_FP, opline->op2.var, op2_info, ZREG_FP, opline->op1.var, OP1_DEF_INFO()); - default: - ZEND_ASSERT(0); - } + op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); + } + + switch (opline->opcode) { + case ZEND_ASSIGN_ADD: + case ZEND_ASSIGN_SUB: + case ZEND_ASSIGN_MUL: + case ZEND_ASSIGN_DIV: + return zend_jit_math_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, op1_addr, OP1_DEF_INFO(), 1); + case ZEND_ASSIGN_CONCAT: + return zend_jit_concat_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, op1_addr, OP1_DEF_INFO()); + default: + ZEND_ASSERT(0); } fallback: @@ -3910,42 +3698,24 @@ static int zend_jit_cmp_long_long(dasm_State **Dst, const zend_op *opline, int b { unsigned int target_label; int swap = 0; + zend_jit_addr op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1); + zend_jit_addr op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2); - if (opline->op1_type == IS_CONST && opline->op2_type != IS_CONST) { - zend_long val = Z_LVAL_P(RT_CONSTANT(op_array, opline->op1)); - - | .if X64 - || if (!IS_SIGNED_32BIT(val)) { - | mov64 r0, val - | cmp aword [FP + opline->op2.var], r0 - || } else { - | cmp aword [FP + opline->op2.var], val - || } - | .else - | cmp aword [FP + opline->op2.var], val - | .endif + if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) != IS_CONST_ZVAL) { + | LONG_OP_MEM_CONST cmp, op2_addr, Z_LVAL_P(Z_ZV(op1_addr)) swap = 1; - } else if (opline->op2_type == IS_CONST && opline->op1_type != IS_CONST) { - zend_long val = Z_LVAL_P(RT_CONSTANT(op_array, opline->op2)); - - | .if X64 - || if (!IS_SIGNED_32BIT(val)) { - | mov64 r0, val - | cmp aword [FP + opline->op1.var], r0 - || } else { - | cmp aword [FP + opline->op1.var], val - || } - | .else - | cmp aword [FP + opline->op1.var], val - | .endif + } else if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_MODE(op1_addr) != IS_CONST_ZVAL) { + | LONG_OP_MEM_CONST cmp, op1_addr, Z_LVAL_P(Z_ZV(op2_addr)) } else { - | LONG_LOAD r0, opline->op1_type, opline->op1 - | LONG_OP cmp, r0, opline->op2_type, opline->op2 + | GET_ZVAL_LVAL r0, op1_addr + | LONG_OP cmp, r0, op2_addr } if (((opline+1)->opcode == ZEND_JMPZ_EX || (opline+1)->opcode == ZEND_JMPNZ_EX) && (opline+1)->op1_type == IS_TMP_VAR && (opline+1)->op1.var == opline->result.var) { + zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result); + switch (opline->opcode) { case ZEND_IS_EQUAL: case ZEND_CASE: @@ -3973,7 +3743,7 @@ static int zend_jit_cmp_long_long(dasm_State **Dst, const zend_op *opline, int b } | movzx eax, al | lea eax, [eax + 2] - | SET_Z_TYPE_INFO FP + opline->result.var, eax + | SET_ZVAL_TYPE_INFO res_addr, eax } if (((opline+1)->opcode == ZEND_JMPZ || (opline+1)->opcode == ZEND_JMPZ_EX) && @@ -4067,6 +3837,8 @@ static int zend_jit_cmp_long_long(dasm_State **Dst, const zend_op *opline, int b target_label = ssa->cfg.blocks[b].successors[1]; | jmp => target_label } else { + zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result); + switch (opline->opcode) { case ZEND_IS_EQUAL: case ZEND_CASE: @@ -4094,7 +3866,7 @@ static int zend_jit_cmp_long_long(dasm_State **Dst, const zend_op *opline, int b } | movzx eax, al | add eax, 2 - | SET_Z_TYPE_INFO FP + opline->result.var, eax + | SET_ZVAL_TYPE_INFO res_addr, eax } return 1; @@ -4184,35 +3956,37 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, i } else if ((opline+1)->opcode == ZEND_JMPZ_EX && (opline+1)->op1_type == IS_TMP_VAR && (opline+1)->op1.var == opline->result.var) { + zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result); + target_label = ssa->cfg.blocks[b].successors[0]; switch (opline->opcode) { case ZEND_IS_EQUAL: case ZEND_CASE: - | SET_Z_TYPE_INFO FP + opline->result.var, IS_FALSE + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE | jp >1 | jne => target_label - | SET_Z_TYPE_INFO FP + opline->result.var, IS_TRUE + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE |1: break; case ZEND_IS_NOT_EQUAL: | jp >1 - | SET_Z_TYPE_INFO FP + opline->result.var, IS_TRUE + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE | je => target_label |1: - | SET_Z_TYPE_INFO FP + opline->result.var, IS_FALSE + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE break; case ZEND_IS_SMALLER: | setnae al | movzx eax, al | lea eax, [eax + 2] - | SET_Z_TYPE_INFO FP + opline->result.var, eax + | SET_ZVAL_TYPE_INFO res_addr, eax | jae => target_label break; case ZEND_IS_SMALLER_OR_EQUAL: | setna al | movzx eax, al | lea eax, [eax + 2] - | SET_Z_TYPE_INFO FP + opline->result.var, eax + | SET_ZVAL_TYPE_INFO res_addr, eax | ja => target_label break; default: @@ -4221,41 +3995,45 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, i } else if ((opline+1)->opcode == ZEND_JMPNZ_EX && (opline+1)->op1_type == IS_TMP_VAR && (opline+1)->op1.var == opline->result.var) { + zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result); + target_label = ssa->cfg.blocks[b].successors[0]; switch (opline->opcode) { case ZEND_IS_EQUAL: case ZEND_CASE: | jp >1 - | SET_Z_TYPE_INFO FP + opline->result.var, IS_TRUE + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE | je => target_label |1: - | SET_Z_TYPE_INFO FP + opline->result.var, IS_FALSE + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE break; case ZEND_IS_NOT_EQUAL: - | SET_Z_TYPE_INFO FP + opline->result.var, IS_TRUE + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE | jp >1 | jne => target_label - | SET_Z_TYPE_INFO FP + opline->result.var, IS_FALSE + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE |1: break; case ZEND_IS_SMALLER: | setnae al | movzx eax, al | lea eax, [eax + 2] - | SET_Z_TYPE_INFO FP + opline->result.var, eax + | SET_ZVAL_TYPE_INFO res_addr, eax | jnae => target_label break; case ZEND_IS_SMALLER_OR_EQUAL: | setna al | movzx eax, al | lea eax, [eax + 2] - | SET_Z_TYPE_INFO FP + opline->result.var, eax + | SET_ZVAL_TYPE_INFO res_addr, eax | jna => target_label break; default: ZEND_ASSERT(0); } } else { + zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result); + switch (opline->opcode) { case ZEND_IS_EQUAL: case ZEND_CASE: @@ -4287,7 +4065,7 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, i default: ZEND_ASSERT(0); } - | SET_Z_TYPE_INFO FP + opline->result.var, eax + | SET_ZVAL_TYPE_INFO res_addr, eax } return 1; @@ -4295,16 +4073,19 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, i static int zend_jit_cmp_long_double(dasm_State **Dst, const zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa) { + zend_jit_addr op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1); + zend_jit_addr op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2); + |.if X64 or SSE - || zend_reg tmp_reg = ZREG_XMM1; - || - | SSE_LOAD_LONG tmp_reg, opline->op1_type, opline->op1 - | SSE_OP ucomisd, tmp_reg, opline->op2_type, opline->op2 + zend_reg tmp_reg = ZREG_XMM0; + + | SSE_GET_ZVAL_LVAL tmp_reg, op1_addr + | SSE_OP ucomisd, tmp_reg, op2_addr |.else - | FPU_LOAD opline->op2_type, opline->op2 - | FPU_LONG_LOAD opline->op1_type, opline->op1 - | fucomip st1 - | fstp st0 + | FPU_GET_ZVAL_DVAL op2_addr + | FPU_GET_ZVAL_LVAL op1_addr + | fucomip st1 + | fstp st0 |.endif return zend_jit_cmp_double_common(Dst, opline, b, op_array, ssa); @@ -4312,18 +4093,21 @@ static int zend_jit_cmp_long_double(dasm_State **Dst, const zend_op *opline, int static int zend_jit_cmp_double_long(dasm_State **Dst, const zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa) { + zend_jit_addr op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1); + zend_jit_addr op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2); + |.if X64 or SSE zend_reg tmp_reg1 = ZREG_XMM0; zend_reg tmp_reg2 = ZREG_XMM1; - | SSE_LOAD tmp_reg1, opline->op1_type, opline->op1 - | SSE_LOAD_LONG tmp_reg2, opline->op2_type, opline->op2 - | ucomisd xmm(tmp_reg1-ZREG_XMM0), xmm(tmp_reg2-ZREG_XMM0) + | SSE_GET_ZVAL_DVAL tmp_reg1, op1_addr + | SSE_GET_ZVAL_LVAL tmp_reg2, op2_addr + | ucomisd xmm(tmp_reg1-ZREG_XMM0), xmm(tmp_reg2-ZREG_XMM0) |.else - | FPU_LONG_LOAD opline->op2_type, opline->op2 - | FPU_LOAD opline->op1_type, opline->op1 - | fucomip st1 - | fstp st0 + | FPU_GET_ZVAL_LVAL op2_addr + | FPU_GET_ZVAL_DVAL op1_addr + | fucomip st1 + | fstp st0 |.endif return zend_jit_cmp_double_common(Dst, opline, b, op_array, ssa); @@ -4331,16 +4115,19 @@ static int zend_jit_cmp_double_long(dasm_State **Dst, const zend_op *opline, int static int zend_jit_cmp_double_double(dasm_State **Dst, const zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa) { + zend_jit_addr op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1); + zend_jit_addr op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2); + |.if X64 or SSE zend_reg tmp_reg = ZREG_XMM0; - | SSE_LOAD tmp_reg, opline->op1_type, opline->op1 - | SSE_OP ucomisd, tmp_reg, opline->op2_type, opline->op2 + | SSE_GET_ZVAL_DVAL tmp_reg, op1_addr + | SSE_OP ucomisd, tmp_reg, op2_addr |.else - | FPU_LOAD opline->op2_type, opline->op2 - | FPU_LOAD opline->op1_type, opline->op1 - | fucomip st1 - | fstp st0 + | FPU_GET_ZVAL_DVAL op2_addr + | FPU_GET_ZVAL_DVAL op1_addr + | fucomip st1 + | fstp st0 |.endif return zend_jit_cmp_double_common(Dst, opline, b, op_array, ssa); @@ -4349,8 +4136,9 @@ static int zend_jit_cmp_double_double(dasm_State **Dst, const zend_op *opline, i static int zend_jit_cmp_slow(dasm_State **Dst, const zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa) { unsigned int target_label; + zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result); - | cmp aword [FP + opline->result.var], 0 + | LONG_OP_MEM_CONST cmp, res_addr, Z_L(0) if (((opline+1)->opcode == ZEND_JMPZ_EX || (opline+1)->opcode == ZEND_JMPNZ_EX) && (opline+1)->op1_type == IS_TMP_VAR && @@ -4374,7 +4162,7 @@ static int zend_jit_cmp_slow(dasm_State **Dst, const zend_op *opline, int b, zen } | movzx eax, al | lea eax, [eax + 2] - | SET_Z_TYPE_INFO FP + opline->result.var, eax + | SET_ZVAL_TYPE_INFO res_addr, eax } if (((opline+1)->opcode == ZEND_JMPZ || (opline+1)->opcode == ZEND_JMPZ_EX) && @@ -4463,7 +4251,7 @@ static int zend_jit_cmp_slow(dasm_State **Dst, const zend_op *opline, int b, zen } | movzx eax, al | add eax, 2 - | SET_Z_TYPE_INFO FP + opline->result.var, eax + | SET_ZVAL_TYPE_INFO res_addr, eax } return 1; @@ -4474,6 +4262,9 @@ static int zend_jit_cmp(dasm_State **Dst, const zend_op *opline, int b, int *opn uint32_t op1_info, op2_info; zend_bool same_ops = (opline->op1_type == opline->op2_type) && (opline->op1.var == opline->op2.var); zend_bool has_slow; + zend_jit_addr op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1); + zend_jit_addr op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2); + zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result); op1_info = OP1_INFO(); op2_info = OP2_INFO(); @@ -4487,18 +4278,18 @@ static int zend_jit_cmp(dasm_State **Dst, const zend_op *opline, int b, int *opn if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)) { if (op1_info & (MAY_BE_ANY-MAY_BE_LONG)) { if (op1_info & MAY_BE_DOUBLE) { - | IF_NOT_Z_TYPE FP + opline->op1.var, IS_LONG, >4 + | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >4 } else { - | IF_NOT_Z_TYPE FP + opline->op1.var, IS_LONG, >9 + | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >9 } } if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_LONG))) { if (op2_info & MAY_BE_DOUBLE) { - | IF_NOT_Z_TYPE FP + opline->op2.var, IS_LONG, >3 + | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >3 |.cold_code |3: if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { - | IF_NOT_Z_TYPE FP + opline->op2.var, IS_DOUBLE, >9 + | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9 } if (!zend_jit_cmp_long_double(Dst, opline, b, op_array, ssa)) { return 0; @@ -4506,7 +4297,7 @@ static int zend_jit_cmp(dasm_State **Dst, const zend_op *opline, int b, int *opn | jmp >6 |.code } else { - | IF_NOT_Z_TYPE FP + opline->op2.var, IS_LONG, >9 + | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9 } } if (!zend_jit_cmp_long_long(Dst, opline, b, op_array, ssa)) { @@ -4516,14 +4307,14 @@ static int zend_jit_cmp(dasm_State **Dst, const zend_op *opline, int b, int *opn |.cold_code |4: if (op1_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { - | IF_NOT_Z_TYPE FP + opline->op1.var, IS_DOUBLE, >9 + | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >9 } if (op2_info & MAY_BE_DOUBLE) { if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { if (!same_ops) { - | IF_NOT_Z_TYPE FP + opline->op2.var, IS_DOUBLE, >5 + | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >5 } else { - | IF_NOT_Z_TYPE FP + opline->op2.var, IS_DOUBLE, >9 + | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9 } } if (!zend_jit_cmp_double_double(Dst, opline, b, op_array, ssa)) { @@ -4534,7 +4325,7 @@ static int zend_jit_cmp(dasm_State **Dst, const zend_op *opline, int b, int *opn if (!same_ops) { |5: if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { - | IF_NOT_Z_TYPE FP + opline->op2.var, IS_LONG, >9 + | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9 } if (!zend_jit_cmp_double_long(Dst, opline, b, op_array, ssa)) { return 0; @@ -4547,14 +4338,14 @@ static int zend_jit_cmp(dasm_State **Dst, const zend_op *opline, int b, int *opn !(op1_info & MAY_BE_LONG) && (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { if (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) { - | IF_NOT_Z_TYPE FP + opline->op1.var, IS_DOUBLE, >9 + | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >9 } if (op2_info & MAY_BE_DOUBLE) { if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { if (!same_ops && (op2_info & MAY_BE_LONG)) { - | IF_NOT_Z_TYPE FP + opline->op2.var, IS_DOUBLE, >3 + | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >3 } else { - | IF_NOT_Z_TYPE FP + opline->op2.var, IS_DOUBLE, >9 + | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9 } } if (!zend_jit_cmp_double_double(Dst, opline, b, op_array, ssa)) { @@ -4567,7 +4358,7 @@ static int zend_jit_cmp(dasm_State **Dst, const zend_op *opline, int b, int *opn } |3: if (op2_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) { - | IF_NOT_Z_TYPE FP + opline->op2.var, IS_LONG, >9 + | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9 } if (!zend_jit_cmp_double_long(Dst, opline, b, op_array, ssa)) { return 0; @@ -4581,14 +4372,14 @@ static int zend_jit_cmp(dasm_State **Dst, const zend_op *opline, int b, int *opn !(op2_info & MAY_BE_LONG) && (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { if (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) { - | IF_NOT_Z_TYPE FP + opline->op2.var, IS_DOUBLE, >9 + | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9 } if (op1_info & MAY_BE_DOUBLE) { if (!same_ops && (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { if (!same_ops && (op1_info & MAY_BE_LONG)) { - | IF_NOT_Z_TYPE FP + opline->op1.var, IS_DOUBLE, >3 + | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >3 } else { - | IF_NOT_Z_TYPE FP + opline->op1.var, IS_DOUBLE, >9 + | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >9 } } if (!zend_jit_cmp_double_double(Dst, opline, b, op_array, ssa)) { @@ -4601,7 +4392,7 @@ static int zend_jit_cmp(dasm_State **Dst, const zend_op *opline, int b, int *opn } |3: if (op1_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) { - | IF_NOT_Z_TYPE FP + opline->op1.var, IS_LONG, >9 + | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >9 } if (!zend_jit_cmp_long_double(Dst, opline, b, op_array, ssa)) { return 0; @@ -4621,8 +4412,7 @@ static int zend_jit_cmp(dasm_State **Dst, const zend_op *opline, int b, int *opn |9: } | SAVE_VALID_OPLINE opline - | lea FCARG1a, [FP + opline->result.var] - | LOAD_ZVAL_ADDR FCARG2a, opline->op1_type, opline->op1 + | LOAD_ZVAL_ADDR FCARG2a, op1_addr if (opline->op1_type == IS_CV && (op1_info & MAY_BE_UNDEF)) { | IF_NOT_Z_TYPE FCARG2a, IS_UNDEF, >1 | mov FCARG1a, opline->op1.var @@ -4631,39 +4421,40 @@ static int zend_jit_cmp(dasm_State **Dst, const zend_op *opline, int b, int *opn |1: } if (opline->op2_type == IS_CV && (op2_info & MAY_BE_UNDEF)) { - | IF_NOT_Z_TYPE FP + opline->op2.var, IS_UNDEF, >1 + | IF_NOT_ZVAL_TYPE op2_addr, IS_UNDEF, >1 | mov [r4], FCARG2a // save | mov FCARG1a, opline->op2.var | EXT_CALL zend_jit_undefined_op_helper, r0 | mov FCARG2a, [r4] // restore |.if X64 - | LOAD_ADDR CARG3, &EG(uninitialized_zval) + | LOAD_ADDR CARG3, &EG(uninitialized_zval) |.else - | PUSH_ADDR &EG(uninitialized_zval), r0 + | PUSH_ADDR &EG(uninitialized_zval), r0 |.endif | jmp >2 |1: |.if X64 - | LOAD_ZVAL_ADDR CARG3, opline->op2_type, opline->op2 + | LOAD_ZVAL_ADDR CARG3, op2_addr |.else - | PUSH_ZVAL_ADDR opline->op2_type, opline->op2, r0 + | PUSH_ZVAL_ADDR op2_addr, r0 |.endif |2: } else { |.if X64 - | LOAD_ZVAL_ADDR CARG3, opline->op2_type, opline->op2 + | LOAD_ZVAL_ADDR CARG3, op2_addr |.else - | PUSH_ZVAL_ADDR opline->op2_type, opline->op2, r0 + | PUSH_ZVAL_ADDR op2_addr, r0 |.endif } - | EXT_CALL compare_function, r0 - || if (opline->opcode != ZEND_CASE) { - | FREE_OP opline->op1_type, opline->op1, op1_info, 0, op_array, opline - || } - | FREE_OP opline->op2_type, opline->op2, op2_info, 0, op_array, opline - || if (zend_may_throw(opline, op_array, ssa)) { - || zend_jit_check_exception_undef_result(Dst, opline); - || } + | LOAD_ZVAL_ADDR FCARG1a, res_addr + | EXT_CALL compare_function, r0 + if (opline->opcode != ZEND_CASE) { + | FREE_OP opline->op1_type, opline->op1, op1_info, 0, op_array, opline + } + | FREE_OP opline->op2_type, opline->op2, op2_info, 0, op_array, opline + if (zend_may_throw(opline, op_array, ssa)) { + zend_jit_check_exception_undef_result(Dst, opline); + } if (!zend_jit_cmp_slow(Dst, opline, b, op_array, ssa)) { return 0; } @@ -4693,6 +4484,9 @@ static int zend_jit_identical(dasm_State **Dst, const zend_op *opline, int b, in uint32_t identical_label = (uint32_t)-1; uint32_t not_identical_label = (uint32_t)-1; uint32_t op1_info, op2_info; + zend_jit_addr op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1); + zend_jit_addr op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2); + zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result); op1_info = OP1_INFO(); op2_info = OP2_INFO(); @@ -4736,7 +4530,7 @@ static int zend_jit_identical(dasm_State **Dst, const zend_op *opline, int b, in if ((op1_info & MAY_BE_UNDEF) && (op2_info & MAY_BE_UNDEF)) { op1_info |= MAY_BE_NULL; op2_info |= MAY_BE_NULL; - | LOAD_ZVAL_ADDR FCARG1a, IS_CV, opline->op1 + | LOAD_ZVAL_ADDR FCARG1a, op1_addr | IF_Z_TYPE FCARG1a, IS_UNDEF, >1 |.cold_code |1: @@ -4744,14 +4538,14 @@ static int zend_jit_identical(dasm_State **Dst, const zend_op *opline, int b, in | SAVE_VALID_OPLINE opline | mov FCARG1d, opline->op1.var | EXT_CALL zend_jit_undefined_op_helper, r0 - || if (zend_may_throw(opline, op_array, ssa)) { - || zend_jit_check_exception_undef_result(Dst, opline); - || } + if (zend_may_throw(opline, op_array, ssa)) { + zend_jit_check_exception_undef_result(Dst, opline); + } | LOAD_ADDR FCARG1a, &EG(uninitialized_zval) | jmp >1 |.code |1: - | LOAD_ZVAL_ADDR FCARG2a, IS_CV, opline->op2 + | LOAD_ZVAL_ADDR FCARG2a, op2_addr | IF_Z_TYPE FCARG2a, IS_UNDEF, >1 |.cold_code |1: @@ -4760,9 +4554,9 @@ static int zend_jit_identical(dasm_State **Dst, const zend_op *opline, int b, in | mov aword [r4], FCARG1a // save | mov FCARG1d, opline->op2.var | EXT_CALL zend_jit_undefined_op_helper, r0 - || if (zend_may_throw(opline, op_array, ssa)) { - || zend_jit_check_exception_undef_result(Dst, opline); - || } + if (zend_may_throw(opline, op_array, ssa)) { + zend_jit_check_exception_undef_result(Dst, opline); + } | mov FCARG1a, aword [r4] // restore | LOAD_ADDR FCARG2a, &EG(uninitialized_zval) | jmp >1 @@ -4770,7 +4564,7 @@ static int zend_jit_identical(dasm_State **Dst, const zend_op *opline, int b, in |1: } else if (op1_info & MAY_BE_UNDEF) { op1_info |= MAY_BE_NULL; - | LOAD_ZVAL_ADDR FCARG1a, IS_CV, opline->op1 + | LOAD_ZVAL_ADDR FCARG1a, op1_addr | IF_Z_TYPE FCARG1a, IS_UNDEF, >1 |.cold_code |1: @@ -4778,19 +4572,19 @@ static int zend_jit_identical(dasm_State **Dst, const zend_op *opline, int b, in | SAVE_VALID_OPLINE opline | mov FCARG1d, opline->op1.var | EXT_CALL zend_jit_undefined_op_helper, r0 - || if (zend_may_throw(opline, op_array, ssa)) { - || zend_jit_check_exception_undef_result(Dst, opline); - || } + if (zend_may_throw(opline, op_array, ssa)) { + zend_jit_check_exception_undef_result(Dst, opline); + } | LOAD_ADDR FCARG1a, &EG(uninitialized_zval) | jmp >1 |.code |1: if (opline->op2_type != IS_CONST) { - | LOAD_ZVAL_ADDR FCARG2a, opline->op2_type, opline->op2 + | LOAD_ZVAL_ADDR FCARG2a, op2_addr } } else if (op2_info & MAY_BE_UNDEF) { op2_info |= MAY_BE_NULL; - | LOAD_ZVAL_ADDR FCARG2a, IS_CV, opline->op2 + | LOAD_ZVAL_ADDR FCARG2a, op2_addr | IF_Z_TYPE FCARG2a, IS_UNDEF, >1 |.cold_code |1: @@ -4798,22 +4592,22 @@ static int zend_jit_identical(dasm_State **Dst, const zend_op *opline, int b, in | SAVE_VALID_OPLINE opline | mov FCARG1d, opline->op2.var | EXT_CALL zend_jit_undefined_op_helper, r0 - || if (zend_may_throw(opline, op_array, ssa)) { - || zend_jit_check_exception_undef_result(Dst, opline); - || } + if (zend_may_throw(opline, op_array, ssa)) { + zend_jit_check_exception_undef_result(Dst, opline); + } | LOAD_ADDR FCARG2a, &EG(uninitialized_zval) | jmp >1 |.code |1: if (opline->op1_type != IS_CONST) { - | LOAD_ZVAL_ADDR FCARG1a, opline->op1_type, opline->op1 + | LOAD_ZVAL_ADDR FCARG1a, op1_addr } } else { if (opline->op1_type != IS_CONST) { - | LOAD_ZVAL_ADDR FCARG1a, opline->op1_type, opline->op1 + | LOAD_ZVAL_ADDR FCARG1a, op1_addr } if (opline->op2_type != IS_CONST) { - | LOAD_ZVAL_ADDR FCARG2a, opline->op2_type, opline->op2 + | LOAD_ZVAL_ADDR FCARG2a, op2_addr } } if (opline->op1_type & (IS_CV|IS_VAR)) { @@ -4838,7 +4632,7 @@ static int zend_jit_identical(dasm_State **Dst, const zend_op *opline, int b, in | jmp =>not_identical_label } } else { - | SET_Z_TYPE_INFO FP + opline->result.var, (opline->opcode == ZEND_IS_IDENTICAL ? IS_FALSE : IS_TRUE) + | SET_ZVAL_TYPE_INFO res_addr, (opline->opcode == ZEND_IS_IDENTICAL ? IS_FALSE : IS_TRUE) zend_jit_check_exception(Dst); } } else if (has_concrete_type(op1_info) && @@ -4850,16 +4644,16 @@ static int zend_jit_identical(dasm_State **Dst, const zend_op *opline, int b, in | jmp =>identical_label } } else { - | SET_Z_TYPE_INFO FP + opline->result.var, (opline->opcode == ZEND_IS_IDENTICAL ? IS_TRUE : IS_FALSE) + | SET_ZVAL_TYPE_INFO res_addr, (opline->opcode == ZEND_IS_IDENTICAL ? IS_TRUE : IS_FALSE) } - } else if (opline->op1_type == IS_CONST && opline->op2_type == IS_CONST) { - if (zend_is_identical(RT_CONSTANT(op_array, opline->op1), RT_CONSTANT(op_array, opline->op2))) { + } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) == IS_CONST_ZVAL) { + if (zend_is_identical(Z_ZV(op1_addr), Z_ZV(op2_addr))) { if (smart_branch) { if (identical_label != (uint32_t)-1) { | jmp =>identical_label } } else { - | SET_Z_TYPE_INFO FP + opline->result.var, (opline->opcode == ZEND_IS_IDENTICAL ? IS_TRUE : IS_FALSE) + | SET_ZVAL_TYPE_INFO res_addr, (opline->opcode == ZEND_IS_IDENTICAL ? IS_TRUE : IS_FALSE) } } else { if (smart_branch) { @@ -4867,11 +4661,11 @@ static int zend_jit_identical(dasm_State **Dst, const zend_op *opline, int b, in | jmp =>not_identical_label } } else { - | SET_Z_TYPE_INFO FP + opline->result.var, (opline->opcode == ZEND_IS_IDENTICAL ? IS_FALSE : IS_TRUE) + | SET_ZVAL_TYPE_INFO res_addr, (opline->opcode == ZEND_IS_IDENTICAL ? IS_FALSE : IS_TRUE) } } - } else if (opline->op1_type == IS_CONST && Z_TYPE_P(RT_CONSTANT(op_array, opline->op1)) <= IS_TRUE) { - zval *val = RT_CONSTANT(op_array, opline->op1); + } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_TYPE_P(Z_ZV(op1_addr)) <= IS_TRUE) { + zval *val = Z_ZV(op1_addr); | cmp byte [FCARG2a + offsetof(zval, u1.v.type)], Z_TYPE_P(val) if (smart_branch) { @@ -4899,7 +4693,7 @@ static int zend_jit_identical(dasm_State **Dst, const zend_op *opline, int b, in } | movzx eax, al | lea eax, [eax + 2] - | SET_Z_TYPE_INFO FP + opline->result.var, eax + | SET_ZVAL_TYPE_INFO res_addr, eax } if ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) && (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) { @@ -4910,8 +4704,8 @@ static int zend_jit_identical(dasm_State **Dst, const zend_op *opline, int b, in if (smart_branch && not_identical_label != (uint32_t)-1) { | jmp =>not_identical_label } - } else if (opline->op2_type == IS_CONST && Z_TYPE_P(RT_CONSTANT(op_array, opline->op2)) <= IS_TRUE) { - zval *val = RT_CONSTANT(op_array, opline->op2); + } else if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_TYPE_P(Z_ZV(op2_addr)) <= IS_TRUE) { + zval *val = Z_ZV(op2_addr); | cmp byte [FCARG1a + offsetof(zval, u1.v.type)], Z_TYPE_P(val) if (smart_branch) { @@ -4939,7 +4733,7 @@ static int zend_jit_identical(dasm_State **Dst, const zend_op *opline, int b, in } | movzx eax, al | lea eax, [eax + 2] - | SET_Z_TYPE_INFO FP + opline->result.var, eax + | SET_ZVAL_TYPE_INFO res_addr, eax } if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) { @@ -4952,10 +4746,10 @@ static int zend_jit_identical(dasm_State **Dst, const zend_op *opline, int b, in } } else { if (opline->op1_type == IS_CONST) { - | LOAD_ZVAL_ADDR FCARG1a, opline->op1_type, opline->op1 + | LOAD_ZVAL_ADDR FCARG1a, op1_addr } if (opline->op2_type == IS_CONST) { - | LOAD_ZVAL_ADDR FCARG2a, opline->op2_type, opline->op2 + | LOAD_ZVAL_ADDR FCARG2a, op2_addr } | EXT_CALL zend_is_identical, r0 if (((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && @@ -4987,7 +4781,7 @@ static int zend_jit_identical(dasm_State **Dst, const zend_op *opline, int b, in } | movzx eax, al | lea eax, [eax + 2] - | SET_Z_TYPE_INFO FP + opline->result.var, eax + | SET_ZVAL_TYPE_INFO res_addr, eax } } @@ -5004,6 +4798,8 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, int b, zend_bool set_bool = 0; zend_bool set_bool_not = 0; zend_bool jmp_done = 0; + zend_jit_addr op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1); + zend_jit_addr res_addr; if (opline->opcode == ZEND_JMPZ) { false_label = ssa->cfg.blocks[b].successors[0]; @@ -5027,9 +4823,43 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, int b, ZEND_ASSERT(0); } + if (set_bool) { + res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result); + } + + if (Z_MODE(op1_addr) == IS_CONST_ZVAL) { + if (zend_is_true(Z_ZV(op1_addr))) { + /* Always TRUE */ + if (set_bool) { + if (set_bool_not) { + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE + } else { + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE + } + } + if (true_label != (uint32_t)-1) { + | jmp =>true_label; + } + } else { + /* Always FASLE */ + if (set_bool) { + if (set_bool_not) { + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE + } else { + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE + } + } + if (false_label != (uint32_t)-1) { + | jmp =>false_label; + } + } + return 1; + } + if (op1_info & MAY_BE_REF) { - | LOAD_ZVAL_ADDR FCARG1a, opline->op1_type, opline->op1 + | LOAD_ZVAL_ADDR FCARG1a, op1_addr | ZVAL_DEREF FCARG1a, op1_info + op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); } if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE)) { @@ -5037,9 +4867,9 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, int b, /* Always TRUE */ if (set_bool) { if (set_bool_not) { - | SET_Z_TYPE_INFO FP + opline->result.var, IS_FALSE + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE } else { - | SET_Z_TYPE_INFO FP + opline->result.var, IS_TRUE + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE } } if (true_label != (uint32_t)-1) { @@ -5050,17 +4880,13 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, int b, /* Always FASLE */ if (set_bool) { if (set_bool_not) { - | SET_Z_TYPE_INFO FP + opline->result.var, IS_TRUE + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE } else { - | SET_Z_TYPE_INFO FP + opline->result.var, IS_FALSE + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE } } } else { - if (op1_info & MAY_BE_REF) { - | cmp byte [FCARG1a + 8], IS_TRUE - } else { - | cmp byte [FP + opline->op1.var + 8], IS_TRUE - } + | CMP_ZVAL_TYPE op1_addr, IS_TRUE if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) { if ((op1_info & MAY_BE_LONG) && !(op1_info & MAY_BE_UNDEF) && @@ -5079,23 +4905,23 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, int b, /* It's FALSE */ if (set_bool) { if (set_bool_not) { - | SET_Z_TYPE_INFO FP + opline->result.var, IS_TRUE + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE } else { - | SET_Z_TYPE_INFO FP + opline->result.var, IS_FALSE + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE } } } else { if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) { if (set_bool) { | jne >1 - | SET_Z_TYPE_INFO FP + opline->result.var, IS_TRUE + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE if (true_label != (uint32_t)-1) { | jmp =>true_label } else { | jmp >9 } |1: - | SET_Z_TYPE_INFO FP + opline->result.var, IS_FALSE + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE } else { if (true_label != (uint32_t)-1) { | je =>true_label @@ -5115,7 +4941,7 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, int b, } else { | add eax, 2 } - | SET_Z_TYPE_INFO FP + opline->result.var, eax + | SET_ZVAL_TYPE_INFO res_addr, eax } } } @@ -5123,11 +4949,7 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, int b, /* It's FALSE, but may be UNDEF */ if (op1_info & MAY_BE_UNDEF) { if (op1_info & MAY_BE_ANY) { - if (op1_info & MAY_BE_REF) { - | IF_Z_TYPE FCARG1a, IS_UNDEF, >1 - } else { - | IF_Z_TYPE FP + opline->op1.var, IS_UNDEF, >1 - } + | IF_ZVAL_TYPE op1_addr, IS_UNDEF, >1 |.cold_code |1: } @@ -5165,17 +4987,9 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, int b, if (op1_info & MAY_BE_LONG) { |2: if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG))) { - if (op1_info & MAY_BE_REF) { - | IF_NOT_Z_TYPE FCARG1a, IS_LONG, >2 - } else { - | IF_NOT_Z_TYPE FP + opline->op1.var, IS_LONG, >2 - } - } - if (op1_info & MAY_BE_REF) { - | cmp aword [FCARG1a], 0 - } else { - | cmp aword [FP + opline->op1.var], 0 + | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >2 } + | LONG_OP_MEM_CONST, cmp, op1_addr, Z_L(0) if (set_bool) { | setne al | movzx eax, al @@ -5185,7 +4999,7 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, int b, } else { | lea eax, [eax + 2] } - | SET_Z_TYPE_INFO FP + opline->result.var, eax + | SET_ZVAL_TYPE_INFO res_addr, eax } if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) { if (true_label != (uint32_t)-1) { @@ -5204,8 +5018,8 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, int b, |.cold_code |2: } - if (!(op1_info & MAY_BE_REF)) { - | LOAD_ZVAL_ADDR FCARG1a, opline->op1_type, opline->op1 + if (Z_REG(op1_addr) != ZREG_FCARG1a) { + | LOAD_ZVAL_ADDR FCARG1a, op1_addr } | SAVE_VALID_OPLINE opline | EXT_CALL zend_is_true, r0 @@ -5217,7 +5031,7 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, int b, } else { | add eax, 2 } - | SET_Z_TYPE_INFO FP + opline->result.var, eax + | SET_ZVAL_TYPE_INFO res_addr, eax | FREE_OP opline->op1_type, opline->op1, op1_info, !(op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)), op_array, opline if (zend_may_throw(opline, op_array, ssa)) { if (!zend_jit_check_exception_undef_result(Dst, opline)) { @@ -5225,7 +5039,7 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, int b, } } if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) { - | cmp dword [FP + opline->result.var + 8], IS_FALSE + | CMP_ZVAL_TYPE res_addr, IS_FALSE if (true_label != (uint32_t)-1) { | jne =>true_label if (false_label != (uint32_t)-1) { @@ -5245,10 +5059,11 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, int b, if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { - | test byte [FP + opline->op1.var + 9], IS_TYPE_REFCOUNTED - | jz >3 - | mov FCARG1a, aword [FP + opline->op1.var] - | sub dword [FCARG1a], 1 + op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); + + | IF_NOT_ZVAL_REFCOUNTED op1_addr, >3 + | GET_ZVAL_PTR FCARG1a, op1_addr + | GC_DELREF FCARG1a | jnz >3 | mov aword [r4], r0 // save | ZVAL_DTOR_FUNC op1_info, op_array->filename, opline @@ -5290,13 +5105,17 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, int b, static int zend_jit_qm_assign(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) { uint32_t op1_info = OP1_INFO(); + zend_jit_addr op1_addr, res_addr; - return zend_jit_simple_assign(Dst, opline, op_array, ssa, ZREG_FP, opline->result.var, -1, opline->op1_type, opline->op1, op1_info, -1, 0); + op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1); + res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result); + return zend_jit_simple_assign(Dst, opline, op_array, ssa, res_addr, -1, opline->op1_type, opline->op1, op1_addr, op1_info, 0, 0); } static int zend_jit_assign(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) { uint32_t op1_info, op2_info; + zend_jit_addr op1_addr, op2_addr, res_addr; if (opline->op1_type != IS_CV || !ssa->ops || !ssa->var_info) { goto fallback; @@ -5305,18 +5124,22 @@ static int zend_jit_assign(dasm_State **Dst, const zend_op *opline, zend_op_arra op1_info = OP1_INFO(); op2_info = OP2_INFO(); + op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1); + op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2); + if (opline->result_type == IS_UNUSED) { + res_addr = 0; + } else { + res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result); + } + if (op1_info & MAY_BE_REF) { - | lea FCARG1a, [FP + opline->op1.var] + | LOAD_ZVAL_ADDR FCARG1a, op1_addr | ZVAL_DEREF FCARG1a, op1_info - if (!zend_jit_assign_to_variable(Dst, opline, op_array, ssa, ZREG_FCARG1a, 0, op1_info, opline->op2_type, opline->op2, op2_info, - opline->result_type == IS_UNUSED ? -1 : opline->result.var)) { - return 0; - } - } else { - if (!zend_jit_assign_to_variable(Dst, opline, op_array, ssa, ZREG_FP, opline->op1.var, op1_info, opline->op2_type, opline->op2, op2_info, - opline->result_type == IS_UNUSED ? -1 : opline->result.var)) { - return 0; - } + op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); + } + + if (!zend_jit_assign_to_variable(Dst, opline, op_array, ssa, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, res_addr)) { + return 0; } if (zend_may_throw(opline, op_array, ssa)) { @@ -5351,9 +5174,9 @@ static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, zen | sub edx, dword [r0 + offsetof(zend_function, op_array.T)] | shl edx, 5 |.if X64 - | movsxd r2, edx + | movsxd r2, edx |.endif - | sub FCARG1a, r2 + | sub FCARG1a, r2 |1: } @@ -5398,14 +5221,14 @@ static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, zen if (func && func->type == ZEND_INTERNAL_FUNCTION) { |1: |.if X64 - || if (!IS_32BIT(func)) { - | mov aword EX:RX->func, func - || } else { - | LOAD_ADDR r0, func - | mov aword EX:RX->func, r0 - || } + if (!IS_32BIT(func)) { + | mov aword EX:RX->func, func + } else { + | LOAD_ADDR r0, func + | mov aword EX:RX->func, r0 + } |.else - | mov aword EX:RX->func, func + | mov aword EX:RX->func, func |.endif } else { | mov aword EX:RX->func, r0 @@ -5610,6 +5433,14 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar zend_call_info *call_info = NULL; zend_function *func = NULL; uint32_t i; + zend_jit_addr res_addr; + + if (RETURN_VALUE_USED(opline)) { + res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result); + } else { + /* CPU stack alocated temorary zval */ + res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R4, 8); + } if (info) { call_info = info->callee_info; @@ -5707,9 +5538,9 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar if (RETURN_VALUE_USED(opline)) { | // ZVAL_NULL(EX_VAR(opline->result.var)); - | SET_Z_TYPE_INFO FP + opline->result.var, IS_NULL + | SET_ZVAL_TYPE_INFO res_addr, IS_NULL | // EX(return_value) = EX_VAR(opline->result.var); - | lea r2, aword [FP + opline->result.var] + | LOAD_ZVAL_ADDR r2, res_addr | mov aword EX:RX->return_value, r2 } else { | // EX(return_value) = 0; @@ -5742,16 +5573,16 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar | //EX_LOAD_LITERALS(op_array); |.if X64 - || if (func && zend_accel_in_shm(func->op_array.literals)) { - | LOAD_ADDR r2, func->op_array.literals - | mov EX:RX->literals, r2 - || } else { - || if (func) { - | mov r0, EX:RX->func - || } - | mov r2, aword [r0 + offsetof(zend_op_array, literals)] - | mov EX:RX->literals, r2 - || } + if (func && zend_accel_in_shm(func->op_array.literals)) { + | LOAD_ADDR r2, func->op_array.literals + | mov EX:RX->literals, r2 + } else { + if (func) { + | mov r0, EX:RX->func + } + | mov r2, aword [r0 + offsetof(zend_op_array, literals)] + | mov EX:RX->literals, r2 + } |.endif | // EG(current_execute_data) = execute_data; @@ -5793,10 +5624,10 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar | jnz >1 | // opline += num_args; |.if X64 - | movsxd r2, ecx - | imul r2, r2, sizeof(zend_op) + | movsxd r2, ecx + | imul r2, r2, sizeof(zend_op) |.else - | imul r2, ecx, sizeof(zend_op) + | imul r2, ecx, sizeof(zend_op) |.endif | add IP, r2 |1: @@ -5806,7 +5637,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar | jle >3 //??? | // zval *var = EX_VAR_NUM(num_args); |.if X64 - | movsxd r1, ecx + | movsxd r1, ecx |.endif | shl r1, 4 | lea r1, [FP + r1 + (ZEND_CALL_FRAME_SLOT * sizeof(zval))] @@ -5852,15 +5683,12 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar | jg >8 |.cold_code |8: - if (RETURN_VALUE_USED(opline)) { - | // ZVAL_NULL(EX_VAR(opline->result.var)); - | lea FCARG2a, aword [FP + opline->result.var] - | SET_Z_TYPE_INFO FCARG2a, IS_NULL - } else { + if (!RETURN_VALUE_USED(opline)) { | sub r4, 16 /* alloca() */ - | lea FCARG2a, aword [r4 + 8] - | SET_Z_TYPE_INFO FCARG2a, IS_NULL } + | // ZVAL_NULL(EX_VAR(opline->result.var)); + | LOAD_ZVAL_ADDR FCARG2a, res_addr + | SET_Z_TYPE_INFO FCARG2a, IS_NULL | mov FCARG1a, RX | EXT_CALL zend_do_fcall_overloaded, r0 | test r0, r0 @@ -5896,16 +5724,14 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar } } - if (RETURN_VALUE_USED(opline)) { - | // ZVAL_NULL(EX_VAR(opline->result.var)); - | lea FCARG2a, aword [FP + opline->result.var] - | SET_Z_TYPE_INFO FCARG2a, IS_NULL - } else { + if (!RETURN_VALUE_USED(opline)) { | sub r4, 16 /* alloca() */ - | lea FCARG2a, aword [r4 + 8] - | SET_Z_TYPE_INFO FCARG2a, IS_NULL } + | // ZVAL_NULL(EX_VAR(opline->result.var)); + | LOAD_ZVAL_ADDR FCARG2a, res_addr + | SET_Z_TYPE_INFO FCARG2a, IS_NULL + | // EG(current_execute_data) = execute_data; | mov aword [&EG(current_execute_data)], RX @@ -5920,12 +5746,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar | EXT_CALL zend_jit_verify_internal_arg_types_helper, r0 | test r0, r0 | je >8 - | // reload - if (RETURN_VALUE_USED(opline)) { - | lea FCARG2a, aword [FP + opline->result.var] - } else { - | lea FCARG2a, aword [r4 + 8] - } + | LOAD_ZVAL_ADDR FCARG2a, res_addr // reload if (!func) { | mov r0, EX:RX->func // reload | jmp >1 @@ -5938,22 +5759,22 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar | // fbc->internal_function.handler(call, ret); |.if X64 - | mov FCARG1a, RX - if (func) { - | EXT_CALL func->internal_function.handler, r0 - } else { - | call aword [r0 + offsetof(zend_internal_function, handler)] - } + | mov FCARG1a, RX + if (func) { + | EXT_CALL func->internal_function.handler, r0 + } else { + | call aword [r0 + offsetof(zend_internal_function, handler)] + } |.else - | sub r4, 8 - | push FCARG2a - | push RX - if (func) { - | EXT_CALL func->internal_function.handler, r0 - } else { - | call aword [r0 + offsetof(zend_internal_function, handler)] - } - | add r4, 16 + | sub r4, 8 + | push FCARG2a + | push RX + if (func) { + | EXT_CALL func->internal_function.handler, r0 + } else { + | call aword [r0 + offsetof(zend_internal_function, handler)] + } + | add r4, 16 |.endif | // EG(current_execute_data) = execute_data; @@ -5963,7 +5784,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar if (func) { for (i = 0; i < call_info->num_args; i++ ) { uint32_t offset = (uint32_t)(uintptr_t)ZEND_CALL_VAR_NUM(NULL, i); - | ZVAL_PTR_DTOR RX + offset, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN, 0, 1, 1, op_array->filename, opline + | ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_RX, offset), MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN, 0, 1, 1, op_array->filename, opline } } else { | mov FCARG1a, RX @@ -6009,13 +5830,13 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar | mov aword [&EG(vm_stack_top)], RX |1: - if (opline->result_type == IS_UNUSED) { + if (!RETURN_VALUE_USED(opline)) { uint32_t func_info = call_info ? zend_get_func_info(call_info, ssa) : (MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN); if (func_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { - | ZVAL_PTR_DTOR r4 + 8, func_info, 1, 1, 0, op_array->filename, opline + | ZVAL_PTR_DTOR res_addr, func_info, 1, 1, 0, op_array->filename, opline } | add r4, 16 /* revert alloca() */ } @@ -6062,6 +5883,7 @@ static int zend_jit_send_val(dasm_State **Dst, const zend_op *opline, zend_op_ar { uint32_t op1_info; uint32_t arg_num = opline->op2.num; + zend_jit_addr op1_addr, arg_addr; if (opline->opcode == ZEND_SEND_VAL_EX && arg_num > MAX_ARG_FLAG_NUM) { goto fallback; @@ -6092,15 +5914,18 @@ static int zend_jit_send_val(dasm_State **Dst, const zend_op *opline, zend_op_ar |.code } + op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1); + arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var); + if (opline->op1_type == IS_CONST) { zval *zv = RT_CONSTANT(op_array, opline->op1); - | ZVAL_COPY_CONST RX + opline->result.var, -1, zv, r0 - || if (Z_REFCOUNTED_P(zv)) { - | ADDREF_CONST zv, r0 - || } + | ZVAL_COPY_CONST arg_addr, -1, zv, r0 + if (Z_REFCOUNTED_P(zv)) { + | ADDREF_CONST zv, r0 + } } else { - | ZVAL_COPY_VALUE RX + opline->result.var, FP + opline->op1.var, op1_info, r0, eax, r2 + | ZVAL_COPY_VALUE arg_addr, op1_addr, op1_info, ZREG_R0, ZREG_R2 } return 1; @@ -6113,7 +5938,10 @@ fallback: static int zend_jit_send_ref(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, int cold) { uint32_t op1_info; + zend_jit_addr op1_addr, arg_addr, ref_addr; + op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1); + arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var); op1_info = OP1_INFO(); if (!reuse_ip) { @@ -6123,7 +5951,7 @@ static int zend_jit_send_ref(dasm_State **Dst, const zend_op *opline, zend_op_ar } if (opline->op1_type == IS_VAR) { - | lea r0, [FP + opline->op1.var] + | LOAD_ZVAL_ADDR r0, op1_addr | // if (EXPECTED(Z_TYPE_P(ret) == IS_INDIRECT)) { | IF_NOT_Z_TYPE r0, IS_INDIRECT, >1 | // ret = Z_INDIRECT_P(ret); @@ -6142,12 +5970,13 @@ static int zend_jit_send_ref(dasm_State **Dst, const zend_op *opline, zend_op_ar | // ZVAL_NEW_EMPTY_REF(arg); | EMALLOC sizeof(zend_reference), op_array, opline - | SET_Z_PTR RX + opline->result.var, r0 - | SET_Z_TYPE_INFO RX + opline->result.var, IS_REFERENCE_EX + | SET_ZVAL_PTR arg_addr, r0 + | SET_ZVAL_TYPE_INFO arg_addr, IS_REFERENCE_EX | mov dword [r0], 1 | mov dword [r0 + 4], IS_REFERENCE; + ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 8); | // ZVAL_NULL(Z_REFVAL_P(arg)); - | SET_Z_TYPE_INFO r0 + 8, IS_NULL + | SET_ZVAL_TYPE_INFO ref_addr, IS_NULL if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { | jmp >7 @@ -6161,8 +5990,8 @@ static int zend_jit_send_ref(dasm_State **Dst, const zend_op *opline, zend_op_ar } else if (opline->op1_type == IS_CV) { if (op1_info & MAY_BE_UNDEF) { if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { - | IF_NOT_Z_TYPE FP + opline->op1.var, IS_UNDEF, >1 - | SET_Z_TYPE_INFO FP + opline->op1.var, IS_NULL + | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1 + | SET_ZVAL_TYPE_INFO op1_addr, IS_NULL | jmp >2 |1: } @@ -6179,12 +6008,12 @@ static int zend_jit_send_ref(dasm_State **Dst, const zend_op *opline, zend_op_ar | IF_NOT_Z_TYPE r0, IS_REFERENCE, >2 | GET_Z_PTR r1, r0 } else { - | IF_NOT_Z_TYPE FP + opline->op1.var, IS_REFERENCE, >2 - | GET_Z_PTR r1, FP + opline->op1.var + | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >2 + | GET_ZVAL_PTR r1, op1_addr } | GC_ADDREF r1 - | SET_Z_PTR RX + opline->result.var, r1 - | SET_Z_TYPE_INFO RX + opline->result.var, IS_REFERENCE_EX + | SET_ZVAL_PTR arg_addr, r1 + | SET_ZVAL_TYPE_INFO arg_addr, IS_REFERENCE_EX | jmp >6 } |2: @@ -6195,19 +6024,21 @@ static int zend_jit_send_ref(dasm_State **Dst, const zend_op *opline, zend_op_ar | EMALLOC sizeof(zend_reference), op_array, opline | mov dword [r0], 2 | mov dword [r0 + 4], IS_REFERENCE; + ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 8); if (opline->op1_type == IS_VAR) { + zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R1, 0); + | mov r1, aword [r4] // restore - | ZVAL_COPY_VALUE_clobber_src r0 + 8, r1, op1_info, r1, ecx, r2 - | mov r1, aword [r4] // restore - | SET_Z_PTR r1, r0 - | SET_Z_TYPE_INFO r1, IS_REFERENCE_EX + | ZVAL_COPY_VALUE ref_addr, val_addr, op1_info, ZREG_R2, ZREG_R2 + | SET_ZVAL_PTR val_addr, r0 + | SET_ZVAL_TYPE_INFO val_addr, IS_REFERENCE_EX } else { - | ZVAL_COPY_VALUE r0 + 8, FP + opline->op1.var, op1_info, r1, ecx, r2 - | SET_Z_PTR FP + opline->op1.var, r0 - | SET_Z_TYPE_INFO FP + opline->op1.var, IS_REFERENCE_EX + | ZVAL_COPY_VALUE ref_addr, op1_addr, op1_info, ZREG_R1, ZREG_R2 + | SET_ZVAL_PTR op1_addr, r0 + | SET_ZVAL_TYPE_INFO op1_addr, IS_REFERENCE_EX } - | SET_Z_PTR RX + opline->result.var, r0 - | SET_Z_TYPE_INFO RX + opline->result.var, IS_REFERENCE_EX + | SET_ZVAL_PTR arg_addr, r0 + | SET_ZVAL_TYPE_INFO arg_addr, IS_REFERENCE_EX } |6: @@ -6221,6 +6052,8 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, zend_op_ar { uint32_t op1_info; uint32_t arg_num = opline->op2.num; + zend_jit_addr op1_addr, arg_addr; + if ((opline->opcode == ZEND_SEND_VAR_EX || opline->opcode == ZEND_SEND_VAR_NO_REF_EX) && @@ -6228,6 +6061,8 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, zend_op_ar goto fallback; } + op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1); + arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var); op1_info = OP1_INFO(); if (!reuse_ip) { @@ -6263,7 +6098,7 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, zend_op_ar } else if (opline->opcode == ZEND_SEND_VAR_NO_REF_EX) { mask = ZEND_SEND_PREFER_REF << ((arg_num + 3) * 2); - | ZVAL_COPY_VALUE RX + opline->result.var, FP + opline->op1.var, op1_info, r1, ecx, r2 + | ZVAL_COPY_VALUE arg_addr, op1_addr, op1_info, ZREG_R1, ZREG_R2 if (op1_info & MAY_BE_REF) { | cmp cl, IS_REFERENCE | je >7 @@ -6272,15 +6107,15 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, zend_op_ar | jnz >7 | SAVE_VALID_OPLINE opline |.if X64 - | mov CARG1, E_NOTICE - | LOAD_ADDR CARG2, "Only variables should be passed by reference" - | EXT_CALL zend_error, r0 + | mov CARG1, E_NOTICE + | LOAD_ADDR CARG2, "Only variables should be passed by reference" + | EXT_CALL zend_error, r0 |.else - | sub r4, 8 - | push "Only variables should be passed by reference" - | push E_NOTICE - | EXT_CALL zend_error, r0 - | add r4, 16 + | sub r4, 8 + | push "Only variables should be passed by reference" + | push E_NOTICE + | EXT_CALL zend_error, r0 + | add r4, 16 |.endif if (!zend_jit_check_exception(Dst)) { return 0; @@ -6295,7 +6130,7 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, zend_op_ar if (op1_info & MAY_BE_UNDEF) { if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { - | IF_Z_TYPE FP + opline->op1.var, IS_UNDEF, >1 + | IF_ZVAL_TYPE op1_addr, IS_UNDEF, >1 |.cold_code |1: } @@ -6303,7 +6138,7 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, zend_op_ar | SAVE_VALID_OPLINE opline | mov FCARG1d, opline->op1.var | EXT_CALL zend_jit_undefined_op_helper, r0 - | SET_Z_TYPE_INFO RX + opline->result.var, IS_NULL + | SET_ZVAL_TYPE_INFO arg_addr, IS_NULL if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { | jmp >7 @@ -6312,22 +6147,22 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, zend_op_ar } if (opline->opcode == ZEND_SEND_VAR_NO_REF) { - | ZVAL_COPY_VALUE RX + opline->result.var, FP + opline->op1.var, op1_info, r1, ecx, r2 + | ZVAL_COPY_VALUE arg_addr, op1_addr, op1_info, ZREG_R1, ZREG_R2 if (op1_info & MAY_BE_REF) { | cmp cl, IS_REFERENCE | je >7 } | SAVE_VALID_OPLINE opline |.if X64 - | mov CARG1, E_NOTICE - | LOAD_ADDR CARG2, "Only variables should be passed by reference" - | EXT_CALL zend_error, r0 + | mov CARG1, E_NOTICE + | LOAD_ADDR CARG2, "Only variables should be passed by reference" + | EXT_CALL zend_error, r0 |.else - | sub r4, 8 - | push "Only variables should be passed by reference" - | push E_NOTICE - | EXT_CALL zend_error, r0 - | add r4, 16 + | sub r4, 8 + | push "Only variables should be passed by reference" + | push E_NOTICE + | EXT_CALL zend_error, r0 + | add r4, 16 |.endif if (!zend_jit_check_exception(Dst)) { return 0; @@ -6335,18 +6170,22 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, zend_op_ar } else if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { if (op1_info & MAY_BE_REF) { if (opline->op1_type == IS_CV) { - | lea FCARG1a, [FP + opline->op1.var] + zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); + + | LOAD_ZVAL_ADDR FCARG1a, op1_addr | ZVAL_DEREF FCARG1a, op1_info - | ZVAL_COPY_VALUE RX + opline->result.var, FCARG1a, op1_info, r0, eax, r2 + | ZVAL_COPY_VALUE arg_addr, val_addr, op1_info, ZREG_R0, ZREG_R2 | TRY_ADDREF op1_info, ah, r2 } else { - | IF_Z_TYPE FP + opline->op1.var, IS_REFERENCE, >1 + zend_jit_addr ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 8); + + | IF_ZVAL_TYPE op1_addr, IS_REFERENCE, >1 |.cold_code |1: | // zend_refcounted *ref = Z_COUNTED_P(retval_ptr); - | GET_Z_PTR FCARG1a, FP + opline->op1.var + | GET_ZVAL_PTR FCARG1a, op1_addr | // ZVAL_COPY_VALUE(return_value, &ref->value); - | ZVAL_COPY_VALUE RX + opline->result.var, FCARG1a + 8, op1_info, r0, eax, r2 + | ZVAL_COPY_VALUE arg_addr, ref_addr, op1_info, ZREG_R0, ZREG_R2 | GC_DELREF FCARG1a | je >1 | IF_NOT_REFCOUNTED ah, >2 @@ -6356,14 +6195,14 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, zend_op_ar | EFREE_REG_24 op_array, opline | jmp >2 |.code - | ZVAL_COPY_VALUE RX + opline->result.var, FP + opline->op1.var, op1_info, r0, eax, r2 + | ZVAL_COPY_VALUE arg_addr, op1_addr, op1_info, ZREG_R0, ZREG_R2 |2: } } else { - | ZVAL_COPY_VALUE RX + opline->result.var, FP + opline->op1.var, op1_info, r0, eax, r2 - || if (opline->op1_type == IS_CV) { - | TRY_ADDREF op1_info, ah, r2 - || } + | ZVAL_COPY_VALUE arg_addr, op1_addr, op1_info, ZREG_R0, ZREG_R2 + if (opline->op1_type == IS_CV) { + | TRY_ADDREF op1_info, ah, r2 + } } } |7: @@ -6396,7 +6235,9 @@ static int zend_jit_smart_true(dasm_State **Dst, const zend_op *opline, int b, z target_label = ssa->cfg.blocks[b].successors[1]; | jmp =>target_label } else { - | SET_Z_TYPE_INFO FP + opline->result.var, IS_TRUE + zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result); + + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE if (jmp) { | jmp >7 } @@ -6426,7 +6267,9 @@ static int zend_jit_smart_false(dasm_State **Dst, const zend_op *opline, int b, target_label = ssa->cfg.blocks[b].successors[0]; | jmp =>target_label } else { - | SET_Z_TYPE_INFO FP + opline->result.var, IS_FALSE + zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result); + + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE if (jmp) { | jmp >7 } @@ -6441,6 +6284,7 @@ static int zend_jit_defined(dasm_State **Dst, const zend_op *opline, int b, int uint32_t defined_label = (uint32_t)-1; uint32_t undefined_label = (uint32_t)-1; zval *zv = RT_CONSTANT(op_array, opline->op1); + zend_jit_addr res_addr; if (((opline+1)->opcode == ZEND_JMPZ || (opline+1)->opcode == ZEND_JMPNZ || @@ -6481,6 +6325,8 @@ static int zend_jit_defined(dasm_State **Dst, const zend_op *opline, int b, int } else { | jz >3 } + } else { + res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result); } | mov r0, EX->run_time_cache | mov r0, aword [r0 + Z_CACHE_SLOT_P(zv)] @@ -6492,7 +6338,7 @@ static int zend_jit_defined(dasm_State **Dst, const zend_op *opline, int b, int } } else { | add eax, IS_FALSE - | SET_Z_TYPE_INFO FP + opline->result.var, eax + | SET_ZVAL_TYPE_INFO res_addr, eax | jmp >3 } |.code @@ -6501,7 +6347,7 @@ static int zend_jit_defined(dasm_State **Dst, const zend_op *opline, int b, int | jmp =>defined_label } } else { - | SET_Z_TYPE_INFO FP + opline->result.var, IS_TRUE + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE } |3: @@ -6514,6 +6360,7 @@ static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, int b, i uint32_t target_label; zend_uchar type; zend_bool smart_branch = 0; + zend_jit_addr op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1); if (opline->extended_value == IS_RESOURCE) { // TODO: support for is_resource() ??? @@ -6532,7 +6379,7 @@ static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, int b, i op1_info = OP1_INFO(); if (op1_info & MAY_BE_UNDEF) { if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { - | IF_Z_TYPE FP + opline->op1.var, IS_UNDEF, >1 + | IF_ZVAL_TYPE op1_addr, IS_UNDEF, >1 |.cold_code |1: } @@ -6573,7 +6420,7 @@ static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, int b, i } } else { if (op1_info & MAY_BE_REF) { - | LOAD_ZVAL_ADDR r0, opline->op1_type, opline->op1 + | LOAD_ZVAL_ADDR r0, op1_addr | ZVAL_DEREF r0, op1_info } if (opline->extended_value == _IS_BOOL) { @@ -6582,12 +6429,12 @@ static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, int b, i (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { | // if (Z_REFCOUNTED_P(cv)) { - | IF_Z_REFCOUNTED FP + opline->op1.var, >1 + | IF_ZVAL_REFCOUNTED op1_addr, >1 |.cold_code |1: } | // if (!Z_DELREF_P(cv)) { - | GET_Z_PTR FCARG1a, FP + opline->op1.var + | GET_ZVAL_PTR FCARG1a, op1_addr | GC_DELREF FCARG1a if (RC_MAY_BE_1(op1_info)) { if (RC_MAY_BE_N(op1_info)) { @@ -6645,10 +6492,12 @@ static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, int b, i target_label = ssa->cfg.blocks[b].successors[1]; | jmp =>target_label } else { + zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result); + | setne al | movzx eax, al | add eax, 2 - | SET_Z_TYPE_INFO FP + opline->result.var, eax + | SET_ZVAL_TYPE_INFO res_addr, eax | FREE_OP opline->op1_type, opline->op1, op1_info, 1, op_array, opline } } else if (opline->extended_value == IS_RESOURCE) { @@ -6659,12 +6508,12 @@ static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, int b, i (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { | // if (Z_REFCOUNTED_P(cv)) { - | IF_Z_REFCOUNTED FP + opline->op1.var, >1 + | IF_ZVAL_REFCOUNTED op1_addr, >1 |.cold_code |1: } | // if (!Z_DELREF_P(cv)) { - | GET_Z_PTR FCARG1a, FP + opline->op1.var + | GET_ZVAL_PTR FCARG1a, op1_addr | GC_DELREF FCARG1a if (RC_MAY_BE_1(op1_info)) { if (RC_MAY_BE_N(op1_info)) { @@ -6720,10 +6569,12 @@ static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, int b, i target_label = ssa->cfg.blocks[b].successors[1]; | jmp =>target_label } else { + zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result); + | sete al | movzx eax, al | add eax, 2 - | SET_Z_TYPE_INFO FP + opline->result.var, eax + | SET_ZVAL_TYPE_INFO res_addr, eax | FREE_OP opline->op1_type, opline->op1, op1_info, 1, op_array, opline } } @@ -6778,7 +6629,7 @@ static int zend_jit_free_compiled_variables(dasm_State **Dst, const zend_op *opl if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { uint32_t offset = (uint32_t)(uintptr_t)ZEND_CALL_VAR_NUM(NULL, i); - | ZVAL_PTR_DTOR FP + offset, info, 1, 1, 1, op_array->filename, opline + | ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_FP, offset), info, 1, 1, 1, op_array->filename, opline } } return 1; @@ -6894,6 +6745,7 @@ static int zend_jit_leave_func(dasm_State **Dst, const zend_op *opline, zend_op_ static int zend_jit_return(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) { uint32_t op1_info; + zend_jit_addr op1_addr, ret_addr; if (op_array->type == ZEND_EVAL_CODE || !op_array->function_name || !ssa->ops || !ssa->var_info) { // TODO: support for top-level code @@ -6901,6 +6753,7 @@ static int zend_jit_return(dasm_State **Dst, const zend_op *opline, zend_op_arra } op1_info = OP1_INFO(); + op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1); if (opline->op1_type == IS_CV && (op1_info & MAY_BE_UNDEF)) { // TODO: support for IS_UNDEF ??? return zend_jit_tail_handler(Dst, opline); @@ -6909,111 +6762,102 @@ static int zend_jit_return(dasm_State **Dst, const zend_op *opline, zend_op_arra // if (!EX(return_value)) | mov r1, EX->return_value | test r1, r1 + ret_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R1, 0); if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { | jz >1 |.cold_code |1: - || if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { - || if (jit_return_label >= 0) { - | IF_NOT_Z_REFCOUNTED FP + opline->op1.var, =>jit_return_label - || } else { - | IF_NOT_Z_REFCOUNTED FP + opline->op1.var, >9 - || } - || } - | GET_Z_PTR FCARG1a, FP + opline->op1.var + if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { + if (jit_return_label >= 0) { + | IF_NOT_ZVAL_REFCOUNTED op1_addr, =>jit_return_label + } else { + | IF_NOT_ZVAL_REFCOUNTED op1_addr, >9 + } + } + | GET_ZVAL_PTR FCARG1a, op1_addr | GC_DELREF FCARG1a - || if (RC_MAY_BE_1(op1_info)) { - || if (RC_MAY_BE_N(op1_info)) { - || if (jit_return_label >= 0) { - | jnz =>jit_return_label - || } else { - | jnz >9 - || } - || } - | //SAVE_OPLINE() - | SAVE_VALID_OPLINE opline - | ZVAL_DTOR_FUNC op1_info, op_array->filename, opline - | //????mov r1, EX->return_value // reload ??? - || } - || if (jit_return_label >= 0) { - | jmp =>jit_return_label - || } else { - | jmp >9 - || } + if (RC_MAY_BE_1(op1_info)) { + if (RC_MAY_BE_N(op1_info)) { + if (jit_return_label >= 0) { + | jnz =>jit_return_label + } else { + | jnz >9 + } + } + | //SAVE_OPLINE() + | SAVE_VALID_OPLINE opline + | ZVAL_DTOR_FUNC op1_info, op_array->filename, opline + | //????mov r1, EX->return_value // reload ??? + } + if (jit_return_label >= 0) { + | jmp =>jit_return_label + } else { + | jmp >9 + } |.code } else { - || if (jit_return_label >= 0) { - | jz =>jit_return_label - || } else { - | jz >9 - || } + if (jit_return_label >= 0) { + | jz =>jit_return_label + } else { + | jz >9 + } } if (opline->op1_type == IS_CONST) { zval *zv = RT_CONSTANT(op_array, opline->op1); - | ZVAL_COPY_CONST r1, -1, zv, r0 + | ZVAL_COPY_CONST ret_addr, -1, zv, r0 if (Z_REFCOUNTED_P(zv)) { - | ADDREF_CONST zv, r0 + | ADDREF_CONST zv, r0 } } else if (opline->op1_type == IS_TMP_VAR) { - | ZVAL_COPY_VALUE r1, FP + opline->op1.var, op1_info, r0, eax, r2 + | ZVAL_COPY_VALUE ret_addr, op1_addr, op1_info, ZREG_R0, ZREG_R2 } else if (opline->op1_type == IS_CV) { if (op1_info & MAY_BE_REF) { - | lea r0, [FP + opline->op1.var] + | LOAD_ZVAL_ADDR r0, op1_addr | ZVAL_DEREF r0, op1_info - | ZVAL_COPY_VALUE_clobber_src r1, r0, op1_info, r0, eax, r2 - | TRY_ADDREF op1_info, ah, r2 - } else { - | ZVAL_COPY_VALUE r1, FP + opline->op1.var, op1_info, r0, eax, r2 - | // TODO: JIT: if (EXPECTED(!(EX_CALL_INFO() & ZEND_CALL_CODE))) ZVAL_NULL(retval_ptr); ??? - | TRY_ADDREF op1_info, ah, r2 + op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); } + | ZVAL_COPY_VALUE ret_addr, op1_addr, op1_info, ZREG_R0, ZREG_R2 + | // TODO: JIT: if (EXPECTED(!(EX_CALL_INFO() & ZEND_CALL_CODE))) ZVAL_NULL(retval_ptr); ??? + | TRY_ADDREF op1_info, ah, r2 } else { if (op1_info & MAY_BE_REF) { - | IF_Z_TYPE FP + opline->op1.var, IS_REFERENCE, >1 + zend_jit_addr ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, offsetof(zend_reference, val)); + + | IF_ZVAL_TYPE op1_addr, IS_REFERENCE, >1 |.cold_code |1: | // zend_refcounted *ref = Z_COUNTED_P(retval_ptr); - | GET_Z_PTR r0, FP + opline->op1.var + | GET_ZVAL_PTR r0, op1_addr | // ZVAL_COPY_VALUE(return_value, &ref->value); - |.if X64 - | GET_Z_PTR r2, r0 + offsetof(zend_reference, val) - | SET_Z_PTR r1, r2 - |.else - | GET_Z_PTR r2, r0 + offsetof(zend_reference, val) - | SET_Z_PTR r1, r2 - | GET_Z_W2 r2, r0 + offsetof(zend_reference, val) - | SET_Z_W2 r1 + 4, r2 - |.endif - | GET_Z_TYPE_INFO edx, r0 + offsetof(zend_reference, val) - | SET_Z_TYPE_INFO r1, edx + | ZVAL_COPY_VALUE ret_addr, ref_addr, op1_info, ZREG_R2, ZREG_R2 | GC_DELREF r0 | je >2 | // if (IS_REFCOUNTED()) - || if (jit_return_label >= 0) { - | IF_NOT_REFCOUNTED dh, =>jit_return_label - || } else { - | IF_NOT_REFCOUNTED dh, >9 - || } + if (jit_return_label >= 0) { + | IF_NOT_REFCOUNTED dh, =>jit_return_label + } else { + | IF_NOT_REFCOUNTED dh, >9 + } | // ADDREF - | GET_Z_PTR r2, r1 + | GET_ZVAL_PTR r2, ret_addr // reload | GC_ADDREF r2 - || if (jit_return_label >= 0) { - | jmp =>jit_return_label - || } else { - | jmp >9 - || } + if (jit_return_label >= 0) { + | jmp =>jit_return_label + } else { + | jmp >9 + } |2: | EFREE_24 r0, op_array, opline - || if (jit_return_label >= 0) { - | jmp =>jit_return_label - || } else { - | jmp >9 - || } + if (jit_return_label >= 0) { + | jmp =>jit_return_label + } else { + | jmp >9 + } |.code } - | ZVAL_COPY_VALUE r1, FP + opline->op1.var, op1_info, r0, eax, r2 + | ZVAL_COPY_VALUE ret_addr, op1_addr, op1_info, ZREG_R0, ZREG_R2 } |9: @@ -7024,6 +6868,7 @@ static int zend_jit_return(dasm_State **Dst, const zend_op *opline, zend_op_arra static int zend_jit_fetch_dim_read(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) { uint32_t op1_info, op2_info, res_info; + zend_jit_addr op1_addr, op2_addr, res_addr; if (!ssa->ops || !ssa->var_info) { goto fallback; @@ -7033,21 +6878,21 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, const zend_op *opline, zend op2_info = OP2_INFO(); res_info = RES_INFO(); + op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1); + op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2); + res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result); + if (op1_info & MAY_BE_REF) { - | lea FCARG1a, [FP + opline->op1.var] + | LOAD_ZVAL_ADDR FCARG1a, op1_addr | ZVAL_DEREF FCARG1a, op1_info + op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); } if (op1_info & MAY_BE_ARRAY) { - if (op1_info & MAY_BE_REF) { - | IF_NOT_Z_TYPE FCARG1a, IS_ARRAY, >7 - | GET_Z_PTR FCARG1a, FCARG1a - } else { - if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { - | IF_NOT_Z_TYPE FP + opline->op1.var, IS_ARRAY, >7 - } - | LONG_LOAD FCARG1a, opline->op1_type, opline->op1 + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7 } + | GET_ZVAL_LVAL FCARG1a, op1_addr if (!zend_jit_fetch_dimension_address_inner(Dst, opline, op_array, (opline->opcode == ZEND_FETCH_DIM_R) ? BP_VAR_R : BP_VAR_IS, op1_info, op2_info, 8, 9)) { return 0; } @@ -7061,22 +6906,17 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, const zend_op *opline, zend if (op1_info & MAY_BE_STRING) { if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_STRING))) { - if (op1_info & MAY_BE_REF) { - | IF_NOT_Z_TYPE FCARG1a, IS_STRING, >6 - } else { - | IF_NOT_Z_TYPE FP + opline->op1.var, IS_STRING, >6 - } + | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >6 } | SAVE_VALID_OPLINE opline - if (!(op1_info & MAY_BE_REF)) { - | LOAD_ZVAL_ADDR FCARG1a, opline->op1_type, opline->op1 + if (Z_REG(op1_addr) != ZREG_FCARG1a) { + | LOAD_ZVAL_ADDR FCARG1a, op1_addr } - | LOAD_ZVAL_ADDR FCARG2a, opline->op2_type, opline->op2 + | LOAD_ZVAL_ADDR FCARG2a, op2_addr |.if X64 - | lea CARG3, [FP + opline->result.var] + | LOAD_ZVAL_ADDR CARG3, res_addr |.else - | lea r0, [FP + opline->result.var] - | push r0 + | PUSH_ZVAL_ADDR res_addr, r0 |.endif if (opline->opcode == ZEND_FETCH_DIM_R) { | EXT_CALL zend_jit_fetch_dim_str_r_helper, r0 @@ -7087,29 +6927,24 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, const zend_op *opline, zend } if ((op1_info & MAY_BE_ARRAY) || (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_STRING)))) { - | jmp >9 // END + | jmp >9 // END } |6: } if (op1_info & MAY_BE_OBJECT) { if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_STRING|MAY_BE_OBJECT))) { - if (op1_info & MAY_BE_REF) { - | IF_NOT_Z_TYPE FCARG1a, IS_OBJECT, >6 - } else { - | IF_NOT_Z_TYPE FP + opline->op1.var, IS_OBJECT, >6 - } + | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >6 } | SAVE_VALID_OPLINE opline - if (!(op1_info & MAY_BE_REF)) { - | LOAD_ZVAL_ADDR FCARG1a, opline->op1_type, opline->op1 + if (Z_REG(op1_addr) != ZREG_FCARG1a) { + | LOAD_ZVAL_ADDR FCARG1a, op1_addr } - | LOAD_ZVAL_ADDR FCARG2a, opline->op2_type, opline->op2 + | LOAD_ZVAL_ADDR FCARG2a, op2_addr |.if X64 - | lea CARG3, [FP + opline->result.var] + | LOAD_ZVAL_ADDR CARG3, res_addr |.else - | lea r0, [FP + opline->result.var] - | push r0 + | PUSH_ZVAL_ADDR res_addr, r0 |.endif if (opline->opcode == ZEND_FETCH_DIM_R) { | EXT_CALL zend_jit_fetch_dim_obj_r_helper, r0 @@ -7120,7 +6955,7 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, const zend_op *opline, zend } if ((op1_info & MAY_BE_ARRAY) || (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_STRING|MAY_BE_OBJECT)))) { - | jmp >9 // END + | jmp >9 // END } |6: } @@ -7128,11 +6963,7 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, const zend_op *opline, zend if ((opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) || (op2_info & MAY_BE_UNDEF)) { | SAVE_VALID_OPLINE opline if (opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) { - if (op1_info & MAY_BE_REF) { - | IF_NOT_Z_TYPE FCARG1a, IS_UNDEF, >1 - } else { - | IF_NOT_Z_TYPE FP + opline->op1.var, IS_UNDEF, >1 - } + | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1 | // zend_error(E_NOTICE, "Undefined variable: %s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); | mov FCARG1d, opline->op1.var | EXT_CALL zend_jit_undefined_op_helper, r0 @@ -7140,7 +6971,7 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, const zend_op *opline, zend } if (op2_info & MAY_BE_UNDEF) { - | IF_NOT_Z_TYPE FP + opline->op2.var, IS_UNDEF, >1 + | IF_NOT_ZVAL_TYPE op2_addr, IS_UNDEF, >1 | mov FCARG1d, opline->op2.var | EXT_CALL zend_jit_undefined_op_helper, r0 |1: @@ -7148,7 +6979,7 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, const zend_op *opline, zend } if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_STRING|MAY_BE_OBJECT))) { - | SET_Z_TYPE_INFO FP + opline->result.var, IS_NULL + | SET_ZVAL_TYPE_INFO res_addr, IS_NULL if (op1_info & MAY_BE_ARRAY) { | jmp >9 // END } @@ -7160,28 +6991,30 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, const zend_op *opline, zend } if (op1_info & MAY_BE_ARRAY) { + zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); + |8: if (res_info & MAY_BE_REF) { - | // ZVAL_COPY_UNREF + | // ZVAL_COPY_UNREF if (res_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { - | IF_NOT_Z_REFCOUNTED r0, >2 + | IF_NOT_ZVAL_REFCOUNTED val_addr, >2 } - | GET_Z_PTR r1, r0 - | IF_NOT_Z_TYPE r0, IS_REFERENCE, >1 - | cmp dword [r1], 1 - | jne >1 - | lea FCARG1a, [FP + opline->result.var] - | mov FCARG2a, r0 - | EXT_CALL zend_jit_zval_copy_unref_helper, r0 - | jmp >9 - |1: - | GC_ADDREF r1 - |2: - | ZVAL_COPY_VALUE FP + opline->result.var, r0, MAY_BE_ANY, r1, ecx, r2 + | GET_ZVAL_PTR r1, val_addr + | IF_NOT_ZVAL_TYPE val_addr, IS_REFERENCE, >1 + | cmp dword [r1], 1 + | jne >1 + | LOAD_ZVAL_ADDR FCARG1a, res_addr + | LOAD_ZVAL_ADDR FCARG2a, val_addr + | EXT_CALL zend_jit_zval_copy_unref_helper, r0 + | jmp >9 + |1: + | GC_ADDREF r1 + |2: + | ZVAL_COPY_VALUE res_addr, val_addr, MAY_BE_ANY, ZREG_R1, ZREG_R2 } else { - | // ZVAL_COPY - | ZVAL_COPY_VALUE FP + opline->result.var, r0, MAY_BE_ANY, r1, ecx, r2 - | TRY_ADDREF res_info, ch, r2 + | // ZVAL_COPY + | ZVAL_COPY_VALUE res_addr, val_addr, MAY_BE_ANY, ZREG_R1, ZREG_R2 + | TRY_ADDREF res_info, ch, r2 } } |9: // END @@ -7212,6 +7045,7 @@ fallback: static int zend_jit_isset_isempty_dim(dasm_State **Dst, const zend_op *opline, int b, int *opnum, zend_op_array *op_array, zend_ssa *ssa) { uint32_t op1_info, op2_info; + zend_jit_addr op1_addr, op2_addr, res_addr; if (!ssa->ops || !ssa->var_info || !(opline->extended_value & ZEND_ISSET)) { // TODO: support for empty() ??? @@ -7221,21 +7055,21 @@ static int zend_jit_isset_isempty_dim(dasm_State **Dst, const zend_op *opline, i op1_info = OP1_INFO(); op2_info = OP2_INFO(); + op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1); + op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2); + res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result); + if (op1_info & MAY_BE_REF) { - | lea FCARG1a, [FP + opline->op1.var] + | LOAD_ZVAL_ADDR FCARG1a, op1_addr | ZVAL_DEREF FCARG1a, op1_info + op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); } if (op1_info & MAY_BE_ARRAY) { - if (op1_info & MAY_BE_REF) { - | IF_NOT_Z_TYPE FCARG1a, IS_ARRAY, >7 - | GET_Z_PTR FCARG1a, FCARG1a - } else { - if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { - | IF_NOT_Z_TYPE FP + opline->op1.var, IS_ARRAY, >7 - } - | LONG_LOAD FCARG1a, opline->op1_type, opline->op1 + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7 } + | GET_ZVAL_LVAL FCARG1a, op1_addr if (!zend_jit_fetch_dimension_address_inner(Dst, opline, op_array, BP_JIT_IS, op1_info, op2_info, 8, 9)) { return 0; } @@ -7248,10 +7082,10 @@ static int zend_jit_isset_isempty_dim(dasm_State **Dst, const zend_op *opline, i } | SAVE_VALID_OPLINE opline - if (!(op1_info & MAY_BE_REF)) { - | LOAD_ZVAL_ADDR FCARG1a, opline->op1_type, opline->op1 + if (Z_REG(op1_addr) != ZREG_FCARG1a) { + | LOAD_ZVAL_ADDR FCARG1a, op1_addr } - | LOAD_ZVAL_ADDR FCARG2a, opline->op2_type, opline->op2 + | LOAD_ZVAL_ADDR FCARG2a, op2_addr | EXT_CALL zend_jit_isset_dim_helper, r0 | test r0, r0 | jz >9 @@ -7288,7 +7122,7 @@ static int zend_jit_isset_isempty_dim(dasm_State **Dst, const zend_op *opline, i unsigned int target_label = ssa->cfg.blocks[b].successors[1]; | jmp =>target_label } else { - | SET_Z_TYPE_INFO FP + opline->result.var, IS_TRUE + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE | jmp >8 } } else { @@ -7313,7 +7147,7 @@ static int zend_jit_isset_isempty_dim(dasm_State **Dst, const zend_op *opline, i unsigned int target_label = ssa->cfg.blocks[b].successors[0]; | jmp =>target_label } else { - | SET_Z_TYPE_INFO FP + opline->result.var, IS_FALSE + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE } } else { | //???? @@ -7340,6 +7174,7 @@ fallback: static int zend_jit_bind_global(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) { uint32_t op1_info; + zend_jit_addr op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1); zval *varname = RT_CONSTANT(op_array, opline->op2); if (!ssa->ops || !ssa->var_info) { @@ -7357,33 +7192,33 @@ static int zend_jit_bind_global(dasm_State **Dst, const zend_op *opline, zend_op | jae >9 //Bucket *p = EG(symbol_table).arData + idx; |.if X64 - | shl r0, 5 + | shl r0, 5 |.else - | imul r0, sizeof(Bucket) + | imul r0, sizeof(Bucket) |.endif |.if X64 - | LOAD_ADDR r1, &EG(symbol_table.arData) - | add r0, [r1] + | LOAD_ADDR r1, &EG(symbol_table.arData) + | add r0, [r1] |.else - | add r0, [&EG(symbol_table).arData] + | add r0, [&EG(symbol_table).arData] |.endif | IF_Z_TYPE r0, IS_UNDEF, >9 // (EXPECTED(p->key == Z_STR_P(varname)) |.if X64 - | LOAD_ADDR r1, Z_PTR_P(varname) - | cmp [r0 + offsetof(Bucket, key)], r1 + | LOAD_ADDR r1, Z_PTR_P(varname) + | cmp [r0 + offsetof(Bucket, key)], r1 |.else - | cmp aword [r0 + offsetof(Bucket, key)], Z_PTR_P(varname) + | cmp aword [r0 + offsetof(Bucket, key)], Z_PTR_P(varname) |.endif | jne >1 |.cold_code |1: //(EXPECTED(p->h == ZSTR_H(Z_STR_P(varname))) |.if X64 - | mov64 r1, ZSTR_H(Z_STR_P(varname)) - | cmp qword [r0 + offsetof(Bucket, h)], r1 + | mov64 r1, ZSTR_H(Z_STR_P(varname)) + | cmp qword [r0 + offsetof(Bucket, h)], r1 |.else - | cmp dword [r0 + offsetof(Bucket, h)], ZSTR_H(Z_STR_P(varname)) + | cmp dword [r0 + offsetof(Bucket, h)], ZSTR_H(Z_STR_P(varname)) |.endif | jne >9 //EXPECTED(p->key != NULL) @@ -7397,17 +7232,17 @@ static int zend_jit_bind_global(dasm_State **Dst, const zend_op *opline, zend_op | add r1, offsetof(zend_string, val) | mov [r4], r0 |.if X64 - | mov CARG1, r1 - | LOAD_ADDR CARG2, Z_STRVAL_P(varname) - | mov CARG3, Z_STRLEN_P(varname) - | call &memcmp + | mov CARG1, r1 + | LOAD_ADDR CARG2, Z_STRVAL_P(varname) + | mov CARG3, Z_STRLEN_P(varname) + | call &memcmp |.else - | sub r4, 4 - | push Z_STRLEN_P(varname) - | push Z_STRVAL_P(varname) - | push r1 - | call &memcmp - | add r4, 16 + | sub r4, 4 + | push Z_STRLEN_P(varname) + | push Z_STRVAL_P(varname) + | push r1 + | call &memcmp + | add r4, 16 |.endif | test al, al | mov r0, aword [r4] @@ -7442,13 +7277,13 @@ static int zend_jit_bind_global(dasm_State **Dst, const zend_op *opline, zend_op //if (UNEXPECTED(Z_REFCOUNTED_P(variable_ptr))) if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { if (op1_info & (MAY_BE_ANY - (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { - | IF_Z_REFCOUNTED FP + opline->op1.var, >2 + | IF_ZVAL_REFCOUNTED op1_addr, >2 } |.cold_code //zval_dtor_func(Z_COUNTED_P(variable_ptr)) |2: //if (EXPECTED(variable_ptr != value)) - | lea FCARG1a, aword [FP + opline->op1.var] + | LOAD_ZVAL_ADDR FCARG1a, op1_addr | cmp FCARG1a, r2 | je >4 | GET_Z_PTR FCARG1a, FCARG1a @@ -7477,8 +7312,8 @@ static int zend_jit_bind_global(dasm_State **Dst, const zend_op *opline, zend_op } |5: //ZVAL_REF(variable_ptr, ref) - | SET_Z_PTR FP + opline->op1.var, r0 - | SET_Z_TYPE_INFO FP + opline->op1.var, IS_REFERENCE_EX + | SET_ZVAL_PTR op1_addr, r0 + | SET_ZVAL_TYPE_INFO op1_addr, IS_REFERENCE_EX //END of handler |.cold_code @@ -7502,10 +7337,11 @@ static int zend_jit_recv_init(dasm_State **Dst, const zend_op *opline, zend_op_a zend_bool has_slow = 0; uint32_t arg_num = opline->op1.num; zval *zv = RT_CONSTANT(op_array, opline->op2); + zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result); | cmp dword EX->This.u2.num_args, arg_num | jae >5 - | ZVAL_COPY_CONST FP + opline->result.var, -1, zv, r0 + | ZVAL_COPY_CONST res_addr, -1, zv, r0 if (Z_REFCOUNTED_P(zv)) { | ADDREF_CONST zv, r0 } @@ -7513,18 +7349,18 @@ static int zend_jit_recv_init(dasm_State **Dst, const zend_op *opline, zend_op_a has_slow = 1; | SAVE_VALID_OPLINE opline |.if X64 - | lea CARG1, [FP + opline->result.var] - | mov r0, EX->func - | mov CARG2, [r0 + offsetof(zend_op_array, scope)] - | EXT_CALL zval_update_constant_ex, r0 + | LOAD_ZVAL_ADDR CARG1, res_addr + | mov r0, EX->func + | mov CARG2, [r0 + offsetof(zend_op_array, scope)] + | EXT_CALL zval_update_constant_ex, r0 |.else - | sub r4, 8 - | mov r0, EX->func - | push dword [r0 + offsetof(zend_op_array, scope)] - | lea r0, [FP + opline->result.var] - | push r0 - | EXT_CALL zval_update_constant_ex, r0 - | add r4, 16 + | sub r4, 8 + | mov r0, EX->func + | push dword [r0 + offsetof(zend_op_array, scope)] + | LOAD_ZVAL_ADDR r0, res_addr + | push r0 + | EXT_CALL zval_update_constant_ex, r0 + | add r4, 16 |.endif | test al, al | jnz >7 @@ -7543,7 +7379,7 @@ static int zend_jit_recv_init(dasm_State **Dst, const zend_op *opline, zend_op_a break; } has_slow += 2; - | lea r0, aword [FP + opline->result.var] + | LOAD_ZVAL_ADDR r0, res_addr | ZVAL_DEREF r0, MAY_BE_REF if (!ZEND_TYPE_IS_CLASS(arg_info->type)) { | cmp byte [r0 + 8], ZEND_TYPE_CODE(arg_info->type) @@ -7556,13 +7392,13 @@ static int zend_jit_recv_init(dasm_State **Dst, const zend_op *opline, zend_op_a | lea r0, [r0 + Z_CACHE_SLOT_P(zv)] | LOAD_ADDR FCARG2a, (ptrdiff_t)op_array |.if X64 - | mov CARG3, arg_num - | LOAD_ADDR CARG4, (ptrdiff_t)arg_info - | mov CARG5, r0 + | mov CARG3, arg_num + | LOAD_ADDR CARG4, (ptrdiff_t)arg_info + | mov CARG5, r0 |.else - | push r0 - | push (ptrdiff_t)arg_info - | push arg_num + | push r0 + | push (ptrdiff_t)arg_info + | push arg_num |.endif | EXT_CALL zend_jit_verify_arg_object, r0 } @@ -7583,8 +7419,8 @@ static int zend_jit_recv_init(dasm_State **Dst, const zend_op *opline, zend_op_a |.cold_code if (has_slow & 1) { |7: - | ZVAL_PTR_DTOR FP + opline->result.var, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN, 1, 0, 0, op_array->filename, opline - | SET_Z_TYPE_INFO FP + opline->result.var, IS_UNDEF + | ZVAL_PTR_DTOR res_addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN, 1, 0, 0, op_array->filename, opline + | SET_ZVAL_TYPE_INFO res_addr, IS_UNDEF | jmp <5 } if (has_slow & 2) { @@ -7594,15 +7430,15 @@ static int zend_jit_recv_init(dasm_State **Dst, const zend_op *opline, zend_op_a | lea r0, [r0 + Z_CACHE_SLOT_P(zv)] | LOAD_ADDR FCARG2a, (ptrdiff_t)op_array |.if X64 - | mov CARG3, arg_num - | LOAD_ADDR CARG4, (ptrdiff_t)arg_info - | mov CARG5, r0 - | LOAD_ADDR CARG6, zv + | mov CARG3, arg_num + | LOAD_ADDR CARG4, (ptrdiff_t)arg_info + | mov CARG5, r0 + | LOAD_ADDR CARG6, zv |.else - | push zv - | push r0 - | push (ptrdiff_t)arg_info - | push arg_num + | push zv + | push r0 + | push (ptrdiff_t)arg_info + | push arg_num |.endif | EXT_CALL zend_jit_verify_arg_slow, r0 | jmp <9 @@ -7671,6 +7507,10 @@ static int zend_jit_fetch_obj_read(dasm_State **Dst, zend_op *opline, zend_op_ar zend_class_entry *ce = NULL; zval *member; uint32_t offset; + zend_jit_addr op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1); + zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result); + zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); + zend_jit_addr prop_addr; if (opline->op2_type != IS_CONST) { goto fallback; @@ -7703,23 +7543,23 @@ static int zend_jit_fetch_obj_read(dasm_State **Dst, zend_op *opline, zend_op_ar offset = zend_get_known_property_offset(ce, Z_STR_P(member), opline->op1_type == IS_UNUSED, op_array->filename); if (opline->op1_type == IS_UNUSED) { - | IF_Z_TYPE FP + offsetof(zend_execute_data, This), IS_UNDEF, >1 + | IF_ZVAL_TYPE this_addr, IS_UNDEF, >1 |.cold_code |1: | SAVE_VALID_OPLINE opline | jmp ->not_obj |.code - | GET_Z_PTR FCARG1a, FP + offsetof(zend_execute_data, This) - } else if (op1_info & MAY_BE_REF) { - | lea r0, [FP + opline->op1.var] - | ZVAL_DEREF r0, op1_info - | IF_NOT_Z_TYPE r0, IS_OBJECT, >7 - | GET_Z_PTR FCARG1a, r0 + | GET_ZVAL_PTR FCARG1a, this_addr } else { + if (op1_info & MAY_BE_REF) { + | LOAD_ZVAL_ADDR r0, op1_addr + | ZVAL_DEREF r0, op1_info + op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); + } if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { - | IF_NOT_Z_TYPE FP + opline->op1.var, IS_OBJECT, >7 + | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >7 } - | GET_Z_PTR FCARG1a, FP + opline->op1.var + | GET_ZVAL_PTR FCARG1a, op1_addr } if (offset == ZEND_WRONG_PROPERTY_OFFSET) { @@ -7733,12 +7573,13 @@ static int zend_jit_fetch_obj_read(dasm_State **Dst, zend_op *opline, zend_op_ar | mov edx, dword [FCARG1a + r0 + 8] | IF_TYPE dl, IS_UNDEF, >5 | add FCARG1a, r0 - | GET_Z_PTR r0, FCARG1a + prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); } else { + prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, offset); | mov edx, dword [FCARG1a + offset + 8] | IF_TYPE dl, IS_UNDEF, >5 - | GET_Z_PTR r0, FCARG1a + offset } + | GET_ZVAL_PTR r0, prop_addr | IF_NOT_REFCOUNTED dh, >2 if (opline->opcode == ZEND_FETCH_OBJ_R) { | IF_TYPE dl, IS_REFERENCE, >6 @@ -7747,26 +7588,21 @@ static int zend_jit_fetch_obj_read(dasm_State **Dst, zend_op *opline, zend_op_ar | GC_ADDREF r0 |2: |.if X64 - | SET_Z_PTR FP + opline->result.var, r0 + | SET_ZVAL_PTR res_addr, r0 |.else - | SET_Z_PTR FP + opline->result.var, r0 - if (offset == ZEND_WRONG_PROPERTY_OFFSET) { - | GET_Z_W2 r0, FCARG1a - } else { - | GET_Z_W2 r0, FCARG1a + offset - } - | SET_Z_W2 FP + opline->result.var, r0 + | SET_ZVAL_PTR res_addr, r0 + | GET_ZVAL_W2 r0, prop_addr + | SET_ZVAL_W2 res_addr, r0 |.endif - | SET_Z_TYPE_INFO FP + opline->result.var, edx + | SET_ZVAL_TYPE_INFO res_addr, edx |.cold_code |5: | LOAD_ADDR FCARG2a, member |.if X64 - | lea CARG3, [FP + opline->result.var] + | LOAD_ZVAL_ADDR CARG3, res_addr |.else - | lea r0, [FP + opline->result.var] - | push r0 + | PUSH_ZVAL_ADDR res_addr, r0 |.endif | SAVE_VALID_OPLINE opline if (opline->opcode == ZEND_FETCH_OBJ_R) { @@ -7787,7 +7623,7 @@ static int zend_jit_fetch_obj_read(dasm_State **Dst, zend_op *opline, zend_op_ar } else { | lea FCARG2a, [FCARG1a + offset] } - | lea FCARG1a, [FP + opline->result.var] + | LOAD_ZVAL_ADDR FCARG1a, res_addr | EXT_CALL zend_jit_zval_copy_unref_helper, r0 | jmp >9 } @@ -7798,29 +7634,25 @@ static int zend_jit_fetch_obj_read(dasm_State **Dst, zend_op *opline, zend_op_ar | SAVE_VALID_OPLINE opline if (op1_info & MAY_BE_UNDEF) { if (op1_info & MAY_BE_ANY) { - if (op1_info & MAY_BE_REF) { - | IF_NOT_Z_TYPE r0, IS_UNDEF, >1 - } else { - | IF_NOT_Z_TYPE FP + opline->op1.var, IS_UNDEF, >1 - } + | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1 } | mov FCARG1d, opline->op1.var | EXT_CALL zend_jit_undefined_op_helper, r0 |1: } |.if X64 - | mov CARG1, E_NOTICE - | LOAD_ADDR CARG2, "Trying to get property of non-object" - | EXT_CALL zend_error, r0 + | mov CARG1, E_NOTICE + | LOAD_ADDR CARG2, "Trying to get property of non-object" + | EXT_CALL zend_error, r0 |.else - | sub r4, 8 - | push "Trying to get property of non-object" - | push E_NOTICE - | EXT_CALL zend_error, r0 - | add r4, 16 + | sub r4, 8 + | push "Trying to get property of non-object" + | push E_NOTICE + | EXT_CALL zend_error, r0 + | add r4, 16 |.endif } - | SET_Z_TYPE_INFO FP + opline->result.var, IS_NULL + | SET_ZVAL_TYPE_INFO res_addr, IS_NULL | jmp >9 } @@ -7828,10 +7660,9 @@ static int zend_jit_fetch_obj_read(dasm_State **Dst, zend_op *opline, zend_op_ar |8: | LOAD_ADDR FCARG2a, member |.if X64 - | lea CARG3, [FP + opline->result.var] + | LOAD_ZVAL_ADDR CARG3, res_addr |.else - | lea r0, [FP + opline->result.var] - | push r0 + | PUSH_ZVAL_ADDR res_addr, r0 |.endif | SAVE_VALID_OPLINE opline if (opline->opcode == ZEND_FETCH_OBJ_R) { @@ -7862,6 +7693,7 @@ fallback: static int zend_jit_free(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) { uint32_t op1_info = OP1_INFO(); + zend_jit_addr op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1); if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { if (zend_may_throw(opline, op_array, ssa)) { @@ -7869,7 +7701,7 @@ static int zend_jit_free(dasm_State **Dst, const zend_op *opline, zend_op_array } if (opline->opcode == ZEND_FE_FREE && (op1_info & (MAY_BE_OBJECT|MAY_BE_REF))) { if (op1_info & MAY_BE_ARRAY) { - | IF_Z_TYPE FP + opline->op1.var, IS_ARRAY, >7 + | IF_ZVAL_TYPE op1_addr, IS_ARRAY, >7 } | mov FCARG1d, dword [FP + opline->op1.var + offsetof(zval, u2.fe_iter_idx)] | cmp FCARG1d, -1 @@ -7877,7 +7709,7 @@ static int zend_jit_free(dasm_State **Dst, const zend_op *opline, zend_op_array | EXT_CALL zend_hash_iterator_del, r0 |7: } - | ZVAL_PTR_DTOR FP + opline->op1.var, op1_info, 0, 0, 0, op_array->filename, opline + | ZVAL_PTR_DTOR op1_addr, op1_info, 0, 0, 0, op_array->filename, opline if (zend_may_throw(opline, op_array, ssa)) { if (!zend_jit_check_exception(Dst)) { return 0; @@ -7888,6 +7720,76 @@ static int zend_jit_free(dasm_State **Dst, const zend_op *opline, zend_op_array return 1; } +static zend_bool zend_jit_opline_supports_reg(zend_op_array *op_array, zend_ssa *ssa, zend_op *opline, int var) +{ + uint32_t op1_info, op2_info; + + switch (opline->opcode) { + case ZEND_QM_ASSIGN: + return 1; + case ZEND_PRE_INC: + case ZEND_PRE_DEC: + case ZEND_POST_INC: + case ZEND_POST_DEC: + op1_info = OP1_INFO(); + return + !(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))); + case ZEND_ADD: + case ZEND_SUB: + case ZEND_MUL: + op1_info = OP1_INFO(); + op2_info = OP1_INFO(); + return + !(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) && + !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))); + case ZEND_IS_SMALLER: + case ZEND_IS_SMALLER_OR_EQUAL: + case ZEND_IS_EQUAL: + case ZEND_IS_NOT_EQUAL: + case ZEND_IS_IDENTICAL: + case ZEND_IS_NOT_IDENTICAL: + op1_info = OP1_INFO(); + op2_info = OP1_INFO(); + return + !(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) && + !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))); + case ZEND_BOOL: + case ZEND_BOOL_NOT: + case ZEND_JMPZ: + case ZEND_JMPNZ: + case ZEND_JMPZNZ: + case ZEND_JMPZ_EX: + case ZEND_JMPNZ_EX: + op1_info = OP1_INFO(); + return + !(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE))); + } + return 0; +} + +static zend_bool zend_jit_may_be_in_reg(zend_op_array *op_array, zend_ssa *ssa, int var) +{ + if ((ssa->var_info[var].type & MAY_BE_ANY) == MAY_BE_DOUBLE) { + if (ssa->vars[var].definition >= 0) { + if (!zend_jit_opline_supports_reg(op_array, ssa, op_array->opcodes + ssa->vars[var].definition, var)) { + return 0; + } + } + if (ssa->vars[var].use_chain >= 0) { + int use = ssa->vars[var].use_chain; + + do { + if (!zend_jit_opline_supports_reg(op_array, ssa, op_array->opcodes + use, var)) { + return 0; + } + use = zend_ssa_next_use(ssa->ops, var, use); + } while (use >= 0); + } + return 1; + } + return 0; +} + static zend_regset zend_jit_get_scratch_regset(zend_op_array *op_array, zend_ssa *ssa, zend_op *opline) { uint32_t op1_info, op2_info, res_info; diff --git a/ext/opcache/jit/zend_jit_x86.h b/ext/opcache/jit/zend_jit_x86.h index 713ce6873a1df..db2702696beaa 100644 --- a/ext/opcache/jit/zend_jit_x86.h +++ b/ext/opcache/jit/zend_jit_x86.h @@ -141,6 +141,50 @@ typedef uint32_t zend_regset; (ZEND_REGSET(ZREG_RAX) | ZEND_REGSET(ZREG_RCX) | ZEND_REGSET(ZREG_RDX) | ZEND_REGSET_FP) #endif +typedef uintptr_t zend_jit_addr; + +#define IS_CONST_ZVAL 0 +#define IS_MEM_ZVAL 1 +#define IS_REG 2 + +#define _ZEND_ADDR_MODE_MASK 0x3 +#define _ZEND_ADDR_REG_SHIFT 2 +#define _ZEND_ADDR_REG_MASK 0x3f +#define _ZEND_ADDR_OFFSET_SHIFT 8 + +#define ZEND_ADDR_CONST_ZVAL(zv) \ + (((zend_jit_addr)(uintptr_t)(zv)) | IS_CONST_ZVAL) +#define ZEND_ADDR_MEM_ZVAL(reg, offset) \ + ((((zend_jit_addr)(uintptr_t)(offset)) << _ZEND_ADDR_OFFSET_SHIFT) | \ + (((zend_jit_addr)(uintptr_t)(reg)) << _ZEND_ADDR_REG_SHIFT) | \ + IS_MEM_ZVAL) +#define ZEND_ADDR_REG(reg) \ + ((((zend_jit_addr)(uintptr_t)(reg)) << _ZEND_ADDR_REG_SHIFT) | \ + IS_REG) + +#define Z_MODE(addr) (((addr) & _ZEND_ADDR_MODE_MASK)) +#define Z_ZV(addr) ((zval*)(addr)) +#define Z_OFFSET(addr) ((uint32_t)((addr)>>_ZEND_ADDR_OFFSET_SHIFT)) +#define Z_REG(addr) ((zend_reg)(((addr)>>_ZEND_ADDR_REG_SHIFT) & _ZEND_ADDR_REG_MASK)) + +static zend_always_inline zend_jit_addr zend_jit_decode_op(const zend_op_array *op_array, zend_uchar op_type, znode_op op) +{ + if (op_type == IS_CONST) { +#if ZEND_USE_ABS_CONST_ADDR + return ZEND_ADDR_CONST_ZVAL(op.zv); +#else + return ZEND_ADDR_CONST_ZVAL(RT_CONSTANT(op_array, op)); +#endif + } else { + return ZEND_ADDR_MEM_ZVAL(ZREG_FP, op.var); + } +} + +static zend_always_inline zend_bool zend_jit_same_addr(zend_jit_addr addr1, zend_jit_addr addr2) +{ + return (addr1 == addr2); +} + #endif /* ZEND_JIT_X86_H */ /* * Local variables: From 17051f38f49aecc163ac7a1485e151529300a4c8 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 16 Mar 2017 12:53:06 +0300 Subject: [PATCH 377/569] Sort intervals (this is not necessary) --- ext/opcache/jit/zend_jit.c | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index ee43ffe7bbd12..c8e6b71dfe482 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -34,7 +34,7 @@ #include "Optimizer/zend_call_graph.h" #include "Optimizer/zend_dump.h" -//#define REG_ALLOC +#define REG_ALLOC #define DEBUG_REG_ALLOC //#define CONTEXT_THREADED_JIT #define PREFER_MAP_32BIT @@ -1535,6 +1535,23 @@ static int zend_jit_linear_scan(zend_op_array *op_array, zend_ssa *ssa, zend_lif } #ifdef DEBUG_REG_ALLOC + { + int count = 0; + zend_lifetime_interval *intervals[3]; + + if (handled) { + intervals[count++] = handled; + } + if (inactive) { + intervals[count++] = inactive; + } + if (active) { + intervals[count++] = active; + } + if (count) { + handled = zend_jit_sort_intervals(intervals, count); + } + } { zend_lifetime_interval *ival = handled; @@ -1547,7 +1564,7 @@ static int zend_jit_linear_scan(zend_op_array *op_array, zend_ssa *ssa, zend_lif } ival = ival->next; } - +/* ival = inactive; while (ival != NULL) { if (ival->reg > ZREG_NONE) { @@ -1567,6 +1584,7 @@ static int zend_jit_linear_scan(zend_op_array *op_array, zend_ssa *ssa, zend_lif } ival = ival->next; } +*/ } #endif From 65bbf4054f3579718282d9b635999f0eb5df4295 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 16 Mar 2017 12:53:36 +0300 Subject: [PATCH 378/569] Added code generatin rules for math on register-allocated variables --- ext/opcache/jit/zend_jit_x86.dasc | 121 ++++++++++++++++++++---------- 1 file changed, 80 insertions(+), 41 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 457d3a9e09c4a..1658cc34ea98a 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -287,6 +287,8 @@ static void* dasm_labels[zend_lb_MAX]; | .endif || } else if (Z_MODE(addr) == IS_MEM_ZVAL) { | sse_ins xmm(reg-ZREG_XMM0), qword [Ra(Z_REG(addr))+Z_OFFSET(addr)] +|| } else if (Z_MODE(addr) == IS_REG) { +| sse_ins xmm(reg-ZREG_XMM0), xmm(Z_REG(addr)-ZREG_XMM0) || } else { || ZEND_ASSERT(0); || } @@ -306,13 +308,17 @@ static void* dasm_labels[zend_lb_MAX]; | .endif || } else if (Z_MODE(addr) == IS_MEM_ZVAL) { | cvtsi2sd xmm(reg-ZREG_XMM0), aword [Ra(Z_REG(addr))+Z_OFFSET(addr)] +|| } else if (Z_MODE(addr) == IS_REG) { +| cvtsi2sd xmm(reg-ZREG_XMM0), Ra(Z_REG(addr)) || } else { || ZEND_ASSERT(0); || } |.endmacro |.macro SSE_GET_ZVAL_DVAL, reg, addr -| SSE_OP movsd, reg, addr +|| if (Z_MODE(addr) != IS_REG || reg != Z_REG(addr)) { +| SSE_OP movsd, reg, addr +|| } |.endmacro |.macro SSE_MATH, opcode, reg, addr @@ -358,8 +364,14 @@ static void* dasm_labels[zend_lb_MAX]; |.endmacro |.macro SSE_SET_ZVAL_DVAL, addr, reg -|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); -| movsd qword [Ra(Z_REG(addr))+Z_OFFSET(addr)], xmm(reg-ZREG_XMM0) +|| if (Z_MODE(addr) == IS_REG) { +|| if (reg != Z_REG(addr)) { +| movsd xmm(Z_REG(addr)-ZREG_XMM0), xmm(reg-ZREG_XMM0) +|| } +|| } else { +|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); +| movsd qword [Ra(Z_REG(addr))+Z_OFFSET(addr)], xmm(reg-ZREG_XMM0) +|| } |.endmacro |.macro LONG_OP, long_ins, reg, addr @@ -376,6 +388,8 @@ static void* dasm_labels[zend_lb_MAX]; | .endif || } else if (Z_MODE(addr) == IS_MEM_ZVAL) { | long_ins reg, aword [Ra(Z_REG(addr))+Z_OFFSET(addr)] +|| } else if (Z_MODE(addr) == IS_REG) { +| long_ins reg, Ra(Z_REG(addr)) || } else { || ZEND_ASSERT(0); || } @@ -399,15 +413,19 @@ static void* dasm_labels[zend_lb_MAX]; || if (Z_MODE(addr) == IS_CONST_ZVAL) { | .if X64 || if (!IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(addr)))) { -| mov64 reg, Z_LVAL_P(Z_ZV(addr)) +| mov64 Ra(reg), Z_LVAL_P(Z_ZV(addr)) || } else { -| mov reg, Z_LVAL_P(Z_ZV(addr)) +| mov Ra(reg), Z_LVAL_P(Z_ZV(addr)) || } | .else -| mov reg, Z_LVAL_P(Z_ZV(addr)) +| mov Ra(reg), Z_LVAL_P(Z_ZV(addr)) | .endif || } else if (Z_MODE(addr) == IS_MEM_ZVAL) { -| mov reg, aword [Ra(Z_REG(addr))+Z_OFFSET(addr)] +| mov Ra(reg), aword [Ra(Z_REG(addr))+Z_OFFSET(addr)] +|| } else if (Z_MODE(addr) == IS_REG) { +|| if (reg != Z_REG(addr)) { +| mov Ra(reg), Ra(Z_REG(addr)) +|| } || } else { || ZEND_ASSERT(0); || } @@ -456,8 +474,12 @@ static void* dasm_labels[zend_lb_MAX]; |.endmacro |.macro SET_ZVAL_LVAL, addr, lval -|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); -| mov aword [Ra(Z_REG(addr))+Z_OFFSET(addr)], lval +|| if (Z_MODE(addr) == IS_REG) { +| mov Ra(Z_REG(addr)), lval +|| } else { +|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); +| mov aword [Ra(Z_REG(addr))+Z_OFFSET(addr)], lval +|| } |.endmacro |.macro FPU_LONG_OP, fp_ins, addr @@ -474,6 +496,8 @@ static void* dasm_labels[zend_lb_MAX]; | .endif || } else if (Z_MODE(addr) == IS_MEM_ZVAL) { | fp_ins aword [Ra(Z_REG(addr))+Z_OFFSET(addr)] +|| } else if (Z_MODE(addr) == IS_REG) { +| fp_ins Ra(Z_REG(addr)) || } else { || ZEND_ASSERT(0); || } @@ -1040,7 +1064,7 @@ static void* dasm_labels[zend_lb_MAX]; || zend_reg tmp_reg; || || tmp_reg = (Z_REG(addr) == ZREG_FCARG1a) ? ZREG_R0 : ZREG_FCARG1a; -| GET_ZVAL_LVAL Ra(tmp_reg), addr +| GET_ZVAL_LVAL tmp_reg, addr || if (RC_MAY_BE_1(op_info)) { | cmp dword [Ra(tmp_reg)], 1 // if (GC_REFCOUNTED() > 1) || if (cold) { @@ -1071,7 +1095,7 @@ static void* dasm_labels[zend_lb_MAX]; || } |2: || } -| GET_ZVAL_LVAL FCARG1a, addr +| GET_ZVAL_LVAL ZREG_FCARG1a, addr |.endmacro |.macro EFREE_REG_24, op_array, opline @@ -1976,6 +2000,8 @@ static int zend_jit_math_long_long(dasm_State **Dst, uint32_t res_info) { zend_bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); + zend_reg result_reg = + (Z_MODE(res_addr) == IS_REG) ? Z_REG(res_addr) : ZREG_R0; if (opline->opcode == ZEND_MUL && ((Z_MODE(op2_addr) == IS_CONST_ZVAL && @@ -1985,23 +2011,23 @@ static int zend_jit_math_long_long(dasm_State **Dst, IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(op1_addr))) && is_power_of_two(Z_LVAL_P(Z_ZV(op1_addr)))))) { if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { - | GET_ZVAL_LVAL r0, op1_addr - | shl r0, floor_log2(Z_LVAL_P(Z_ZV(op2_addr))) + | GET_ZVAL_LVAL result_reg, op1_addr + | shl Ra(result_reg), floor_log2(Z_LVAL_P(Z_ZV(op2_addr))) } else { - | GET_ZVAL_LVAL r0, op2_addr - | shl r0, floor_log2(Z_LVAL_P(Z_ZV(op1_addr))) + | GET_ZVAL_LVAL result_reg, op2_addr + | shl Ra(result_reg), floor_log2(Z_LVAL_P(Z_ZV(op1_addr))) } } else if (opline->opcode == ZEND_DIV && (Z_MODE(op2_addr) == IS_CONST_ZVAL && is_power_of_two(Z_LVAL_P(Z_ZV(op2_addr))))) { - | GET_ZVAL_LVAL r0, op1_addr - | shr r0, floor_log2(Z_LVAL_P(Z_ZV(op2_addr))) + | GET_ZVAL_LVAL result_reg, op1_addr + | shr Ra(result_reg), floor_log2(Z_LVAL_P(Z_ZV(op2_addr))) } else { - | GET_ZVAL_LVAL r0, op1_addr + | GET_ZVAL_LVAL result_reg, op1_addr if (same_ops && opline->opcode != ZEND_DIV) { - | LONG_MATH_REG opline->opcode, r0, r0 + | LONG_MATH_REG opline->opcode, Ra(result_reg), Ra(result_reg) } else { - | LONG_MATH opline->opcode, r0, op2_addr + | LONG_MATH opline->opcode, Ra(result_reg), op2_addr } } if ((res_info & MAY_BE_DOUBLE) && zend_may_overflow(opline, op_array, ssa)) { @@ -2025,13 +2051,13 @@ static int zend_jit_math_long_long(dasm_State **Dst, | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE | jmp >2 |.code - | SET_ZVAL_LVAL res_addr, r0 + | SET_ZVAL_LVAL res_addr, Ra(result_reg) if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { | SET_ZVAL_TYPE_INFO res_addr, IS_LONG } |2: - } else { - | SET_ZVAL_LVAL res_addr, r0 + } else if (Z_MODE(res_addr) == IS_MEM_ZVAL) { + | SET_ZVAL_LVAL res_addr, Ra(result_reg) if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { | SET_ZVAL_TYPE_INFO res_addr, IS_LONG } @@ -2047,7 +2073,8 @@ static int zend_jit_math_long_double(dasm_State **Dst, zend_jit_addr res_addr) { |.if X64 or SSE - zend_reg result_reg = ZREG_XMM0; + zend_reg result_reg = + (Z_MODE(res_addr) == IS_REG) ? Z_REG(res_addr) : ZREG_XMM0; | SSE_GET_ZVAL_LVAL result_reg, op1_addr | SSE_MATH opline->opcode, result_reg, op2_addr @@ -2057,7 +2084,10 @@ static int zend_jit_math_long_double(dasm_State **Dst, | FPU_MATH opline->opcode, op2_addr | FPU_SET_ZVAL_DVAL res_addr |.endif - | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE + + if (Z_MODE(res_addr) == IS_MEM_ZVAL) { + | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE + } return 1; } @@ -2069,7 +2099,8 @@ static int zend_jit_math_double_long(dasm_State **Dst, zend_jit_addr res_addr) { |.if X64 or SSE - zend_reg result_reg = ZREG_XMM0; + zend_reg result_reg = + (Z_MODE(res_addr) == IS_REG) ? Z_REG(res_addr) : ZREG_XMM0; if (opline->opcode == ZEND_ADD || opline->opcode == ZEND_MUL || opline->opcode == ZEND_ASSIGN_ADD || opline->opcode == ZEND_ASSIGN_MUL) { | SSE_GET_ZVAL_LVAL result_reg, op2_addr @@ -2092,9 +2123,13 @@ static int zend_jit_math_double_long(dasm_State **Dst, } | FPU_SET_ZVAL_DVAL res_addr |.endif - if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { - | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE + + if (Z_MODE(res_addr) == IS_MEM_ZVAL) { + if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { + | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE + } } + return 1; } @@ -2107,7 +2142,8 @@ static int zend_jit_math_double_double(dasm_State **Dst, zend_bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); |.if X64 or SSE - zend_reg result_reg = ZREG_XMM0; + zend_reg result_reg = + (Z_MODE(res_addr) == IS_REG) ? Z_REG(res_addr) : ZREG_XMM0; | SSE_GET_ZVAL_DVAL result_reg, op1_addr if (same_ops) { @@ -2121,8 +2157,11 @@ static int zend_jit_math_double_double(dasm_State **Dst, | FPU_MATH opline->opcode, op2_addr | FPU_SET_ZVAL_DVAL res_addr |.endif - if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { - | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE + + if (Z_MODE(res_addr) == IS_MEM_ZVAL) { + if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { + | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE + } } return 1; @@ -2156,7 +2195,7 @@ static int zend_jit_shift(dasm_State **Dst, const zend_op *opline, zend_op_array if (op1_info & (MAY_BE_ANY-MAY_BE_LONG)) { | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >9 } - | GET_ZVAL_LVAL r0, op1_addr + | GET_ZVAL_LVAL ZREG_R0, op1_addr if (opline->opcode == ZEND_SR) { | shr, r0, Z_LVAL_P(Z_ZV(op2_addr)) } else { @@ -2574,7 +2613,7 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >3 } | // hval = Z_LVAL_P(dim); - | GET_ZVAL_LVAL FCARG2a, op2_addr + | GET_ZVAL_LVAL ZREG_FCARG2a, op2_addr if (op1_info & MAY_BE_ARRAY_KEY_LONG) { if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { zend_long val = Z_LVAL_P(Z_ZV(op2_addr)); @@ -2739,7 +2778,7 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o | IF_NOT_ZVAL_TYPE op2_addr, IS_STRING, >3 } | // offset_key = Z_STR_P(dim); - | GET_ZVAL_LVAL FCARG2a, op2_addr + | GET_ZVAL_LVAL ZREG_FCARG2a, op2_addr | // retval = zend_hash_find(ht, offset_key); switch (type) { case BP_JIT_IS: @@ -3178,7 +3217,7 @@ static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, zend_op_ if (Z_REG(op1_addr) != ZREG_FP) { | mov Ra(Z_REG(op1_addr)), [r4] // restore } - | GET_ZVAL_LVAL FCARG1a, op1_addr + | GET_ZVAL_LVAL ZREG_FCARG1a, op1_addr } if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { @@ -3285,7 +3324,7 @@ static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, zend_op_ if (Z_REG(op1_addr) != ZREG_FP) { | mov Ra(Z_REG(op1_addr)), [r4] // restore } - | GET_ZVAL_LVAL FCARG1a, op1_addr + | GET_ZVAL_LVAL ZREG_FCARG1a, op1_addr | // ZEND_VM_C_GOTO(assign_dim_op_new_array); | jmp <6 |2: @@ -3442,7 +3481,7 @@ static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, zend_ if (Z_REG(op1_addr) != ZREG_FP) { | mov Ra(Z_REG(op1_addr)), [r4] // restore } - | GET_ZVAL_LVAL FCARG1a, op1_addr + | GET_ZVAL_LVAL ZREG_FCARG1a, op1_addr } if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { @@ -3564,7 +3603,7 @@ static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, zend_ if (Z_REG(op1_addr) != ZREG_FP) { | mov Ra(Z_REG(op1_addr)), [r4] // restore } - | GET_ZVAL_LVAL FCARG1a, op1_addr + | GET_ZVAL_LVAL ZREG_FCARG1a, op1_addr | // ZEND_VM_C_GOTO(assign_dim_op_new_array); | jmp <6 |2: @@ -3707,7 +3746,7 @@ static int zend_jit_cmp_long_long(dasm_State **Dst, const zend_op *opline, int b } else if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_MODE(op1_addr) != IS_CONST_ZVAL) { | LONG_OP_MEM_CONST cmp, op1_addr, Z_LVAL_P(Z_ZV(op2_addr)) } else { - | GET_ZVAL_LVAL r0, op1_addr + | GET_ZVAL_LVAL ZREG_R0, op1_addr | LONG_OP cmp, r0, op2_addr } if (((opline+1)->opcode == ZEND_JMPZ_EX || @@ -6892,7 +6931,7 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, const zend_op *opline, zend if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7 } - | GET_ZVAL_LVAL FCARG1a, op1_addr + | GET_ZVAL_LVAL ZREG_FCARG1a, op1_addr if (!zend_jit_fetch_dimension_address_inner(Dst, opline, op_array, (opline->opcode == ZEND_FETCH_DIM_R) ? BP_VAR_R : BP_VAR_IS, op1_info, op2_info, 8, 9)) { return 0; } @@ -7069,7 +7108,7 @@ static int zend_jit_isset_isempty_dim(dasm_State **Dst, const zend_op *opline, i if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7 } - | GET_ZVAL_LVAL FCARG1a, op1_addr + | GET_ZVAL_LVAL ZREG_FCARG1a, op1_addr if (!zend_jit_fetch_dimension_address_inner(Dst, opline, op_array, BP_JIT_IS, op1_info, op2_info, 8, 9)) { return 0; } From 3b0c3ebb15b2c55f39e6590c8c98192622d13344 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 16 Mar 2017 16:46:30 +0300 Subject: [PATCH 379/569] Disable register-allocator (it's not ready yet) --- ext/opcache/jit/zend_jit.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index c8e6b71dfe482..47c8e92c3bcf4 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -34,7 +34,7 @@ #include "Optimizer/zend_call_graph.h" #include "Optimizer/zend_dump.h" -#define REG_ALLOC +//#define REG_ALLOC #define DEBUG_REG_ALLOC //#define CONTEXT_THREADED_JIT #define PREFER_MAP_32BIT From d0f33f6e0d1abafe463eb3b76bd49b2177cfb0a5 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 17 Mar 2017 14:30:24 +0300 Subject: [PATCH 380/569] Initial integration of register-allocator and code-generator --- ext/opcache/jit/zend_jit.c | 297 +++++++++++++++--------------- ext/opcache/jit/zend_jit.h | 11 ++ ext/opcache/jit/zend_jit_x86.dasc | 161 ++++++++-------- ext/opcache/jit/zend_jit_x86.h | 13 +- 4 files changed, 254 insertions(+), 228 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 47c8e92c3bcf4..c767784d8ec2e 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -35,7 +35,7 @@ #include "Optimizer/zend_dump.h" //#define REG_ALLOC -#define DEBUG_REG_ALLOC +//#define DEBUG_REG_ALLOC //#define CONTEXT_THREADED_JIT #define PREFER_MAP_32BIT //#define ZEND_JIT_RECORD @@ -960,17 +960,6 @@ static int zend_jit_op_array_analyze2(zend_op_array *op_array, zend_script *scri } #ifdef REG_ALLOC -typedef struct _zend_lifetime_interval zend_lifetime_interval; - -struct _zend_lifetime_interval { - int ssa_var; - int8_t reg; - zend_bool split; - uint32_t start; - uint32_t end; - zend_lifetime_interval *next; -}; - static int zend_jit_add_range(zend_lifetime_interval **intervals, int var, uint32_t from, uint32_t to) { zend_lifetime_interval *ival = intervals[var]; @@ -1011,26 +1000,6 @@ static int zend_jit_begin_range(zend_lifetime_interval **intervals, int var, uin return SUCCESS; } -static void zend_jit_free_intervals(zend_lifetime_interval **intervals, int count) -{ - int i; - - if (intervals) { - for (i = 0; i < count; i++) { - if (intervals[i]) { - zend_lifetime_interval *ival = intervals[i]; - - intervals[i] = NULL; - do { - zend_lifetime_interval *next = ival->next; - // TODO: efree(ival); - ival = next; - } while (ival); - } - } - } -} - static void zend_jit_insert_interval(zend_lifetime_interval **list, zend_lifetime_interval *ival) { while (1) { @@ -1047,13 +1016,20 @@ static void zend_jit_insert_interval(zend_lifetime_interval **list, zend_lifetim } } -static int zend_jit_split_interval(zend_lifetime_interval *current, uint32_t pos, zend_lifetime_interval **list) +static int zend_jit_split_interval(zend_lifetime_interval *current, uint32_t pos, zend_lifetime_interval **list, zend_lifetime_interval **free) { // TODO: emalloc(); - zend_lifetime_interval *ival = zend_arena_alloc(&CG(arena), sizeof(zend_lifetime_interval)); + zend_lifetime_interval *ival; - if (!ival) { - return FAILURE; + if (*free) { + ival = *free; + *free = ival->next; + } else { + ival = zend_arena_alloc(&CG(arena), sizeof(zend_lifetime_interval)); + + if (!ival) { + return FAILURE; + } } ival->ssa_var = current->ssa_var; @@ -1146,18 +1122,29 @@ static int zend_jit_compute_liveness(zend_op_array *op_array, zend_ssa *ssa, zen uint32_t *loop_end; zend_ssa_phi *phi; zend_lifetime_interval **intervals; + ALLOCA_FLAG(use_heap); set_size = zend_bitset_len(ssa->vars_count); - intervals = zend_arena_calloc(&CG(arena), ssa->vars_count, sizeof(zend_lifetime_interval*)); - live_in = zend_arena_calloc(&CG(arena), set_size * ssa->cfg.blocks_count, ZEND_BITSET_ELM_SIZE); - live = zend_arena_alloc(&CG(arena), set_size * ZEND_BITSET_ELM_SIZE); - pi_vars = zend_arena_alloc(&CG(arena), set_size * ZEND_BITSET_ELM_SIZE); - loop_end = zend_arena_calloc(&CG(arena), ssa->cfg.blocks_count, sizeof(uint32_t)); - - if (!intervals || !live || !live_in || !pi_vars) { - goto failure; + intervals = do_alloca( + ZEND_MM_ALIGNED_SIZE(ssa->vars_count * sizeof(zend_lifetime_interval*)) + + ZEND_MM_ALIGNED_SIZE((set_size * ssa->cfg.blocks_count + 2) * ZEND_BITSET_ELM_SIZE) + + ZEND_MM_ALIGNED_SIZE(ssa->cfg.blocks_count * sizeof(uint32_t)), + use_heap); + + if (!intervals) { + *list = NULL; + return FAILURE; } + live_in = (zend_bitset)((char*)intervals + ZEND_MM_ALIGNED_SIZE(ssa->vars_count * sizeof(zend_lifetime_interval*))); + live = (zend_bitset)((char*)live_in + ZEND_MM_ALIGNED_SIZE((set_size * ssa->cfg.blocks_count) * ZEND_BITSET_ELM_SIZE)); + pi_vars = (zend_bitset)((char*)live + ZEND_MM_ALIGNED_SIZE(set_size * ZEND_BITSET_ELM_SIZE)); + loop_end = (uint32_t*)((char*)pi_vars + ZEND_MM_ALIGNED_SIZE(set_size * ZEND_BITSET_ELM_SIZE)); + + memset(intervals, 0, ssa->vars_count * sizeof(zend_lifetime_interval*)); + zend_bitset_clear(live_in, set_size * ssa->cfg.blocks_count); + memset(loop_end, 0, ssa->cfg.blocks_count * sizeof(uint32_t)); + for (i = ssa->cfg.blocks_count - 1; i >= 0; i--) { zend_basic_block *b = ssa->cfg.blocks + i; @@ -1305,11 +1292,12 @@ static int zend_jit_compute_liveness(zend_op_array *op_array, zend_ssa *ssa, zen #endif *list = zend_jit_sort_intervals(intervals, ssa->vars_count); + free_alloca(intervals, use_heap); return SUCCESS; failure: - zend_jit_free_intervals(intervals, ssa->vars_count); *list = NULL; + free_alloca(intervals, use_heap); return FAILURE; } @@ -1317,7 +1305,7 @@ static int zend_jit_compute_liveness(zend_op_array *op_array, zend_ssa *ssa, zen Christian Wimmer VEE'05 (2005), Figure 4. Allocation without spilling. and "Linear Scan Register Allocation on SSA Form", Christian Wimmer and Michael Franz, CGO'10 (2010), Figure 6. */ -static int zend_jit_try_allocate_free_reg(zend_op_array *op_array, zend_ssa *ssa, zend_lifetime_interval *current, zend_regset available, zend_lifetime_interval *active, zend_lifetime_interval *inactive, zend_lifetime_interval **list) +static int zend_jit_try_allocate_free_reg(zend_op_array *op_array, zend_ssa *ssa, zend_lifetime_interval *current, zend_regset available, zend_lifetime_interval *active, zend_lifetime_interval *inactive, zend_lifetime_interval **list, zend_lifetime_interval **free) { zend_lifetime_interval *it; uint32_t freeUntilPos[ZREG_NUM]; @@ -1431,17 +1419,13 @@ static int zend_jit_try_allocate_free_reg(zend_op_array *op_array, zend_ssa *ssa current->reg = reg; return 1; } else { - /* TODO: enable interval splitting */ -#if 1 - return 0; -#else + /* TODO: enable interval splitting ??? */ /* register available for the first part of the interval */ - if (zend_jit_split_interval(current, pos, list) != SUCCESS) { + if (1 || zend_jit_split_interval(current, pos, list, free) != SUCCESS) { return 0; } current->reg = reg; return 1; -#endif } } @@ -1457,28 +1441,18 @@ static int zend_jit_allocate_blocked_reg(void) /* See "Optimized Interval Splitting in a Linear Scan Register Allocator", Christian Wimmer VEE'10 (2005), Figure 2. */ -static int zend_jit_linear_scan(zend_op_array *op_array, zend_ssa *ssa, zend_lifetime_interval *list) +static zend_lifetime_interval* zend_jit_linear_scan(zend_op_array *op_array, zend_ssa *ssa, zend_lifetime_interval *list) { - zend_lifetime_interval *unhandled, *active, *inactive, *handled; + zend_lifetime_interval *unhandled, *active, *inactive, *handled, *free; zend_lifetime_interval *current, **p, *q; uint32_t position; zend_regset available = ZEND_REGSET_UNION(ZEND_REGSET_GP, ZEND_REGSET_FP); - -#ifdef DEBUG_REG_ALLOC - { - zend_lifetime_interval *ival = list; - - fprintf(stderr, "Before Linear Scan\n"); - while (ival != NULL) { - fprintf(stderr, "%u-%u: #%d\n", ival->start, ival->end, ival->ssa_var); - ival = ival->next; - } - } -#endif + int count; + zend_lifetime_interval *intervals[3]; unhandled = list; - /* active = inactive = handled = {} */ - active = inactive = handled = NULL; + /* active = inactive = handled = free = {} */ + active = inactive = handled = free = NULL; while (unhandled != NULL) { current = unhandled; unhandled = unhandled->next; @@ -1523,92 +1497,53 @@ static int zend_jit_linear_scan(zend_op_array *op_array, zend_ssa *ssa, zend_lif } } - if (zend_jit_try_allocate_free_reg(op_array, ssa, current, available, active, inactive, &unhandled) || + if (zend_jit_try_allocate_free_reg(op_array, ssa, current, available, active, inactive, &unhandled, &free) || zend_jit_allocate_blocked_reg()) { ZEND_REGSET_EXCL(available, current->reg); current->next = active; active = current; } else { - current->next = handled; - handled = current; + current->next = free; + free = current; } } -#ifdef DEBUG_REG_ALLOC - { - int count = 0; - zend_lifetime_interval *intervals[3]; + count = 0; - if (handled) { - intervals[count++] = handled; - } - if (inactive) { - intervals[count++] = inactive; - } - if (active) { - intervals[count++] = active; - } - if (count) { - handled = zend_jit_sort_intervals(intervals, count); - } + if (handled) { + intervals[count++] = handled; } - { - zend_lifetime_interval *ival = handled; - - fprintf(stderr, "After Linear Scan\n"); - while (ival != NULL) { - if (ival->reg > ZREG_NONE) { - fprintf(stderr, "%u-%u: #%d (%s)\n", ival->start, ival->end, ival->ssa_var, zend_reg_name[ival->reg]); - } else { - fprintf(stderr, "%u-%u: #%d (no-reg)\n", ival->start, ival->end, ival->ssa_var); - } - ival = ival->next; - } -/* - ival = inactive; - while (ival != NULL) { - if (ival->reg > ZREG_NONE) { - fprintf(stderr, "%u-%u: #%d (%s)\n", ival->start, ival->end, ival->ssa_var, zend_reg_name[ival->reg]); - } else { - fprintf(stderr, "%u-%u: #%d (no-reg)\n", ival->start, ival->end, ival->ssa_var); - } - ival = ival->next; - } - - ival = active; - while (ival != NULL) { - if (ival->reg > ZREG_NONE) { - fprintf(stderr, "%u-%u: #%d (%s)\n", ival->start, ival->end, ival->ssa_var, zend_reg_name[ival->reg]); - } else { - fprintf(stderr, "%u-%u: #%d (no-reg)\n", ival->start, ival->end, ival->ssa_var); - } - ival = ival->next; - } -*/ + if (inactive) { + intervals[count++] = inactive; + } + if (active) { + intervals[count++] = active; + } + if (count) { + return zend_jit_sort_intervals(intervals, count); + } else { + return NULL; } -#endif - - return SUCCESS; } -static int zend_jit_allocate_registers(zend_op_array *op_array, zend_ssa *ssa) +static zend_lifetime_interval** zend_jit_allocate_registers(zend_op_array *op_array, zend_ssa *ssa) { void *checkpoint; int set_size, candidates_count, i; - zend_bitset candidates; - zend_lifetime_interval *list; + zend_bitset candidates = NULL; + zend_lifetime_interval *list, *ival; + zend_lifetime_interval **intervals; + ALLOCA_FLAG(use_heap); if (!ssa->var_info) { - return SUCCESS; + return NULL; } - checkpoint = zend_arena_checkpoint(CG(arena)); - /* Identify SSA variables suitable for register allocation */ set_size = zend_bitset_len(ssa->vars_count); - candidates = zend_arena_alloc(&CG(arena), set_size * ZEND_BITSET_ELM_SIZE); + candidates = ZEND_BITSET_ALLOCA(set_size, use_heap); if (!candidates) { - goto failure; + return NULL; } candidates_count = 0; zend_bitset_clear(candidates, set_size); @@ -1621,26 +1556,80 @@ static int zend_jit_allocate_registers(zend_op_array *op_array, zend_ssa *ssa) } } if (!candidates_count) { - goto success; + free_alloca(candidates, use_heap); + return NULL; } + checkpoint = zend_arena_checkpoint(CG(arena)); + /* Find life-time intervals */ if (zend_jit_compute_liveness(op_array, ssa, candidates, &list) != SUCCESS) { goto failure; } - /* Linear Scan Register Allocation */ - if (zend_jit_linear_scan(op_array, ssa, list) != SUCCESS) { - goto failure; - } + if (list) { +#ifdef DEBUG_REG_ALLOC + fprintf(stderr, "Before Linear Scan\n"); + ival = list; + while (ival != NULL) { + fprintf(stderr, "%u-%u: #%d\n", ival->start, ival->end, ival->ssa_var); + ival = ival->next; + } +#endif -success: - zend_arena_release(&CG(arena), checkpoint); - return SUCCESS; + /* Linear Scan Register Allocation */ + list = zend_jit_linear_scan(op_array, ssa, list); + + if (list) { +#ifdef DEBUG_REG_ALLOC + fprintf(stderr, "After Linear Scan\n"); + ival = list; + while (ival != NULL) { + fprintf(stderr, "%u-%u: #%d (%s)\n", ival->start, ival->end, ival->ssa_var, zend_reg_name[ival->reg]); + ival = ival->next; + } +#endif + + intervals = zend_arena_calloc(&CG(arena), ssa->vars_count, sizeof(zend_lifetime_interval*)); + if (!intervals) { + goto failure; + } + + ival = list; + while (ival != NULL) { + zend_lifetime_interval *next = ival->next; + + ival->next = intervals[ival->ssa_var]; + intervals[ival->ssa_var] = ival; + ival = next; + } + +#ifdef DEBUG_REG_ALLOC + fprintf(stderr, "Allocated Live Ranges\n"); + for (i = 0; i < ssa->vars_count; i++) { + if (intervals[i]) { + ival = intervals[i]; + + fprintf(stderr, "#%d: %u-%u (%s)", ival->ssa_var, ival->start, ival->end, zend_reg_name[ival->reg]); + ival = ival->next; + while (ival) { + fprintf(stderr, ", %u-%u (%s)", ival->start, ival->end, zend_reg_name[ival->reg]); + ival = ival->next; + } + fprintf(stderr, "\n"); + } + } +#endif + + free_alloca(candidates, use_heap); + return intervals; + } + } failure: zend_arena_release(&CG(arena), checkpoint); - return FAILURE; + free_alloca(candidates, use_heap); + return NULL; } #endif @@ -1651,6 +1640,10 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa, const zend_op *rt_op dasm_State* dasm_state = NULL; void *handler; int call_level = 0; +#ifdef REG_ALLOC + void *checkpoint; +#endif + zend_lifetime_interval **ra = NULL; #ifdef ZEND_JIT_FILTER const char *names[] = { @@ -1681,6 +1674,11 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa, const zend_op *rt_op pass: #endif +#ifdef REG_ALLOC + checkpoint = zend_arena_checkpoint(CG(arena)); + ra = zend_jit_allocate_registers(op_array, ssa); +#endif + /* mark hidden branch targets */ for (b = 0; b < ssa->cfg.blocks_count; b++) { if (ssa->cfg.blocks[b].flags & ZEND_BB_REACHABLE && @@ -1804,7 +1802,7 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa, const zend_op *rt_op case ZEND_SUB: case ZEND_MUL: // case ZEND_DIV: // TODO: check for division by zero ??? - if (!zend_jit_math(&dasm_state, opline, &i, op_array, ssa)) { + if (!zend_jit_math(&dasm_state, opline, &i, op_array, ssa, ra)) { goto jit_failure; } goto done; @@ -2126,12 +2124,18 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa, const zend_op *rt_op } dasm_free(&dasm_state); +#ifdef REG_ALLOC + zend_arena_release(&CG(arena), checkpoint); +#endif return SUCCESS; jit_failure: if (dasm_state) { dasm_free(&dasm_state); } +#ifdef REG_ALLOC + zend_arena_release(&CG(arena), checkpoint); +#endif return FAILURE; } @@ -2169,12 +2173,6 @@ static int zend_real_jit_func(zend_op_array *op_array, zend_script *script, cons goto jit_failure; } -#ifdef REG_ALLOC - if (zend_jit_allocate_registers(op_array, &ssa) != SUCCESS) { - goto jit_failure; - } -#endif - if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_SSA) { zend_dump_op_array(op_array, ZEND_DUMP_HIDE_UNREACHABLE|ZEND_DUMP_RC_INFERENCE|ZEND_DUMP_SSA|ZEND_DUMP_RT_CONSTANTS, "JIT", &ssa); } @@ -2465,11 +2463,6 @@ ZEND_API int zend_jit_script(zend_script *script) if (zend_jit_op_array_analyze2(call_graph.op_arrays[i], script, &info->ssa, &info->flags) != SUCCESS) { goto jit_failure; } -#ifdef REG_ALLOC - if (zend_jit_allocate_registers(call_graph.op_arrays[i], &info->ssa) != SUCCESS) { - goto jit_failure; - } -#endif } } diff --git a/ext/opcache/jit/zend_jit.h b/ext/opcache/jit/zend_jit.h index 2108908aad5c1..2d5651c655126 100644 --- a/ext/opcache/jit/zend_jit.h +++ b/ext/opcache/jit/zend_jit.h @@ -72,6 +72,17 @@ ZEND_API void zend_jit_activate(void); ZEND_API void zend_jit_deactivate(void); ZEND_API void zend_jit_status(zval *ret); +typedef struct _zend_lifetime_interval zend_lifetime_interval; + +struct _zend_lifetime_interval { + int ssa_var; + int8_t reg; + zend_bool split; + uint32_t start; + uint32_t end; + zend_lifetime_interval *next; +}; + #endif /* HAVE_JIT_H */ /* diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 1658cc34ea98a..c8f06369c1af8 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -1869,9 +1869,9 @@ static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, zend_op_arr goto fallback; } - op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1); + op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, NULL, -1); if (opline->result_type != IS_UNUSED) { - res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result); + res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1); } if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)-MAY_BE_LONG)) { @@ -2179,9 +2179,9 @@ static int zend_jit_shift(dasm_State **Dst, const zend_op *opline, zend_op_array op1_info = OP1_INFO(); - op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1); - op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2); - res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result); + op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, NULL, -1); + op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2, opline, NULL, -1); + res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1); if ((op1_info & MAY_BE_UNDEF) || Z_MODE(op2_addr) != IS_CONST_ZVAL || @@ -2429,7 +2429,7 @@ static int zend_jit_math_helper(dasm_State **Dst, return 1; } -static int zend_jit_math(dasm_State **Dst, const zend_op *opline, int *opnum, zend_op_array *op_array, zend_ssa *ssa) +static int zend_jit_math(dasm_State **Dst, const zend_op *opline, int *opnum, zend_op_array *op_array, zend_ssa *ssa, zend_lifetime_interval **ra) { uint32_t op1_info, op2_info; zend_jit_addr op1_addr, op2_addr, res_addr; @@ -2450,8 +2450,8 @@ static int zend_jit_math(dasm_State **Dst, const zend_op *opline, int *opnum, ze goto fallback; } - op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1); - op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2); + op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, ra, ssa->ops[opline - op_array->opcodes].op1_use); + op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2, opline, ra, ssa->ops[opline - op_array->opcodes].op2_use); if (opline->result_type == IS_TMP_VAR && (opline+1)->opcode == ZEND_SEND_VAL && @@ -2466,7 +2466,7 @@ static int zend_jit_math(dasm_State **Dst, const zend_op *opline, int *opnum, ze } res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var); } else { - res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result); + res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, ra, ssa->ops[opline - op_array->opcodes].result_def); } return zend_jit_math_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, res_addr, RES_INFO(), 0); @@ -2576,8 +2576,8 @@ static int zend_jit_concat(dasm_State **Dst, const zend_op *opline, int *opnum, goto fallback; } - op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1); - op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2); + op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, NULL, -1); + op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2, opline, NULL, -1); if (opline->result_type == IS_TMP_VAR && (opline+1)->opcode == ZEND_SEND_VAL && @@ -2592,7 +2592,7 @@ static int zend_jit_concat(dasm_State **Dst, const zend_op *opline, int *opnum, } res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var); } else { - res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result); + res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1); } return zend_jit_concat_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, res_addr, RES_INFO()); @@ -2604,8 +2604,8 @@ fallback: static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, uint32_t type, uint32_t op1_info, uint32_t op2_info, uint32_t found, uint32_t not_found) /* Labels: 1,2,3,4,5 */ { - zend_jit_addr op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2); - zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result); + zend_jit_addr op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2, opline, NULL, -1); + zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1); if (op2_info & MAY_BE_LONG) { if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_LONG)) { @@ -3157,13 +3157,13 @@ static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, zend_op_ op2_info = OP2_INFO(); val_info = OP1_DATA_INFO(); - op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1); - op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2); - op3_addr = zend_jit_decode_op(op_array, (opline+1)->op1_type, (opline+1)->op1); + op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, NULL, -1); + op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2, opline, NULL, -1); + op3_addr = zend_jit_decode_op(op_array, (opline+1)->op1_type, (opline+1)->op1, opline, NULL, -1); if (opline->result_type == IS_UNUSED) { res_addr = 0; } else { - res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result); + res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1); } if (op1_info & MAY_BE_REF) { @@ -3349,7 +3349,7 @@ static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, zend_op_ } else { zend_jit_addr res_addr; - res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result); + res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1); |.if X64 | LOAD_ZVAL_ADDR CARG4, res_addr |.else @@ -3417,9 +3417,9 @@ static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, zend_ op1_info = OP1_INFO(); op2_info = OP2_INFO(); - op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1); - op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2); - op3_addr = zend_jit_decode_op(op_array, (opline+1)->op1_type, (opline+1)->op1); + op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, NULL, -1); + op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2, opline, NULL, -1); + op3_addr = zend_jit_decode_op(op_array, (opline+1)->op1_type, (opline+1)->op1, opline, NULL, -1); if (op1_info & MAY_BE_REF) { | LOAD_ZVAL_ADDR FCARG1a, op1_addr @@ -3683,8 +3683,8 @@ static int zend_jit_assign_op(dasm_State **Dst, const zend_op *opline, zend_op_a op1_info = OP1_INFO(); op2_info = OP2_INFO(); - op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1); - op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2); + op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, NULL, -1); + op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2, opline, NULL, -1); if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) { goto fallback; @@ -3737,8 +3737,8 @@ static int zend_jit_cmp_long_long(dasm_State **Dst, const zend_op *opline, int b { unsigned int target_label; int swap = 0; - zend_jit_addr op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1); - zend_jit_addr op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2); + zend_jit_addr op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, NULL, -1); + zend_jit_addr op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2, opline, NULL, -1); if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) != IS_CONST_ZVAL) { | LONG_OP_MEM_CONST cmp, op2_addr, Z_LVAL_P(Z_ZV(op1_addr)) @@ -3753,7 +3753,7 @@ static int zend_jit_cmp_long_long(dasm_State **Dst, const zend_op *opline, int b (opline+1)->opcode == ZEND_JMPNZ_EX) && (opline+1)->op1_type == IS_TMP_VAR && (opline+1)->op1.var == opline->result.var) { - zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result); + zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1); switch (opline->opcode) { case ZEND_IS_EQUAL: @@ -3876,7 +3876,7 @@ static int zend_jit_cmp_long_long(dasm_State **Dst, const zend_op *opline, int b target_label = ssa->cfg.blocks[b].successors[1]; | jmp => target_label } else { - zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result); + zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1); switch (opline->opcode) { case ZEND_IS_EQUAL: @@ -3995,7 +3995,7 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, i } else if ((opline+1)->opcode == ZEND_JMPZ_EX && (opline+1)->op1_type == IS_TMP_VAR && (opline+1)->op1.var == opline->result.var) { - zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result); + zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1); target_label = ssa->cfg.blocks[b].successors[0]; switch (opline->opcode) { @@ -4034,7 +4034,7 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, i } else if ((opline+1)->opcode == ZEND_JMPNZ_EX && (opline+1)->op1_type == IS_TMP_VAR && (opline+1)->op1.var == opline->result.var) { - zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result); + zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1); target_label = ssa->cfg.blocks[b].successors[0]; switch (opline->opcode) { @@ -4071,7 +4071,7 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, i ZEND_ASSERT(0); } } else { - zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result); + zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1); switch (opline->opcode) { case ZEND_IS_EQUAL: @@ -4112,8 +4112,8 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, i static int zend_jit_cmp_long_double(dasm_State **Dst, const zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa) { - zend_jit_addr op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1); - zend_jit_addr op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2); + zend_jit_addr op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, NULL, -1); + zend_jit_addr op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2, opline, NULL, -1); |.if X64 or SSE zend_reg tmp_reg = ZREG_XMM0; @@ -4132,8 +4132,8 @@ static int zend_jit_cmp_long_double(dasm_State **Dst, const zend_op *opline, int static int zend_jit_cmp_double_long(dasm_State **Dst, const zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa) { - zend_jit_addr op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1); - zend_jit_addr op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2); + zend_jit_addr op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, NULL, -1); + zend_jit_addr op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2, opline, NULL, -1); |.if X64 or SSE zend_reg tmp_reg1 = ZREG_XMM0; @@ -4154,8 +4154,8 @@ static int zend_jit_cmp_double_long(dasm_State **Dst, const zend_op *opline, int static int zend_jit_cmp_double_double(dasm_State **Dst, const zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa) { - zend_jit_addr op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1); - zend_jit_addr op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2); + zend_jit_addr op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, NULL, -1); + zend_jit_addr op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2, opline, NULL, -1); |.if X64 or SSE zend_reg tmp_reg = ZREG_XMM0; @@ -4175,7 +4175,7 @@ static int zend_jit_cmp_double_double(dasm_State **Dst, const zend_op *opline, i static int zend_jit_cmp_slow(dasm_State **Dst, const zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa) { unsigned int target_label; - zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result); + zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1); | LONG_OP_MEM_CONST cmp, res_addr, Z_L(0) if (((opline+1)->opcode == ZEND_JMPZ_EX || @@ -4301,9 +4301,9 @@ static int zend_jit_cmp(dasm_State **Dst, const zend_op *opline, int b, int *opn uint32_t op1_info, op2_info; zend_bool same_ops = (opline->op1_type == opline->op2_type) && (opline->op1.var == opline->op2.var); zend_bool has_slow; - zend_jit_addr op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1); - zend_jit_addr op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2); - zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result); + zend_jit_addr op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, NULL, -1); + zend_jit_addr op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2, opline, NULL, -1); + zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1); op1_info = OP1_INFO(); op2_info = OP2_INFO(); @@ -4523,9 +4523,9 @@ static int zend_jit_identical(dasm_State **Dst, const zend_op *opline, int b, in uint32_t identical_label = (uint32_t)-1; uint32_t not_identical_label = (uint32_t)-1; uint32_t op1_info, op2_info; - zend_jit_addr op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1); - zend_jit_addr op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2); - zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result); + zend_jit_addr op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, NULL, -1); + zend_jit_addr op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2, opline, NULL, -1); + zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1); op1_info = OP1_INFO(); op2_info = OP2_INFO(); @@ -4837,7 +4837,7 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, int b, zend_bool set_bool = 0; zend_bool set_bool_not = 0; zend_bool jmp_done = 0; - zend_jit_addr op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1); + zend_jit_addr op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, NULL, -1); zend_jit_addr res_addr; if (opline->opcode == ZEND_JMPZ) { @@ -4863,7 +4863,7 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, int b, } if (set_bool) { - res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result); + res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1); } if (Z_MODE(op1_addr) == IS_CONST_ZVAL) { @@ -5146,8 +5146,8 @@ static int zend_jit_qm_assign(dasm_State **Dst, const zend_op *opline, zend_op_a uint32_t op1_info = OP1_INFO(); zend_jit_addr op1_addr, res_addr; - op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1); - res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result); + op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, NULL, -1); + res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1); return zend_jit_simple_assign(Dst, opline, op_array, ssa, res_addr, -1, opline->op1_type, opline->op1, op1_addr, op1_info, 0, 0); } @@ -5163,12 +5163,12 @@ static int zend_jit_assign(dasm_State **Dst, const zend_op *opline, zend_op_arra op1_info = OP1_INFO(); op2_info = OP2_INFO(); - op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1); - op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2); + op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, NULL, -1); + op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2, opline, NULL, -1); if (opline->result_type == IS_UNUSED) { res_addr = 0; } else { - res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result); + res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1); } if (op1_info & MAY_BE_REF) { @@ -5475,7 +5475,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar zend_jit_addr res_addr; if (RETURN_VALUE_USED(opline)) { - res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result); + res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1); } else { /* CPU stack alocated temorary zval */ res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R4, 8); @@ -5953,7 +5953,7 @@ static int zend_jit_send_val(dasm_State **Dst, const zend_op *opline, zend_op_ar |.code } - op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1); + op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, NULL, -1); arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var); if (opline->op1_type == IS_CONST) { @@ -5979,7 +5979,7 @@ static int zend_jit_send_ref(dasm_State **Dst, const zend_op *opline, zend_op_ar uint32_t op1_info; zend_jit_addr op1_addr, arg_addr, ref_addr; - op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1); + op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, NULL, -1); arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var); op1_info = OP1_INFO(); @@ -6100,7 +6100,7 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, zend_op_ar goto fallback; } - op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1); + op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, NULL, -1); arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var); op1_info = OP1_INFO(); @@ -6274,7 +6274,7 @@ static int zend_jit_smart_true(dasm_State **Dst, const zend_op *opline, int b, z target_label = ssa->cfg.blocks[b].successors[1]; | jmp =>target_label } else { - zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result); + zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1); | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE if (jmp) { @@ -6306,7 +6306,7 @@ static int zend_jit_smart_false(dasm_State **Dst, const zend_op *opline, int b, target_label = ssa->cfg.blocks[b].successors[0]; | jmp =>target_label } else { - zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result); + zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1); | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE if (jmp) { @@ -6365,7 +6365,7 @@ static int zend_jit_defined(dasm_State **Dst, const zend_op *opline, int b, int | jz >3 } } else { - res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result); + res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1); } | mov r0, EX->run_time_cache | mov r0, aword [r0 + Z_CACHE_SLOT_P(zv)] @@ -6399,7 +6399,7 @@ static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, int b, i uint32_t target_label; zend_uchar type; zend_bool smart_branch = 0; - zend_jit_addr op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1); + zend_jit_addr op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, NULL, -1); if (opline->extended_value == IS_RESOURCE) { // TODO: support for is_resource() ??? @@ -6531,7 +6531,7 @@ static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, int b, i target_label = ssa->cfg.blocks[b].successors[1]; | jmp =>target_label } else { - zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result); + zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1); | setne al | movzx eax, al @@ -6608,7 +6608,7 @@ static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, int b, i target_label = ssa->cfg.blocks[b].successors[1]; | jmp =>target_label } else { - zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result); + zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1); | sete al | movzx eax, al @@ -6792,7 +6792,7 @@ static int zend_jit_return(dasm_State **Dst, const zend_op *opline, zend_op_arra } op1_info = OP1_INFO(); - op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1); + op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, NULL, -1); if (opline->op1_type == IS_CV && (op1_info & MAY_BE_UNDEF)) { // TODO: support for IS_UNDEF ??? return zend_jit_tail_handler(Dst, opline); @@ -6917,9 +6917,9 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, const zend_op *opline, zend op2_info = OP2_INFO(); res_info = RES_INFO(); - op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1); - op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2); - res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result); + op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, NULL, -1); + op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2, opline, NULL, -1); + res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1); if (op1_info & MAY_BE_REF) { | LOAD_ZVAL_ADDR FCARG1a, op1_addr @@ -7094,9 +7094,9 @@ static int zend_jit_isset_isempty_dim(dasm_State **Dst, const zend_op *opline, i op1_info = OP1_INFO(); op2_info = OP2_INFO(); - op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1); - op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2); - res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result); + op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, NULL, -1); + op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2, opline, NULL, -1); + res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1); if (op1_info & MAY_BE_REF) { | LOAD_ZVAL_ADDR FCARG1a, op1_addr @@ -7213,7 +7213,7 @@ fallback: static int zend_jit_bind_global(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) { uint32_t op1_info; - zend_jit_addr op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1); + zend_jit_addr op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, NULL, -1); zval *varname = RT_CONSTANT(op_array, opline->op2); if (!ssa->ops || !ssa->var_info) { @@ -7376,7 +7376,7 @@ static int zend_jit_recv_init(dasm_State **Dst, const zend_op *opline, zend_op_a zend_bool has_slow = 0; uint32_t arg_num = opline->op1.num; zval *zv = RT_CONSTANT(op_array, opline->op2); - zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result); + zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1); | cmp dword EX->This.u2.num_args, arg_num | jae >5 @@ -7546,8 +7546,8 @@ static int zend_jit_fetch_obj_read(dasm_State **Dst, zend_op *opline, zend_op_ar zend_class_entry *ce = NULL; zval *member; uint32_t offset; - zend_jit_addr op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1); - zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result); + zend_jit_addr op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, NULL, -1); + zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1); zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); zend_jit_addr prop_addr; @@ -7732,7 +7732,7 @@ fallback: static int zend_jit_free(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) { uint32_t op1_info = OP1_INFO(); - zend_jit_addr op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1); + zend_jit_addr op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, NULL, -1); if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { if (zend_may_throw(opline, op_array, ssa)) { @@ -7759,11 +7759,13 @@ static int zend_jit_free(dasm_State **Dst, const zend_op *opline, zend_op_array return 1; } +#ifdef REG_ALLOC static zend_bool zend_jit_opline_supports_reg(zend_op_array *op_array, zend_ssa *ssa, zend_op *opline, int var) { uint32_t op1_info, op2_info; switch (opline->opcode) { +#if 0 case ZEND_QM_ASSIGN: return 1; case ZEND_PRE_INC: @@ -7773,14 +7775,16 @@ static zend_bool zend_jit_opline_supports_reg(zend_op_array *op_array, zend_ssa op1_info = OP1_INFO(); return !(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))); +#endif case ZEND_ADD: case ZEND_SUB: case ZEND_MUL: op1_info = OP1_INFO(); - op2_info = OP1_INFO(); + op2_info = OP2_INFO(); return !(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) && !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))); +#if 0 case ZEND_IS_SMALLER: case ZEND_IS_SMALLER_OR_EQUAL: case ZEND_IS_EQUAL: @@ -7788,7 +7792,7 @@ static zend_bool zend_jit_opline_supports_reg(zend_op_array *op_array, zend_ssa case ZEND_IS_IDENTICAL: case ZEND_IS_NOT_IDENTICAL: op1_info = OP1_INFO(); - op2_info = OP1_INFO(); + op2_info = OP2_INFO(); return !(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) && !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))); @@ -7802,12 +7806,18 @@ static zend_bool zend_jit_opline_supports_reg(zend_op_array *op_array, zend_ssa op1_info = OP1_INFO(); return !(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE))); +#endif } return 0; } static zend_bool zend_jit_may_be_in_reg(zend_op_array *op_array, zend_ssa *ssa, int var) { + // TODO: enable global register allocation ??? + if (ssa->vars[var].definition_phi || ssa->vars[var].phi_use_chain) { + return 0; + } + if ((ssa->var_info[var].type & MAY_BE_ANY) == MAY_BE_DOUBLE) { if (ssa->vars[var].definition >= 0) { if (!zend_jit_opline_supports_reg(op_array, ssa, op_array->opcodes + ssa->vars[var].definition, var)) { @@ -7935,6 +7945,7 @@ static zend_regset zend_jit_get_scratch_regset(zend_op_array *op_array, zend_ssa return regset; } +#endif #ifdef ZEND_JIT_RECORD static int zend_jit_func_header(dasm_State **Dst, zend_op_array *op_array) diff --git a/ext/opcache/jit/zend_jit_x86.h b/ext/opcache/jit/zend_jit_x86.h index db2702696beaa..3514d7d56584b 100644 --- a/ext/opcache/jit/zend_jit_x86.h +++ b/ext/opcache/jit/zend_jit_x86.h @@ -167,7 +167,7 @@ typedef uintptr_t zend_jit_addr; #define Z_OFFSET(addr) ((uint32_t)((addr)>>_ZEND_ADDR_OFFSET_SHIFT)) #define Z_REG(addr) ((zend_reg)(((addr)>>_ZEND_ADDR_REG_SHIFT) & _ZEND_ADDR_REG_MASK)) -static zend_always_inline zend_jit_addr zend_jit_decode_op(const zend_op_array *op_array, zend_uchar op_type, znode_op op) +static zend_always_inline zend_jit_addr zend_jit_decode_op(const zend_op_array *op_array, zend_uchar op_type, znode_op op, const zend_op *opline, zend_lifetime_interval **ra, int ssa_var) { if (op_type == IS_CONST) { #if ZEND_USE_ABS_CONST_ADDR @@ -176,6 +176,17 @@ static zend_always_inline zend_jit_addr zend_jit_decode_op(const zend_op_array * return ZEND_ADDR_CONST_ZVAL(RT_CONSTANT(op_array, op)); #endif } else { + if (ra && ssa_var >= 0 && ra[ssa_var]) { + zend_lifetime_interval *ival = ra[ssa_var]; + uint32_t line = opline - op_array->opcodes; + + do { + if (line >= ival->start && line <= ival->end) { + return ZEND_ADDR_REG(ival->reg); + } + ival = ival->next; + } while (ival); + } return ZEND_ADDR_MEM_ZVAL(ZREG_FP, op.var); } } From d6843daab60f5052517ba8891a0d32e6931a026f Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 20 Mar 2017 12:44:26 +0300 Subject: [PATCH 381/569] ws --- ext/opcache/jit/zend_jit_x86.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.h b/ext/opcache/jit/zend_jit_x86.h index 3514d7d56584b..91fbf74430d5f 100644 --- a/ext/opcache/jit/zend_jit_x86.h +++ b/ext/opcache/jit/zend_jit_x86.h @@ -128,7 +128,7 @@ typedef uint32_t zend_regset; ZEND_REGSET_DIFFERENCE(ZEND_REGSET_INTERVAL(ZREG_R0, ZREG_R15), ZEND_REGSET_FIXED) # define ZEND_REGSET_FP \ ZEND_REGSET_DIFFERENCE(ZEND_REGSET_INTERVAL(ZREG_XMM0, ZREG_XMM15), ZEND_REGSET_FIXED) -#define ZEND_REGSET_SCRATCH \ +# define ZEND_REGSET_SCRATCH \ (ZEND_REGSET(ZREG_R0) | ZEND_REGSET(ZREG_R1) | ZEND_REGSET(ZREG_R2) | ZEND_REGSET(ZREG_R3) | ZEND_REGSET(ZREG_R12) | ZEND_REGSET_FP) #else # define ZEND_REGSET_FIXED \ @@ -137,7 +137,7 @@ typedef uint32_t zend_regset; ZEND_REGSET_DIFFERENCE(ZEND_REGSET_INTERVAL(ZREG_R0, ZREG_R7), ZEND_REGSET_FIXED) # define ZEND_REGSET_FP \ ZEND_REGSET_DIFFERENCE(ZEND_REGSET_INTERVAL(ZREG_XMM0, ZREG_XMM7), ZEND_REGSET_FIXED) -#define ZEND_REGSET_SCRATCH \ +# define ZEND_REGSET_SCRATCH \ (ZEND_REGSET(ZREG_RAX) | ZEND_REGSET(ZREG_RCX) | ZEND_REGSET(ZREG_RDX) | ZEND_REGSET_FP) #endif From 40253a875c840a94cc16899cfa8c749603445e75 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 20 Mar 2017 12:44:34 +0300 Subject: [PATCH 382/569] Fixed local register allocation (works for bench.php) --- ext/opcache/jit/zend_jit.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index c767784d8ec2e..117577245aa64 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -1462,6 +1462,23 @@ static zend_lifetime_interval* zend_jit_linear_scan(zend_op_array *op_array, zen while (*p) { q = *p; if (q->end <= position) { + if (q->end == position) { + /* In some cases, we may (and should) reuse operand + registers for result */ + switch (op_array->opcodes[position].opcode) { + case ZEND_ADD: + case ZEND_MUL: + case ZEND_QM_ASSIGN: + if (q->ssa_var != ssa->ops[position].op1_use) { + p = &q->next; + continue; + } + break; + default: + p = &q->next; + continue; + } + } /* move ival from active to handled */ ZEND_REGSET_INCL(available, q->reg); *p = q->next; From cba760d01c27e8c2f868284833ab383ca921fe92 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 20 Mar 2017 13:04:15 +0300 Subject: [PATCH 383/569] Fixed support for inner-function inference --- ext/opcache/jit/zend_jit.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 117577245aa64..cd43ac9fcf595 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -2470,6 +2470,16 @@ ZEND_API int zend_jit_script(zend_script *script) } } + for (i = 0; i < call_graph.op_arrays_count; i++) { + info = ZEND_FUNC_INFO(call_graph.op_arrays[i]); + if (info) { + info->call_map = zend_build_call_map(&CG(arena), info, call_graph.op_arrays[i]); + if (call_graph.op_arrays[i]->fn_flags & ZEND_ACC_HAS_RETURN_TYPE) { + zend_init_func_return_info(call_graph.op_arrays[i], script, &info->return_info); + } + } + } + for (i = 0; i < call_graph.op_arrays_count; i++) { if (zend_jit_trigger == ZEND_JIT_ON_DOC_COMMENT && !zend_needs_manual_jit(call_graph.op_arrays[i])) { From bd0e72267161ffd776d1678ebfba9349b7455560 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 22 Mar 2017 13:53:51 +0300 Subject: [PATCH 384/569] Fixed linear-scan algorithm implementation (it still may work improperly because of unexpected basic block order) --- ext/opcache/jit/zend_jit.c | 396 ++++++++++++++++++------------ ext/opcache/jit/zend_jit.h | 12 +- ext/opcache/jit/zend_jit_x86.dasc | 4 + ext/opcache/jit/zend_jit_x86.h | 7 +- 4 files changed, 252 insertions(+), 167 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index cd43ac9fcf595..5b6091aa5a535 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -964,8 +964,7 @@ static int zend_jit_add_range(zend_lifetime_interval **intervals, int var, uint3 { zend_lifetime_interval *ival = intervals[var]; - if (!ival || ival->start > to + 1 || ival->end < from - 1) { - // TODO: emalloc(); + if (!ival) { ival = zend_arena_alloc(&CG(arena), sizeof(zend_lifetime_interval)); if (!ival) { return FAILURE; @@ -973,16 +972,30 @@ static int zend_jit_add_range(zend_lifetime_interval **intervals, int var, uint3 ival->ssa_var = var; ival->reg = ZREG_NONE; ival->split = 0; - ival->start = from; - ival->end = to; - ival->next = intervals[var]; + ival->range.start = from; + ival->range.end = to; + ival->range.next = NULL; intervals[var] = ival; + } else if (ival->range.start > to + 1) { + zend_life_range *range = zend_arena_alloc(&CG(arena), sizeof(zend_life_range)); + + if (!range) { + return FAILURE; + } + range->start = ival->range.start; + range->end = ival->range.end; + range->next = ival->range.next; + ival->range.start = from; + ival->range.end = to; + ival->range.next = range; } else { - if (from < ival->start) { - ival->start = from; + ZEND_ASSERT(from <= ival->range.start); + ZEND_ASSERT(to <= ival->range.end); + if (from < ival->range.start) { + ival->range.start = from; } - if (to > ival->end) { - ival->end = to; + if (to > ival->range.end) { + ival->range.end = to; } } return SUCCESS; @@ -990,12 +1003,12 @@ static int zend_jit_add_range(zend_lifetime_interval **intervals, int var, uint3 static int zend_jit_begin_range(zend_lifetime_interval **intervals, int var, uint32_t from) { - if (!intervals[var] || intervals[var]->start > from) { + if (!intervals[var] || intervals[var]->range.start > from) { // dead store return zend_jit_add_range(intervals, var, from, from); } - intervals[var]->start = from; + intervals[var]->range.start = from; return SUCCESS; } @@ -1005,25 +1018,26 @@ static void zend_jit_insert_interval(zend_lifetime_interval **list, zend_lifetim while (1) { if (*list == NULL) { *list = ival; - ival->next = NULL; + ival->list_next = NULL; return; - } else if (ival->start < (*list)->start) { - ival->next = *list; + } else if (ival->range.start < (*list)->range.start) { + ival->list_next = *list; *list = ival; return; } - list = &(*list)->next; + list = &(*list)->list_next; } } static int zend_jit_split_interval(zend_lifetime_interval *current, uint32_t pos, zend_lifetime_interval **list, zend_lifetime_interval **free) { - // TODO: emalloc(); zend_lifetime_interval *ival; + zend_life_range *range = ¤t->range; + zend_life_range *prev = NULL; if (*free) { ival = *free; - *free = ival->next; + *free = ival->list_next; } else { ival = zend_arena_alloc(&CG(arena), sizeof(zend_lifetime_interval)); @@ -1035,10 +1049,27 @@ static int zend_jit_split_interval(zend_lifetime_interval *current, uint32_t pos ival->ssa_var = current->ssa_var; ival->reg = ZREG_NONE; ival->split = 1; - ival->start = pos; - ival->end = current->end; - current->end = pos; + do { + if (pos >= range->start && pos <= range->end) { + break; + } + prev = range; + range = range->next; + } while(range); + + ZEND_ASSERT(range != NULL); + + ival->range.start = pos; + ival->range.end = range->end; + ival->range.next = range->next; + + if (pos == range->start) { + ZEND_ASSERT(prev != NULL); + prev->next = NULL; + } else { + range->end = pos - 1; + } zend_jit_insert_interval(list, ival); @@ -1047,46 +1078,50 @@ static int zend_jit_split_interval(zend_lifetime_interval *current, uint32_t pos static zend_lifetime_interval *zend_jit_sort_intervals(zend_lifetime_interval **intervals, int count) { - zend_lifetime_interval *list = NULL; - zend_lifetime_interval *last = NULL; + zend_lifetime_interval *list, *last; int i; - for (i = 0; i < count; i++) { + list = NULL; + i = 0; + while (i < count) { + list = intervals[i]; + i++; + if (list) { + last = list; + last->list_next = NULL; + break; + } + } + + while (i < count) { zend_lifetime_interval *ival = intervals[i]; - while (ival != NULL) { - zend_lifetime_interval *next = ival->next; - - /* Optimized version of zend_jit_insert_interval() */ - if (!last) { - list = last = ival; - ival->next = NULL; - } else if ((ival->start > last->start) || - (ival->start == last->start && - ival->end > last->end)) { - last->next = ival; + i++; + if (ival) { + if ((ival->range.start > last->range.start) || + (ival->range.start == last->range.start && + ival->range.end > last->range.end)) { + last->list_next = ival; last = ival; - ival->next = NULL; + ival->list_next = NULL; } else { zend_lifetime_interval **p = &list; while (1) { if (*p == NULL) { *p = last = ival; - ival->next = NULL; + ival->list_next = NULL; break; - } else if ((ival->start < (*p)->start) || - (ival->start == (*p)->start && - ival->end < (*p)->end)) { - ival->next = *p; + } else if ((ival->range.start < (*p)->range.start) || + (ival->range.start == (*p)->range.start && + ival->range.end < (*p)->range.end)) { + ival->list_next = *p; *p = ival; break; } - p = &(*p)->next; + p = &(*p)->list_next; } } - - ival = next; } } @@ -1145,6 +1180,10 @@ static int zend_jit_compute_liveness(zend_op_array *op_array, zend_ssa *ssa, zen zend_bitset_clear(live_in, set_size * ssa->cfg.blocks_count); memset(loop_end, 0, ssa->cfg.blocks_count * sizeof(uint32_t)); + /* TODO: Provide a linear block order where all dominators of a block + * are before this block, and where all blocks belonging to the same loop + * are contiguous ??? + */ for (i = ssa->cfg.blocks_count - 1; i >= 0; i--) { zend_basic_block *b = ssa->cfg.blocks + i; @@ -1181,7 +1220,7 @@ static int zend_jit_compute_liveness(zend_op_array *op_array, zend_ssa *ssa, zen /* addRange(var, b.from, b.to) for each var in live */ ZEND_BITSET_FOREACH(live, set_size, j) { - if (!ssa->vars[j].no_val) { + if (zend_bitset_in(candidates, j)) { if (zend_jit_add_range(intervals, j, b->start, b->start + b->len - 1) != SUCCESS) { goto failure; } @@ -1262,7 +1301,7 @@ static int zend_jit_compute_liveness(zend_op_array *op_array, zend_ssa *ssa, zen loop_end[i] = b->start + b->len; } ZEND_BITSET_FOREACH(live, set_size, j) { - if (!ssa->vars[j].no_val) { + if (zend_bitset_in(candidates, j)) { if (zend_jit_add_range(intervals, j, b->start, loop_end[i]) != SUCCESS) { goto failure; } @@ -1274,23 +1313,6 @@ static int zend_jit_compute_liveness(zend_op_array *op_array, zend_ssa *ssa, zen zend_bitset_copy(live_in + set_size * i, live, set_size); } -#ifdef DEBUG_REG_ALLOC - fprintf(stderr, "Live Ranges\n"); - for (i = 0; i < ssa->vars_count; i++) { - if (intervals[i]) { - zend_lifetime_interval *ival = intervals[i]; - - fprintf(stderr, "#%d: %u-%u", ival->ssa_var, ival->start, ival->end); - ival = ival->next; - while (ival) { - fprintf(stderr, ", %u-%u", ival->start, ival->end); - ival = ival->next; - } - fprintf(stderr, "\n"); - } - } -#endif - *list = zend_jit_sort_intervals(intervals, ssa->vars_count); free_alloca(intervals, use_heap); return SUCCESS; @@ -1301,16 +1323,59 @@ static int zend_jit_compute_liveness(zend_op_array *op_array, zend_ssa *ssa, zen return FAILURE; } +static uint32_t zend_interval_end(zend_lifetime_interval *ival) +{ + zend_life_range *range = &ival->range; + + while (range->next) { + range = range->next; + } + return range->end; +} + +static zend_bool zend_interval_covers(zend_lifetime_interval *ival, uint32_t position) +{ + zend_life_range *range = &ival->range; + + do { + if (position >= range->start && position <= range->end) { + return 1; + } + range = range->next; + } while (range); + + return 0; +} + +static uint32_t zend_interval_intersection(zend_lifetime_interval *ival1, zend_lifetime_interval *ival2) +{ + zend_life_range *r1 = &ival1->range; + zend_life_range *r2 = &ival2->range; + + do { + if (r1->start <= r2->end) { + if (r2->start <= r1->end) { + return MAX(r1->start, r2->start); + } else { + r2 = r2->next; + } + } else { + r1 = r1->next; + } + } while (r1 && r2); + + return 0xffffffff; +} + /* See "Optimized Interval Splitting in a Linear Scan Register Allocator", - Christian Wimmer VEE'05 (2005), Figure 4. Allocation without spilling. - and "Linear Scan Register Allocation on SSA Form", Christian Wimmer and - Michael Franz, CGO'10 (2010), Figure 6. */ + Christian Wimmer VEE'05 (2005), Figure 4. Allocation without spilling */ static int zend_jit_try_allocate_free_reg(zend_op_array *op_array, zend_ssa *ssa, zend_lifetime_interval *current, zend_regset available, zend_lifetime_interval *active, zend_lifetime_interval *inactive, zend_lifetime_interval **list, zend_lifetime_interval **free) { zend_lifetime_interval *it; uint32_t freeUntilPos[ZREG_NUM]; uint32_t pos; zend_reg i, reg; + zend_life_range *range; if ((ssa->var_info[current->ssa_var].type & MAY_BE_ANY) == MAY_BE_DOUBLE) { available = ZEND_REGSET_INTERSECTION(available, ZEND_REGSET_FP); @@ -1332,46 +1397,53 @@ static int zend_jit_try_allocate_free_reg(zend_op_array *op_array, zend_ssa *ssa it = active; while (it) { freeUntilPos[it->reg] = 0; - it = it->next; + it = it->list_next; } + /* See "Linear Scan Register Allocation on SSA Form", Christian Wimmer and + Michael Franz, CGO'10 (2010), Figure 6. */ if (current->split) { /* for each interval it in inactive intersecting with current do */ /* freeUntilPos[it.reg] = next intersection of it with current */ it = inactive; while (it) { - if (current->start < it->end && current->end > it->start) { - uint32_t next = it->start; + uint32_t next = zend_interval_intersection(current, it); - if (next < freeUntilPos[it->reg]) { - freeUntilPos[it->reg] = next; - } + //ZEND_ASSERT(next != 0xffffffff && !current->split); + if (next < freeUntilPos[it->reg]) { + freeUntilPos[it->reg] = next; } - it = it->next; + it = it->list_next; } } /* Handle Scratch Registers */ /* TODO: Optimize ??? */ - if (current->start + 1 < current->end) { - uint32_t line = current->start + 1; + range = ¤t->range; + do { + uint32_t line = range->start; zend_regset regset; zend_reg reg; do { - regset = zend_jit_get_scratch_regset(op_array, ssa, op_array->opcodes + line); - if (!ZEND_REGSET_IS_EMPTY(regset)) { - for (reg = 0; reg < ZREG_NUM; reg++) { - if (ZEND_REGSET_IN(regset, reg)) { - if (line < freeUntilPos[reg]) { - freeUntilPos[reg] = line; + if (ssa->ops[line].op1_def != current->ssa_var && + ssa->ops[line].op2_def != current->ssa_var && + ssa->ops[line].result_def != current->ssa_var) { + regset = zend_jit_get_scratch_regset(op_array, ssa, op_array->opcodes + line); + if (!ZEND_REGSET_IS_EMPTY(regset)) { + for (reg = 0; reg < ZREG_NUM; reg++) { + if (ZEND_REGSET_IN(regset, reg)) { + if (line < freeUntilPos[reg]) { + freeUntilPos[reg] = line; + } } } } } line++; - } while (line != current->end); - } + } while (line < range->end); + range = range->next; + } while (range); #if 0 /* Coalesing */ @@ -1414,7 +1486,7 @@ static int zend_jit_try_allocate_free_reg(zend_op_array *op_array, zend_ssa *ssa if (reg == ZREG_NONE) { /* no register available without spilling */ return 0; - } else if (current->end < pos) { + } else if (zend_interval_end(current) < pos) { /* register available for the whole interval */ current->reg = reg; return 1; @@ -1447,100 +1519,102 @@ static zend_lifetime_interval* zend_jit_linear_scan(zend_op_array *op_array, zen zend_lifetime_interval *current, **p, *q; uint32_t position; zend_regset available = ZEND_REGSET_UNION(ZEND_REGSET_GP, ZEND_REGSET_FP); - int count; - zend_lifetime_interval *intervals[3]; unhandled = list; /* active = inactive = handled = free = {} */ active = inactive = handled = free = NULL; while (unhandled != NULL) { current = unhandled; - unhandled = unhandled->next; - position = current->start; + unhandled = unhandled->list_next; + position = current->range.start; p = &active; while (*p) { + uint32_t end = zend_interval_end(*p); + q = *p; - if (q->end <= position) { - if (q->end == position) { - /* In some cases, we may (and should) reuse operand - registers for result */ - switch (op_array->opcodes[position].opcode) { - case ZEND_ADD: - case ZEND_MUL: - case ZEND_QM_ASSIGN: - if (q->ssa_var != ssa->ops[position].op1_use) { - p = &q->next; - continue; - } - break; - default: - p = &q->next; - continue; - } - } + if (end < position) { /* move ival from active to handled */ +end_interval: ZEND_REGSET_INCL(available, q->reg); - *p = q->next; - q->next = handled; + *p = q->list_next; + q->list_next = handled; handled = q; - } else if (q->start > position) { + } else if (end == position) { + /* In some cases, we may (and should) reuse operand + registers for result (coalesce) */ + switch (op_array->opcodes[position].opcode) { + case ZEND_QM_ASSIGN: + case ZEND_ADD: + case ZEND_MUL: + if (q->ssa_var == ssa->ops[position].op1_use) { + goto end_interval; + } + break; + default: + break; + } + p = &q->list_next; + } else if (!zend_interval_covers(q, position)) { /* move ival from active to inactive */ ZEND_REGSET_INCL(available, q->reg); - *p = q->next; - q->next = inactive; + *p = q->list_next; + q->list_next = inactive; inactive = q; } else { - p = &q->next; + p = &q->list_next; } } p = &inactive; while (*p) { + uint32_t end = zend_interval_end(*p); + q = *p; - if (q->end < position) { + if (end < position) { /* move ival from inactive to handled */ - *p = q->next; - q->next = handled; + *p = q->list_next; + q->list_next = handled; handled = q; - } else if (q->start <= position) { + } else if (zend_interval_covers(q, position)) { /* move ival from inactive to active */ ZEND_REGSET_EXCL(available, q->reg); - *p = q->next; - q->next = active; + *p = q->list_next; + q->list_next = active; active = q; } else { - p = &q->next; + p = &q->list_next; } } if (zend_jit_try_allocate_free_reg(op_array, ssa, current, available, active, inactive, &unhandled, &free) || zend_jit_allocate_blocked_reg()) { ZEND_REGSET_EXCL(available, current->reg); - current->next = active; + current->list_next = active; active = current; } else { - current->next = free; + current->list_next = free; free = current; } } - count = 0; - - if (handled) { - intervals[count++] = handled; + /* move active to handled */ + while (active) { + current = active; + active = active->list_next; + current->list_next = handled; + handled = current; } - if (inactive) { - intervals[count++] = inactive; - } - if (active) { - intervals[count++] = active; - } - if (count) { - return zend_jit_sort_intervals(intervals, count); - } else { - return NULL; + + /* move inactive to handled */ + while (inactive) { + current = inactive; + inactive = inactive->list_next; + current->list_next = handled; + handled = current; } + + return handled; } static zend_lifetime_interval** zend_jit_allocate_registers(zend_op_array *op_array, zend_ssa *ssa) @@ -1586,11 +1660,19 @@ static zend_lifetime_interval** zend_jit_allocate_registers(zend_op_array *op_ar if (list) { #ifdef DEBUG_REG_ALLOC - fprintf(stderr, "Before Linear Scan\n"); + fprintf(stderr, "Live Ranges\n"); ival = list; - while (ival != NULL) { - fprintf(stderr, "%u-%u: #%d\n", ival->start, ival->end, ival->ssa_var); - ival = ival->next; + while (ival) { + zend_life_range *range; + + fprintf(stderr, "#%d: %u-%u", ival->ssa_var, ival->range.start, ival->range.end); + range = ival->range.next; + while (range) { + fprintf(stderr, ", %u-%u", range->start, range->end); + range = range->next; + } + fprintf(stderr, "\n"); + ival = ival->list_next; } #endif @@ -1598,12 +1680,21 @@ static zend_lifetime_interval** zend_jit_allocate_registers(zend_op_array *op_ar list = zend_jit_linear_scan(op_array, ssa, list); if (list) { + #ifdef DEBUG_REG_ALLOC - fprintf(stderr, "After Linear Scan\n"); + fprintf(stderr, "Allocated Live Ranges\n"); ival = list; - while (ival != NULL) { - fprintf(stderr, "%u-%u: #%d (%s)\n", ival->start, ival->end, ival->ssa_var, zend_reg_name[ival->reg]); - ival = ival->next; + while (ival) { + zend_life_range *range; + + fprintf(stderr, "#%d: %u-%u", ival->ssa_var, ival->range.start, ival->range.end); + range = ival->range.next; + while (range) { + fprintf(stderr, ", %u-%u", range->start, range->end); + range = range->next; + } + fprintf(stderr, " (%s) \n", zend_reg_name[ival->reg]); + ival = ival->list_next; } #endif @@ -1614,30 +1705,13 @@ static zend_lifetime_interval** zend_jit_allocate_registers(zend_op_array *op_ar ival = list; while (ival != NULL) { - zend_lifetime_interval *next = ival->next; + zend_lifetime_interval *next = ival->list_next; - ival->next = intervals[ival->ssa_var]; + ival->list_next = intervals[ival->ssa_var]; intervals[ival->ssa_var] = ival; ival = next; } -#ifdef DEBUG_REG_ALLOC - fprintf(stderr, "Allocated Live Ranges\n"); - for (i = 0; i < ssa->vars_count; i++) { - if (intervals[i]) { - ival = intervals[i]; - - fprintf(stderr, "#%d: %u-%u (%s)", ival->ssa_var, ival->start, ival->end, zend_reg_name[ival->reg]); - ival = ival->next; - while (ival) { - fprintf(stderr, ", %u-%u (%s)", ival->start, ival->end, zend_reg_name[ival->reg]); - ival = ival->next; - } - fprintf(stderr, "\n"); - } - } -#endif - free_alloca(candidates, use_heap); return intervals; } diff --git a/ext/opcache/jit/zend_jit.h b/ext/opcache/jit/zend_jit.h index 2d5651c655126..176a06732aa3e 100644 --- a/ext/opcache/jit/zend_jit.h +++ b/ext/opcache/jit/zend_jit.h @@ -73,14 +73,20 @@ ZEND_API void zend_jit_deactivate(void); ZEND_API void zend_jit_status(zval *ret); typedef struct _zend_lifetime_interval zend_lifetime_interval; +typedef struct _zend_life_range zend_life_range; + +struct _zend_life_range { + uint32_t start; + uint32_t end; + zend_life_range *next; +}; struct _zend_lifetime_interval { int ssa_var; int8_t reg; zend_bool split; - uint32_t start; - uint32_t end; - zend_lifetime_interval *next; + zend_life_range range; + zend_lifetime_interval *list_next; }; #endif /* HAVE_JIT_H */ diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index c8f06369c1af8..ada41119a0549 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -7818,6 +7818,10 @@ static zend_bool zend_jit_may_be_in_reg(zend_op_array *op_array, zend_ssa *ssa, return 0; } + if (ssa->vars[var].no_val) { + return 0; + } + if ((ssa->var_info[var].type & MAY_BE_ANY) == MAY_BE_DOUBLE) { if (ssa->vars[var].definition >= 0) { if (!zend_jit_opline_supports_reg(op_array, ssa, op_array->opcodes + ssa->vars[var].definition, var)) { diff --git a/ext/opcache/jit/zend_jit_x86.h b/ext/opcache/jit/zend_jit_x86.h index 91fbf74430d5f..ab16e762e70b8 100644 --- a/ext/opcache/jit/zend_jit_x86.h +++ b/ext/opcache/jit/zend_jit_x86.h @@ -178,14 +178,15 @@ static zend_always_inline zend_jit_addr zend_jit_decode_op(const zend_op_array * } else { if (ra && ssa_var >= 0 && ra[ssa_var]) { zend_lifetime_interval *ival = ra[ssa_var]; + zend_life_range *range = &ival->range; uint32_t line = opline - op_array->opcodes; do { - if (line >= ival->start && line <= ival->end) { + if (line >= range->start && line <= range->end) { return ZEND_ADDR_REG(ival->reg); } - ival = ival->next; - } while (ival); + range = range->next; + } while (range); } return ZEND_ADDR_MEM_ZVAL(ZREG_FP, op.var); } From 3c42132a70f273026828db006b81a687805c4b3a Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 22 Mar 2017 15:07:39 +0300 Subject: [PATCH 385/569] Allow register allocation for LONG variables --- ext/opcache/jit/zend_jit_x86.dasc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index ada41119a0549..11912fca4fe37 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -7822,7 +7822,8 @@ static zend_bool zend_jit_may_be_in_reg(zend_op_array *op_array, zend_ssa *ssa, return 0; } - if ((ssa->var_info[var].type & MAY_BE_ANY) == MAY_BE_DOUBLE) { + if (((ssa->var_info[var].type & MAY_BE_ANY) == MAY_BE_DOUBLE) || + ((ssa->var_info[var].type & MAY_BE_ANY) == MAY_BE_LONG)) { if (ssa->vars[var].definition >= 0) { if (!zend_jit_opline_supports_reg(op_array, ssa, op_array->opcodes + ssa->vars[var].definition, var)) { return 0; From 3bc1491ff18cba3d5e0e90e4a8942aa127d7d1e1 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 23 Mar 2017 15:59:43 +0300 Subject: [PATCH 386/569] Keep only local register allocation (temporary disable register-allocation for variables defined/used in different basic blocks). Support register allocation in QM_ASSIGN. Intersection of used and scratch registers may cause failures (temorary disabled). --- ext/opcache/jit/zend_jit.c | 4 +- ext/opcache/jit/zend_jit_x86.dasc | 213 ++++++++++++++++++++++++------ 2 files changed, 174 insertions(+), 43 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 5b6091aa5a535..f534b3c595bb6 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -1441,7 +1441,7 @@ static int zend_jit_try_allocate_free_reg(zend_op_array *op_array, zend_ssa *ssa } } line++; - } while (line < range->end); + } while (line <= range->end); range = range->next; } while (range); @@ -1923,7 +1923,7 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa, const zend_op *rt_op } goto done; case ZEND_QM_ASSIGN: - if (!zend_jit_qm_assign(&dasm_state, opline, op_array, ssa)) { + if (!zend_jit_qm_assign(&dasm_state, opline, op_array, ssa, ra)) { goto jit_failure; } goto done; diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 11912fca4fe37..dbea684a811c3 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -531,18 +531,19 @@ static void* dasm_labels[zend_lb_MAX]; |.macro ZVAL_COPY_CONST, dst_addr, dst_info, zv, tmp_reg || if (Z_TYPE_P(zv) > IS_TRUE) { || if (Z_TYPE_P(zv) == IS_DOUBLE) { +|| zend_reg dst_reg = (Z_MODE(dst_addr) == IS_REG) ? Z_REG(dst_addr) : ZREG_XMM0; | .if X64 or SSE || if (Z_DVAL_P(zv) == 0.0 && !is_signed(Z_DVAL_P(zv))) { -| xorps xmm0, xmm0 +| xorps xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0) | .if X64 || } else if (!IS_32BIT(zv)) { | mov64 tmp_reg, ((uintptr_t)zv) -| movsd xmm0, qword [tmp_reg] +| movsd xmm(dst_reg-ZREG_XMM0), qword [tmp_reg] | .endif || } else { -| movsd xmm0, qword [((uint32_t)(uintptr_t)zv)] +| movsd xmm(dst_reg-ZREG_XMM0), qword [((uint32_t)(uintptr_t)zv)] || } -| SSE_SET_ZVAL_DVAL dst_addr, ZREG_XMM0 +| SSE_SET_ZVAL_DVAL dst_addr, dst_reg | .else || if (Z_DVAL_P(zv) == 0.0 && !is_signed(Z_DVAL_P(zv))) { | fldz @@ -556,8 +557,12 @@ static void* dasm_labels[zend_lb_MAX]; || } else { | .if X64 || if (!IS_SIGNED_32BIT(Z_LVAL_P(zv))) { -| mov64 tmp_reg, ((uintptr_t)Z_LVAL_P(zv)) -| SET_ZVAL_LVAL dst_addr, tmp_reg +|| if (Z_MODE(dst_addr) == IS_REG) { +| mov64 Ra(Z_REG(dst_addr)), ((uintptr_t)Z_LVAL_P(zv)) +|| } else { +| mov64 tmp_reg, ((uintptr_t)Z_LVAL_P(zv)) +| SET_ZVAL_LVAL dst_addr, tmp_reg +|| } || } else { | SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv) || } @@ -566,24 +571,28 @@ static void* dasm_labels[zend_lb_MAX]; | .endif || } || } -|| if (((dst_info & MAY_BE_ANY) != (1< IS_TRUE) { || if (Z_TYPE_P(zv) == IS_DOUBLE) { +|| zend_reg dst_reg = (Z_MODE(dst_addr) == IS_REG) ? +|| Z_REG(dst_addr) : ((Z_MODE(dst_addr) == IS_REG) ? Z_MODE(res_addr) : ZREG_XMM0); | .if X64 or SSE || if (Z_DVAL_P(zv) == 0.0 && !is_signed(Z_DVAL_P(zv))) { -| xorps xmm0, xmm0 +| xorps xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0) | .if X64 || } else if (!IS_32BIT(zv)) { | mov64 tmp_reg, ((uintptr_t)zv) -| movsd xmm0, qword [tmp_reg] +| movsd xmm(dst_reg-ZREG_XMM0), qword [tmp_reg] | .endif || } else { -| movsd xmm0, qword [((uint32_t)(uintptr_t)zv)] +| movsd xmm(dst_reg-ZREG_XMM0), qword [((uint32_t)(uintptr_t)zv)] || } | SSE_SET_ZVAL_DVAL dst_addr, ZREG_XMM0 | SSE_SET_ZVAL_DVAL res_addr, ZREG_XMM0 @@ -608,9 +617,17 @@ static void* dasm_labels[zend_lb_MAX]; || } else { | .if X64 || if (!IS_SIGNED_32BIT(Z_LVAL_P(zv))) { -| mov64 tmp_reg, ((uintptr_t)Z_LVAL_P(zv)) -| SET_ZVAL_LVAL dst_addr, tmp_reg -| SET_ZVAL_LVAL res_addr, tmp_reg +|| if (Z_MODE(dst_addr) == IS_REG) { +| mov64 Ra(Z_REG(dst_addr)), ((uintptr_t)Z_LVAL_P(zv)) +| SET_ZVAL_LVAL res_addr, Ra(Z_REG(dst_addr)) +|| } else if (Z_MODE(res_addr) == IS_REG) { +| mov64 Ra(Z_REG(res_addr)), ((uintptr_t)Z_LVAL_P(zv)) +| SET_ZVAL_LVAL dst_addr, Ra(Z_REG(res_addr)) +|| } else { +| mov64 tmp_reg, ((uintptr_t)Z_LVAL_P(zv)) +| SET_ZVAL_LVAL dst_addr, tmp_reg +| SET_ZVAL_LVAL res_addr, tmp_reg +| } || } else { | SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv) | SET_ZVAL_LVAL res_addr, Z_LVAL_P(zv) @@ -621,26 +638,47 @@ static void* dasm_labels[zend_lb_MAX]; | .endif || } || } -|| if (((dst_info & MAY_BE_ANY) != (1<op1_type, opline->op1, opline, ra, ssa->ops[opline - op_array->opcodes].op1_use); - op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2, opline, ra, ssa->ops[opline - op_array->opcodes].op2_use); + op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].op1_use : -1); + op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].op2_use : -1); if (opline->result_type == IS_TMP_VAR && (opline+1)->opcode == ZEND_SEND_VAL && @@ -2944,8 +3020,7 @@ static int zend_jit_simple_assign(dasm_State **Dst, int in_cold) /* Labels: 1,2,3 */ { - ZEND_ASSERT(Z_MODE(var_addr) == IS_MEM_ZVAL); - ZEND_ASSERT(Z_REG(var_addr) != ZREG_R0); + ZEND_ASSERT(Z_MODE(var_addr) == IS_REG || Z_REG(var_addr) != ZREG_R0); if (Z_MODE(val_addr) == IS_CONST_ZVAL) { zval *zv = Z_ZV(val_addr); @@ -5141,13 +5216,60 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, int b, return 1; } -static int zend_jit_qm_assign(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) +static int zend_jit_update_regs(dasm_State **Dst, uint32_t info, zend_jit_addr src_addr, zend_jit_addr dst_addr) +{ + if (Z_MODE(src_addr) == IS_REG) { + if ((info & MAY_BE_ANY) == MAY_BE_LONG) { + if (Z_MODE(dst_addr) != IS_REG || Z_REG(src_addr) != Z_REG(dst_addr)) { + | SET_ZVAL_LVAL dst_addr, Ra(Z_REG(src_addr)) + } + if (Z_MODE(dst_addr) == IS_MEM_ZVAL) { + | SET_ZVAL_TYPE_INFO dst_addr, IS_LONG + } + } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) { + |.if SSE + | SSE_SET_ZVAL_DVAL dst_addr, Z_REG(src_addr) + |.else + || ZEND_ASSERT(0); + |.endif + if (Z_MODE(dst_addr) == IS_MEM_ZVAL) { + | SET_ZVAL_TYPE_INFO dst_addr, IS_DOUBLE + } + } else { + ZEND_ASSERT(0); + } + } else if (Z_MODE(dst_addr) == IS_REG) { + if ((info & MAY_BE_ANY) == MAY_BE_LONG) { + | GET_ZVAL_LVAL Z_REG(dst_addr), src_addr + } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) { + |.if SSE + | SSE_GET_ZVAL_DVAL Z_REG(dst_addr), src_addr + |.else + || ZEND_ASSERT(0); + |.endif + } else { + ZEND_ASSERT(0); + } + } + return 1; +} + +static int zend_jit_qm_assign(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, zend_lifetime_interval **ra) { uint32_t op1_info = OP1_INFO(); zend_jit_addr op1_addr, res_addr; - op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, NULL, -1); - res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1); + op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].op1_use : -1); + res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].result_def : -1); + + if (ra + && ssa->ops[opline - op_array->opcodes].op1_def >= 0 + && !ssa->vars[ssa->ops[opline - op_array->opcodes].op1_def].no_val + && !zend_jit_update_regs(Dst, op1_info, op1_addr, + zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, ra, ssa->ops[opline - op_array->opcodes].op1_def))) { + return 0; + } + return zend_jit_simple_assign(Dst, opline, op_array, ssa, res_addr, -1, opline->op1_type, opline->op1, op1_addr, op1_info, 0, 0); } @@ -7765,9 +7887,9 @@ static zend_bool zend_jit_opline_supports_reg(zend_op_array *op_array, zend_ssa uint32_t op1_info, op2_info; switch (opline->opcode) { -#if 0 case ZEND_QM_ASSIGN: return 1; +#if 0 case ZEND_PRE_INC: case ZEND_PRE_DEC: case ZEND_POST_INC: @@ -7814,6 +7936,7 @@ static zend_bool zend_jit_opline_supports_reg(zend_op_array *op_array, zend_ssa static zend_bool zend_jit_may_be_in_reg(zend_op_array *op_array, zend_ssa *ssa, int var) { // TODO: enable global register allocation ??? + // SSA resolution is not implemented yet. if (ssa->vars[var].definition_phi || ssa->vars[var].phi_use_chain) { return 0; } @@ -7824,7 +7947,10 @@ static zend_bool zend_jit_may_be_in_reg(zend_op_array *op_array, zend_ssa *ssa, if (((ssa->var_info[var].type & MAY_BE_ANY) == MAY_BE_DOUBLE) || ((ssa->var_info[var].type & MAY_BE_ANY) == MAY_BE_LONG)) { + int definition_block = -1; + if (ssa->vars[var].definition >= 0) { + definition_block = ssa->cfg.map[ssa->vars[var].definition]; if (!zend_jit_opline_supports_reg(op_array, ssa, op_array->opcodes + ssa->vars[var].definition, var)) { return 0; } @@ -7833,6 +7959,11 @@ static zend_bool zend_jit_may_be_in_reg(zend_op_array *op_array, zend_ssa *ssa, int use = ssa->vars[var].use_chain; do { + if (ssa->cfg.map[use] != definition_block) { + // TODO: enable global register allocation ??? + // Incorrect basic block oreder breaks register allocation. + return 0; + } if (!zend_jit_opline_supports_reg(op_array, ssa, op_array->opcodes + use, var)) { return 0; } From 6759b2f3fe0e528bbe919e272a117c7c83c4bbf0 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 24 Mar 2017 13:50:00 +0300 Subject: [PATCH 387/569] Register allocation support for comparison opcodes --- ext/opcache/jit/zend_jit.c | 8 +- ext/opcache/jit/zend_jit_x86.dasc | 256 +++++++++++++++++++++--------- 2 files changed, 184 insertions(+), 80 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index f534b3c595bb6..07dfb8145b588 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -1660,7 +1660,7 @@ static zend_lifetime_interval** zend_jit_allocate_registers(zend_op_array *op_ar if (list) { #ifdef DEBUG_REG_ALLOC - fprintf(stderr, "Live Ranges\n"); + fprintf(stderr, "Live Ranges \"%s\"\n", op_array->function_name ? ZSTR_VAL(op_array->function_name) : "[main]"); ival = list; while (ival) { zend_life_range *range; @@ -1682,7 +1682,7 @@ static zend_lifetime_interval** zend_jit_allocate_registers(zend_op_array *op_ar if (list) { #ifdef DEBUG_REG_ALLOC - fprintf(stderr, "Allocated Live Ranges\n"); + fprintf(stderr, "Allocated Live Ranges \"%s\"\n", op_array->function_name ? ZSTR_VAL(op_array->function_name) : "[main]"); ival = list; while (ival) { zend_life_range *range; @@ -1965,13 +1965,13 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa, const zend_op *rt_op case ZEND_IS_SMALLER: case ZEND_IS_SMALLER_OR_EQUAL: case ZEND_CASE: - if (!zend_jit_cmp(&dasm_state, opline, b, &i, op_array, ssa)) { + if (!zend_jit_cmp(&dasm_state, opline, b, &i, op_array, ssa, ra)) { goto jit_failure; } goto done; case ZEND_IS_IDENTICAL: case ZEND_IS_NOT_IDENTICAL: - if (!zend_jit_identical(&dasm_state, opline, b, &i, op_array, ssa)) { + if (!zend_jit_identical(&dasm_state, opline, b, &i, op_array, ssa, ra)) { goto jit_failure; } goto done; diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index dbea684a811c3..4f2fedc4c49ee 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -3808,14 +3808,17 @@ fallback: return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); } -static int zend_jit_cmp_long_long(dasm_State **Dst, const zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa) +static int zend_jit_cmp_long_long(dasm_State **Dst, const zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa, zend_jit_addr op1_addr, zend_jit_addr op2_addr) { unsigned int target_label; - int swap = 0; - zend_jit_addr op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, NULL, -1); - zend_jit_addr op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2, opline, NULL, -1); + zend_bool swap = 0; - if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) != IS_CONST_ZVAL) { + if (Z_MODE(op1_addr) == IS_REG) { + | LONG_OP cmp, Ra(Z_REG(op1_addr)), op2_addr + } else if (Z_MODE(op2_addr) == IS_REG) { + | LONG_OP cmp, Ra(Z_REG(op2_addr)), op1_addr + swap = 1; + } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) != IS_CONST_ZVAL) { | LONG_OP_MEM_CONST cmp, op2_addr, Z_LVAL_P(Z_ZV(op1_addr)) swap = 1; } else if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_MODE(op1_addr) != IS_CONST_ZVAL) { @@ -3832,10 +3835,12 @@ static int zend_jit_cmp_long_long(dasm_State **Dst, const zend_op *opline, int b switch (opline->opcode) { case ZEND_IS_EQUAL: + case ZEND_IS_IDENTICAL: case ZEND_CASE: | sete al break; case ZEND_IS_NOT_EQUAL: + case ZEND_IS_NOT_IDENTICAL: | setne al break; case ZEND_IS_SMALLER: @@ -3866,10 +3871,12 @@ static int zend_jit_cmp_long_long(dasm_State **Dst, const zend_op *opline, int b target_label = ssa->cfg.blocks[b].successors[0]; switch (opline->opcode) { case ZEND_IS_EQUAL: + case ZEND_IS_IDENTICAL: case ZEND_CASE: | jne => target_label break; case ZEND_IS_NOT_EQUAL: + case ZEND_IS_NOT_IDENTICAL: | je => target_label break; case ZEND_IS_SMALLER: @@ -3896,10 +3903,12 @@ static int zend_jit_cmp_long_long(dasm_State **Dst, const zend_op *opline, int b target_label = ssa->cfg.blocks[b].successors[0]; switch (opline->opcode) { case ZEND_IS_EQUAL: + case ZEND_IS_IDENTICAL: case ZEND_CASE: | je => target_label break; case ZEND_IS_NOT_EQUAL: + case ZEND_IS_NOT_IDENTICAL: | jne => target_label break; case ZEND_IS_SMALLER: @@ -3925,10 +3934,12 @@ static int zend_jit_cmp_long_long(dasm_State **Dst, const zend_op *opline, int b target_label = ssa->cfg.blocks[b].successors[0]; switch (opline->opcode) { case ZEND_IS_EQUAL: + case ZEND_IS_IDENTICAL: case ZEND_CASE: | jne => target_label break; case ZEND_IS_NOT_EQUAL: + case ZEND_IS_NOT_IDENTICAL: | je => target_label break; case ZEND_IS_SMALLER: @@ -3955,10 +3966,12 @@ static int zend_jit_cmp_long_long(dasm_State **Dst, const zend_op *opline, int b switch (opline->opcode) { case ZEND_IS_EQUAL: + case ZEND_IS_IDENTICAL: case ZEND_CASE: | sete al break; case ZEND_IS_NOT_EQUAL: + case ZEND_IS_NOT_IDENTICAL: | setne al break; case ZEND_IS_SMALLER: @@ -3986,7 +3999,7 @@ static int zend_jit_cmp_long_long(dasm_State **Dst, const zend_op *opline, int b return 1; } -static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa) +static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa, zend_bool swap) { unsigned int target_label; @@ -3996,21 +4009,31 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, i target_label = ssa->cfg.blocks[b].successors[0]; switch (opline->opcode) { case ZEND_IS_EQUAL: + case ZEND_IS_IDENTICAL: case ZEND_CASE: | jp >1 | jne => target_label |1: break; case ZEND_IS_NOT_EQUAL: + case ZEND_IS_NOT_IDENTICAL: | jp >1 | je => target_label |1: break; case ZEND_IS_SMALLER: - | jae => target_label + if (swap) { + | jnae => target_label + } else { + | jae => target_label + } break; case ZEND_IS_SMALLER_OR_EQUAL: - | ja => target_label + if (swap) { + | jnae => target_label + } else { + | ja => target_label + } break; default: ZEND_ASSERT(0); @@ -4021,21 +4044,31 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, i target_label = ssa->cfg.blocks[b].successors[0]; switch (opline->opcode) { case ZEND_IS_EQUAL: + case ZEND_IS_IDENTICAL: case ZEND_CASE: | jp >1 | je => target_label |1: break; case ZEND_IS_NOT_EQUAL: + case ZEND_IS_NOT_IDENTICAL: | jp >1 | jne => target_label |1: break; case ZEND_IS_SMALLER: - | jnae => target_label + if (swap) { + | jae => target_label + } else { + | jnae => target_label + } break; case ZEND_IS_SMALLER_OR_EQUAL: - | jna => target_label + if (swap) { + | jae => target_label + } else { + | jna => target_label + } break; default: ZEND_ASSERT(0); @@ -4046,21 +4079,31 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, i target_label = ssa->cfg.blocks[b].successors[0]; switch (opline->opcode) { case ZEND_IS_EQUAL: + case ZEND_IS_IDENTICAL: case ZEND_CASE: | jp >1 | jne => target_label |1: break; case ZEND_IS_NOT_EQUAL: + case ZEND_IS_NOT_IDENTICAL: | jp >1 | je => target_label |1: break; case ZEND_IS_SMALLER: - | jae => target_label + if (swap) { + | jnae => target_label + } else { + | jae => target_label + } break; case ZEND_IS_SMALLER_OR_EQUAL: - | ja => target_label + if (swap) { + | jnae => target_label + } else { + | ja => target_label + } break; default: ZEND_ASSERT(0); @@ -4075,6 +4118,7 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, i target_label = ssa->cfg.blocks[b].successors[0]; switch (opline->opcode) { case ZEND_IS_EQUAL: + case ZEND_IS_IDENTICAL: case ZEND_CASE: | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE | jp >1 @@ -4083,6 +4127,7 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, i |1: break; case ZEND_IS_NOT_EQUAL: + case ZEND_IS_NOT_IDENTICAL: | jp >1 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE | je => target_label @@ -4090,18 +4135,34 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, i | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE break; case ZEND_IS_SMALLER: - | setnae al - | movzx eax, al - | lea eax, [eax + 2] - | SET_ZVAL_TYPE_INFO res_addr, eax - | jae => target_label + if (swap) { + | setae al + | movzx eax, al + | lea eax, [eax + 2] + | SET_ZVAL_TYPE_INFO res_addr, eax + | jnae => target_label + } else { + | setnae al + | movzx eax, al + | lea eax, [eax + 2] + | SET_ZVAL_TYPE_INFO res_addr, eax + | jae => target_label + } break; case ZEND_IS_SMALLER_OR_EQUAL: - | setna al - | movzx eax, al - | lea eax, [eax + 2] - | SET_ZVAL_TYPE_INFO res_addr, eax - | ja => target_label + if (swap) { + | setae al + | movzx eax, al + | lea eax, [eax + 2] + | SET_ZVAL_TYPE_INFO res_addr, eax + | jnae => target_label + } else { + | setna al + | movzx eax, al + | lea eax, [eax + 2] + | SET_ZVAL_TYPE_INFO res_addr, eax + | ja => target_label + } break; default: ZEND_ASSERT(0); @@ -4114,6 +4175,7 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, i target_label = ssa->cfg.blocks[b].successors[0]; switch (opline->opcode) { case ZEND_IS_EQUAL: + case ZEND_IS_IDENTICAL: case ZEND_CASE: | jp >1 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE @@ -4122,6 +4184,7 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, i | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE break; case ZEND_IS_NOT_EQUAL: + case ZEND_IS_NOT_IDENTICAL: | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE | jp >1 | jne => target_label @@ -4129,18 +4192,34 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, i |1: break; case ZEND_IS_SMALLER: - | setnae al - | movzx eax, al - | lea eax, [eax + 2] - | SET_ZVAL_TYPE_INFO res_addr, eax - | jnae => target_label + if (swap) { + | setae al + | movzx eax, al + | lea eax, [eax + 2] + | SET_ZVAL_TYPE_INFO res_addr, eax + | jae => target_label + } else { + | setnae al + | movzx eax, al + | lea eax, [eax + 2] + | SET_ZVAL_TYPE_INFO res_addr, eax + | jnae => target_label + } break; case ZEND_IS_SMALLER_OR_EQUAL: - | setna al - | movzx eax, al - | lea eax, [eax + 2] - | SET_ZVAL_TYPE_INFO res_addr, eax - | jna => target_label + if (swap) { + | setae al + | movzx eax, al + | lea eax, [eax + 2] + | SET_ZVAL_TYPE_INFO res_addr, eax + | jae => target_label + } else { + | setna al + | movzx eax, al + | lea eax, [eax + 2] + | SET_ZVAL_TYPE_INFO res_addr, eax + | jna => target_label + } break; default: ZEND_ASSERT(0); @@ -4150,6 +4229,7 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, i switch (opline->opcode) { case ZEND_IS_EQUAL: + case ZEND_IS_IDENTICAL: case ZEND_CASE: | jp >1 | mov eax, IS_TRUE @@ -4159,6 +4239,7 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, i |2: break; case ZEND_IS_NOT_EQUAL: + case ZEND_IS_NOT_IDENTICAL: | jp >1 | mov eax, IS_FALSE | je >2 @@ -4167,14 +4248,26 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, i |2: break; case ZEND_IS_SMALLER: - | setnae al - | movzx eax, al - | add eax, 2 + if (swap) { + | setae al + | movzx eax, al + | add eax, 2 + } else { + | setnae al + | movzx eax, al + | add eax, 2 + } break; case ZEND_IS_SMALLER_OR_EQUAL: - | setna al - | movzx eax, al - | add eax, 2 + if (swap) { + | setae al + | movzx eax, al + | add eax, 2 + } else { + | setna al + | movzx eax, al + | add eax, 2 + } break; default: ZEND_ASSERT(0); @@ -4185,11 +4278,8 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, i return 1; } -static int zend_jit_cmp_long_double(dasm_State **Dst, const zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa) +static int zend_jit_cmp_long_double(dasm_State **Dst, const zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa, zend_jit_addr op1_addr, zend_jit_addr op2_addr) { - zend_jit_addr op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, NULL, -1); - zend_jit_addr op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2, opline, NULL, -1); - |.if X64 or SSE zend_reg tmp_reg = ZREG_XMM0; @@ -4202,21 +4292,19 @@ static int zend_jit_cmp_long_double(dasm_State **Dst, const zend_op *opline, int | fstp st0 |.endif - return zend_jit_cmp_double_common(Dst, opline, b, op_array, ssa); + return zend_jit_cmp_double_common(Dst, opline, b, op_array, ssa, 0); } -static int zend_jit_cmp_double_long(dasm_State **Dst, const zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa) +static int zend_jit_cmp_double_long(dasm_State **Dst, const zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa, zend_jit_addr op1_addr, zend_jit_addr op2_addr) { - zend_jit_addr op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, NULL, -1); - zend_jit_addr op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2, opline, NULL, -1); + zend_bool swap = 0; |.if X64 or SSE - zend_reg tmp_reg1 = ZREG_XMM0; - zend_reg tmp_reg2 = ZREG_XMM1; + zend_reg tmp_reg = ZREG_XMM0; - | SSE_GET_ZVAL_DVAL tmp_reg1, op1_addr - | SSE_GET_ZVAL_LVAL tmp_reg2, op2_addr - | ucomisd xmm(tmp_reg1-ZREG_XMM0), xmm(tmp_reg2-ZREG_XMM0) + | SSE_GET_ZVAL_LVAL tmp_reg, op2_addr + | SSE_OP ucomisd, tmp_reg, op1_addr + swap = 1; |.else | FPU_GET_ZVAL_LVAL op2_addr | FPU_GET_ZVAL_DVAL op1_addr @@ -4224,19 +4312,25 @@ static int zend_jit_cmp_double_long(dasm_State **Dst, const zend_op *opline, int | fstp st0 |.endif - return zend_jit_cmp_double_common(Dst, opline, b, op_array, ssa); + return zend_jit_cmp_double_common(Dst, opline, b, op_array, ssa, swap); } -static int zend_jit_cmp_double_double(dasm_State **Dst, const zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa) +static int zend_jit_cmp_double_double(dasm_State **Dst, const zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa, zend_jit_addr op1_addr, zend_jit_addr op2_addr) { - zend_jit_addr op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, NULL, -1); - zend_jit_addr op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2, opline, NULL, -1); + zend_bool swap = 0; |.if X64 or SSE - zend_reg tmp_reg = ZREG_XMM0; + if (Z_MODE(op1_addr) == IS_REG) { + | SSE_OP ucomisd, Z_REG(op1_addr), op2_addr + } else if (Z_MODE(op2_addr) == IS_REG) { + | SSE_OP ucomisd, Z_REG(op2_addr), op1_addr + swap = 1; + } else { + zend_reg tmp_reg = ZREG_XMM0; - | SSE_GET_ZVAL_DVAL tmp_reg, op1_addr - | SSE_OP ucomisd, tmp_reg, op2_addr + | SSE_GET_ZVAL_DVAL tmp_reg, op1_addr + | SSE_OP ucomisd, tmp_reg, op2_addr + } |.else | FPU_GET_ZVAL_DVAL op2_addr | FPU_GET_ZVAL_DVAL op1_addr @@ -4244,7 +4338,7 @@ static int zend_jit_cmp_double_double(dasm_State **Dst, const zend_op *opline, i | fstp st0 |.endif - return zend_jit_cmp_double_common(Dst, opline, b, op_array, ssa); + return zend_jit_cmp_double_common(Dst, opline, b, op_array, ssa, swap); } static int zend_jit_cmp_slow(dasm_State **Dst, const zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa) @@ -4371,14 +4465,14 @@ static int zend_jit_cmp_slow(dasm_State **Dst, const zend_op *opline, int b, zen return 1; } -static int zend_jit_cmp(dasm_State **Dst, const zend_op *opline, int b, int *opnum, zend_op_array *op_array, zend_ssa *ssa) +static int zend_jit_cmp(dasm_State **Dst, const zend_op *opline, int b, int *opnum, zend_op_array *op_array, zend_ssa *ssa, zend_lifetime_interval **ra) { uint32_t op1_info, op2_info; zend_bool same_ops = (opline->op1_type == opline->op2_type) && (opline->op1.var == opline->op2.var); zend_bool has_slow; - zend_jit_addr op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, NULL, -1); - zend_jit_addr op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2, opline, NULL, -1); - zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1); + zend_jit_addr op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].op1_use : -1); + zend_jit_addr op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].op2_use : -1); + zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].result_def : -1); op1_info = OP1_INFO(); op2_info = OP2_INFO(); @@ -4405,7 +4499,7 @@ static int zend_jit_cmp(dasm_State **Dst, const zend_op *opline, int b, int *opn if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9 } - if (!zend_jit_cmp_long_double(Dst, opline, b, op_array, ssa)) { + if (!zend_jit_cmp_long_double(Dst, opline, b, op_array, ssa, op1_addr, op2_addr)) { return 0; } | jmp >6 @@ -4414,7 +4508,7 @@ static int zend_jit_cmp(dasm_State **Dst, const zend_op *opline, int b, int *opn | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9 } } - if (!zend_jit_cmp_long_long(Dst, opline, b, op_array, ssa)) { + if (!zend_jit_cmp_long_long(Dst, opline, b, op_array, ssa, op1_addr, op2_addr)) { return 0; } if (op1_info & MAY_BE_DOUBLE) { @@ -4431,7 +4525,7 @@ static int zend_jit_cmp(dasm_State **Dst, const zend_op *opline, int b, int *opn | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9 } } - if (!zend_jit_cmp_double_double(Dst, opline, b, op_array, ssa)) { + if (!zend_jit_cmp_double_double(Dst, opline, b, op_array, ssa, op1_addr, op2_addr)) { return 0; } | jmp >6 @@ -4441,7 +4535,7 @@ static int zend_jit_cmp(dasm_State **Dst, const zend_op *opline, int b, int *opn if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9 } - if (!zend_jit_cmp_double_long(Dst, opline, b, op_array, ssa)) { + if (!zend_jit_cmp_double_long(Dst, opline, b, op_array, ssa, op1_addr, op2_addr)) { return 0; } | jmp >6 @@ -4462,7 +4556,7 @@ static int zend_jit_cmp(dasm_State **Dst, const zend_op *opline, int b, int *opn | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9 } } - if (!zend_jit_cmp_double_double(Dst, opline, b, op_array, ssa)) { + if (!zend_jit_cmp_double_double(Dst, opline, b, op_array, ssa, op1_addr, op2_addr)) { return 0; } } @@ -4474,7 +4568,7 @@ static int zend_jit_cmp(dasm_State **Dst, const zend_op *opline, int b, int *opn if (op2_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) { | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9 } - if (!zend_jit_cmp_double_long(Dst, opline, b, op_array, ssa)) { + if (!zend_jit_cmp_double_long(Dst, opline, b, op_array, ssa, op1_addr, op2_addr)) { return 0; } if (op2_info & MAY_BE_DOUBLE) { @@ -4496,7 +4590,7 @@ static int zend_jit_cmp(dasm_State **Dst, const zend_op *opline, int b, int *opn | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >9 } } - if (!zend_jit_cmp_double_double(Dst, opline, b, op_array, ssa)) { + if (!zend_jit_cmp_double_double(Dst, opline, b, op_array, ssa, op1_addr, op2_addr)) { return 0; } } @@ -4508,7 +4602,7 @@ static int zend_jit_cmp(dasm_State **Dst, const zend_op *opline, int b, int *opn if (op1_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) { | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >9 } - if (!zend_jit_cmp_long_double(Dst, opline, b, op_array, ssa)) { + if (!zend_jit_cmp_long_double(Dst, opline, b, op_array, ssa, op1_addr, op2_addr)) { return 0; } if (op1_info & MAY_BE_DOUBLE) { @@ -4592,15 +4686,15 @@ static int zend_jit_cmp(dasm_State **Dst, const zend_op *opline, int b, int *opn return 1; } -static int zend_jit_identical(dasm_State **Dst, const zend_op *opline, int b, int *opnum, zend_op_array *op_array, zend_ssa *ssa) +static int zend_jit_identical(dasm_State **Dst, const zend_op *opline, int b, int *opnum, zend_op_array *op_array, zend_ssa *ssa, zend_lifetime_interval **ra) { zend_bool smart_branch = 0; uint32_t identical_label = (uint32_t)-1; uint32_t not_identical_label = (uint32_t)-1; uint32_t op1_info, op2_info; - zend_jit_addr op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, NULL, -1); - zend_jit_addr op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2, opline, NULL, -1); - zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1); + zend_jit_addr op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].op1_use : -1); + zend_jit_addr op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].op2_use : -1); + zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].result_def : -1); op1_info = OP1_INFO(); op2_info = OP2_INFO(); @@ -4858,6 +4952,16 @@ static int zend_jit_identical(dasm_State **Dst, const zend_op *opline, int b, in if (smart_branch && not_identical_label != (uint32_t)-1) { | jmp =>not_identical_label } + } else if ((op1_info & MAY_BE_ANY) == MAY_BE_LONG && + (op2_info & MAY_BE_ANY) == MAY_BE_LONG) { + if (!zend_jit_cmp_long_long(Dst, opline, b, op_array, ssa, op1_addr, op2_addr)) { + return 0; + } + } else if ((op1_info & MAY_BE_ANY) == MAY_BE_DOUBLE && + (op2_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { + if (!zend_jit_cmp_double_double(Dst, opline, b, op_array, ssa, op1_addr, op2_addr)) { + return 0; + } } else { if (opline->op1_type == IS_CONST) { | LOAD_ZVAL_ADDR FCARG1a, op1_addr @@ -7906,7 +8010,6 @@ static zend_bool zend_jit_opline_supports_reg(zend_op_array *op_array, zend_ssa return !(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) && !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))); -#if 0 case ZEND_IS_SMALLER: case ZEND_IS_SMALLER_OR_EQUAL: case ZEND_IS_EQUAL: @@ -7918,6 +8021,7 @@ static zend_bool zend_jit_opline_supports_reg(zend_op_array *op_array, zend_ssa return !(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) && !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))); +#if 0 case ZEND_BOOL: case ZEND_BOOL_NOT: case ZEND_JMPZ: From 345c482d79e8f4916057474e747638e4c7e0e1c6 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 24 Mar 2017 15:50:23 +0300 Subject: [PATCH 388/569] Enable register allocation for BOOL, BOOL_NOT and conditional branches --- ext/opcache/jit/zend_jit.c | 4 +- ext/opcache/jit/zend_jit_x86.dasc | 93 ++++++++++++++++++++++++++----- 2 files changed, 80 insertions(+), 17 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 07dfb8145b588..1a9a26dd51a8b 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -1992,7 +1992,7 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa, const zend_op *rt_op goto done; case ZEND_BOOL: case ZEND_BOOL_NOT: - if (!zend_jit_bool_jmpznz(&dasm_state, opline, b, op_array, ssa)) { + if (!zend_jit_bool_jmpznz(&dasm_state, opline, b, op_array, ssa, ra)) { goto jit_failure; } goto done; @@ -2023,7 +2023,7 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa, const zend_op *rt_op goto jit_failure; } } else { - if (!zend_jit_bool_jmpznz(&dasm_state, opline, b, op_array, ssa)) { + if (!zend_jit_bool_jmpznz(&dasm_state, opline, b, op_array, ssa, ra)) { goto jit_failure; } } diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 4f2fedc4c49ee..82ccd14e51b9c 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -2110,7 +2110,7 @@ static int zend_jit_math_long_long(dasm_State **Dst, | jo >1 |.cold_code |1: - |.if X64 or SSE + |.if SSE zend_reg tmp_reg1 = ZREG_XMM0; zend_reg tmp_reg2 = ZREG_XMM1; @@ -2148,7 +2148,7 @@ static int zend_jit_math_long_double(dasm_State **Dst, zend_jit_addr op2_addr, zend_jit_addr res_addr) { - |.if X64 or SSE + |.if SSE zend_reg result_reg = (Z_MODE(res_addr) == IS_REG) ? Z_REG(res_addr) : ZREG_XMM0; @@ -2174,7 +2174,7 @@ static int zend_jit_math_double_long(dasm_State **Dst, zend_jit_addr op2_addr, zend_jit_addr res_addr) { - |.if X64 or SSE + |.if SSE zend_reg result_reg = (Z_MODE(res_addr) == IS_REG) ? Z_REG(res_addr) : ZREG_XMM0; @@ -2217,7 +2217,7 @@ static int zend_jit_math_double_double(dasm_State **Dst, { zend_bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); - |.if X64 or SSE + |.if SSE zend_reg result_reg = (Z_MODE(res_addr) == IS_REG) ? Z_REG(res_addr) : ZREG_XMM0; @@ -4280,7 +4280,7 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, i static int zend_jit_cmp_long_double(dasm_State **Dst, const zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa, zend_jit_addr op1_addr, zend_jit_addr op2_addr) { - |.if X64 or SSE + |.if SSE zend_reg tmp_reg = ZREG_XMM0; | SSE_GET_ZVAL_LVAL tmp_reg, op1_addr @@ -4299,7 +4299,7 @@ static int zend_jit_cmp_double_long(dasm_State **Dst, const zend_op *opline, int { zend_bool swap = 0; - |.if X64 or SSE + |.if SSE zend_reg tmp_reg = ZREG_XMM0; | SSE_GET_ZVAL_LVAL tmp_reg, op2_addr @@ -4319,7 +4319,7 @@ static int zend_jit_cmp_double_double(dasm_State **Dst, const zend_op *opline, i { zend_bool swap = 0; - |.if X64 or SSE + |.if SSE if (Z_MODE(op1_addr) == IS_REG) { | SSE_OP ucomisd, Z_REG(op1_addr), op2_addr } else if (Z_MODE(op2_addr) == IS_REG) { @@ -5008,7 +5008,7 @@ static int zend_jit_identical(dasm_State **Dst, const zend_op *opline, int b, in return 1; } -static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa) +static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa, zend_lifetime_interval **ra) { uint32_t op1_info = OP1_INFO(); uint32_t true_label = -1; @@ -5016,7 +5016,7 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, int b, zend_bool set_bool = 0; zend_bool set_bool_not = 0; zend_bool jmp_done = 0; - zend_jit_addr op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, NULL, -1); + zend_jit_addr op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].op1_use : -1); zend_jit_addr res_addr; if (opline->opcode == ZEND_JMPZ) { @@ -5042,7 +5042,7 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, int b, } if (set_bool) { - res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1); + res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].result_def : -1); } if (Z_MODE(op1_addr) == IS_CONST_ZVAL) { @@ -5207,7 +5207,11 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, int b, if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG))) { | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >2 } - | LONG_OP_MEM_CONST, cmp, op1_addr, Z_L(0) + if (Z_MODE(op1_addr) == IS_REG) { + | test Ra(Z_REG(op1_addr)), Ra(Z_REG(op1_addr)) + } else { + | LONG_OP_MEM_CONST, cmp, op1_addr, Z_L(0) + } if (set_bool) { | setne al | movzx eax, al @@ -5231,7 +5235,65 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, int b, } } - if (op1_info & (MAY_BE_ANY - (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG))) { + if ((op1_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { + |.if SSE + | xorps xmm0, xmm0 + | SSE_OP ucomisd, ZREG_XMM0, op1_addr + |.else + | FPU_GET_ZVAL_DVAL op1_addr + | fldz + | fucomip st1 + | fstp st0 + |.endif + + if (set_bool) { + if (false_label != (uint32_t)-1) { // JMPZ_EX + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE + | jp >1 + | je => false_label + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE + |1: + } else if (true_label != (uint32_t)-1) { // JMPNZ_EX + | jp >1 + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE + | jne => true_label + |1: + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE + } else if (set_bool_not) { // BOOL_NOT + | jp >1 + | mov eax, IS_TRUE + | je >2 + |1: + | mov eax, IS_FALSE + |2: + | SET_ZVAL_TYPE_INFO res_addr, eax + } else { // BOOL + | jp >1 + | mov eax, IS_TRUE + | jne >2 + |1: + | mov eax, IS_FALSE + |2: + | SET_ZVAL_TYPE_INFO res_addr, eax + } + } else { + ZEND_ASSERT(true_label != (uint32_t)-1 || false_label != (uint32_t)-1); + if (false_label != (uint32_t)-1) { + | jp =>false_label + } else { + | jp >1 + } + if (true_label != (uint32_t)-1) { + | jne =>true_label + if (false_label != (uint32_t)-1) { + | jmp =>false_label + } + } else { + | je =>false_label + } + |1: + } + } else if (op1_info & (MAY_BE_ANY - (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG))) { if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { |.cold_code |2: @@ -8000,7 +8062,7 @@ static zend_bool zend_jit_opline_supports_reg(zend_op_array *op_array, zend_ssa case ZEND_POST_DEC: op1_info = OP1_INFO(); return - !(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))); + !(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG))); #endif case ZEND_ADD: case ZEND_SUB: @@ -8021,7 +8083,6 @@ static zend_bool zend_jit_opline_supports_reg(zend_op_array *op_array, zend_ssa return !(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) && !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))); -#if 0 case ZEND_BOOL: case ZEND_BOOL_NOT: case ZEND_JMPZ: @@ -8032,7 +8093,6 @@ static zend_bool zend_jit_opline_supports_reg(zend_op_array *op_array, zend_ssa op1_info = OP1_INFO(); return !(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE))); -#endif } return 0; } @@ -8162,6 +8222,9 @@ static zend_regset zend_jit_get_scratch_regset(zend_op_array *op_array, zend_ssa op1_info = OP1_INFO(); if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE)))) { regset = ZEND_REGSET_EMPTY; + if (op1_info & MAY_BE_DOUBLE) { + ZEND_REGSET_INCL(regset, ZREG_XMM0); + } if (opline->opcode == ZEND_BOOL || opline->opcode == ZEND_BOOL_NOT || opline->opcode == ZEND_JMPZ_EX || From dd253d121b0d5c09ee837ebc1910a084568e7d64 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 24 Mar 2017 16:29:00 +0300 Subject: [PATCH 389/569] Enable register allocation for INC/DEC opcodes --- ext/opcache/jit/zend_jit.c | 2 +- ext/opcache/jit/zend_jit_x86.dasc | 70 +++++++++++++++++++------------ 2 files changed, 44 insertions(+), 28 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 1a9a26dd51a8b..0eea57f3dbe34 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -1879,7 +1879,7 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa, const zend_op *rt_op case ZEND_PRE_DEC: case ZEND_POST_INC: case ZEND_POST_DEC: - if (!zend_jit_inc_dec(&dasm_state, opline, op_array, ssa)) { + if (!zend_jit_inc_dec(&dasm_state, opline, op_array, ssa, ra)) { goto jit_failure; } goto done; diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 82ccd14e51b9c..0ad5aedb1ff10 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -395,18 +395,32 @@ static void* dasm_labels[zend_lb_MAX]; || } |.endmacro -|.macro LONG_OP_MEM_CONST, long_ins, op1_addr, lval -|| ZEND_ASSERT(Z_MODE(op1_addr) == IS_MEM_ZVAL); -| .if X64 -|| if (!IS_SIGNED_32BIT(lval)) { -| mov64 r0, lval -| long_ins aword [Ra(Z_REG(op1_addr))+Z_OFFSET(op1_addr)], r0 -|| } else { -| long_ins aword [Ra(Z_REG(op1_addr))+Z_OFFSET(op1_addr)], lval -|| } -| .else -| long_ins aword [Ra(Z_REG(op1_addr))+Z_OFFSET(op1_addr)], lval -| .endif +|.macro LONG_OP_WITH_CONST, long_ins, op1_addr, lval +|| if (Z_MODE(op1_addr) == IS_MEM_ZVAL) { +| .if X64 +|| if (!IS_SIGNED_32BIT(lval)) { +| mov64 r0, lval +| long_ins aword [Ra(Z_REG(op1_addr))+Z_OFFSET(op1_addr)], r0 +|| } else { +| long_ins aword [Ra(Z_REG(op1_addr))+Z_OFFSET(op1_addr)], lval +|| } +| .else +| long_ins aword [Ra(Z_REG(op1_addr))+Z_OFFSET(op1_addr)], lval +| .endif +|| } else if (Z_MODE(op1_addr) == IS_REG) { +| .if X64 +|| if (!IS_SIGNED_32BIT(lval)) { +| mov64 r0, lval +| long_ins Ra(Z_REG(op1_addr)), r0 +|| } else { +| long_ins Ra(Z_REG(op1_addr)), lval +|| } +| .else +| long_ins Ra(Z_REG(op1_addr)), lval +| .endif +|| } else { +|| ZEND_ASSERT(0); +|| } |.endmacro |.macro GET_ZVAL_LVAL, reg, addr @@ -1935,19 +1949,20 @@ static int zend_jit_new(dasm_State **Dst, const zend_op *opline, int *opnum, zen return 1; } -static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) +static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, zend_lifetime_interval **ra) { uint32_t op1_info, op1_def_info; - zend_jit_addr op1_addr, res_addr; + zend_jit_addr op1_addr, op1_def_addr, res_addr; op1_info = OP1_INFO(); if (opline->op1_type != IS_CV || !(op1_info & MAY_BE_LONG)) { goto fallback; } - op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, NULL, -1); + op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].op1_use : -1); + op1_def_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].op1_def : -1); if (opline->result_type != IS_UNUSED) { - res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1); + res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].result_def : -1); } if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)-MAY_BE_LONG)) { @@ -1957,10 +1972,13 @@ static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, zend_op_arr opline->result_type != IS_UNUSED) { | ZVAL_COPY_VALUE res_addr, op1_addr, MAY_BE_LONG, ZREG_R0, ZREG_R1 } + if (op1_addr != op1_def_addr) { + | ZVAL_COPY_VALUE op1_def_addr, op1_addr, MAY_BE_LONG, ZREG_R0, ZREG_R1 + } if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { - | LONG_OP_MEM_CONST add, op1_addr, Z_L(1) + | LONG_OP_WITH_CONST add, op1_def_addr, Z_L(1) } else { - | LONG_OP_MEM_CONST sub, op1_addr, Z_L(1) + | LONG_OP_WITH_CONST sub, op1_def_addr, Z_L(1) } op1_def_info = OP1_DEF_INFO(); @@ -1968,7 +1986,7 @@ static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, zend_op_arr | jo >1 if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && opline->result_type != IS_UNUSED) { - | ZVAL_COPY_VALUE res_addr, op1_addr, MAY_BE_LONG, ZREG_R0, ZREG_R1 + | ZVAL_COPY_VALUE res_addr, op1_def_addr, MAY_BE_LONG, ZREG_R0, ZREG_R1 } |.cold_code |1: @@ -1992,14 +2010,14 @@ static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, zend_op_arr | SET_ZVAL_TYPE_INFO op1_addr, IS_DOUBLE if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && opline->result_type != IS_UNUSED) { - | ZVAL_COPY_VALUE res_addr, op1_addr, MAY_BE_DOUBLE, ZREG_R0, ZREG_R1 + | ZVAL_COPY_VALUE res_addr, op1_def_addr, MAY_BE_DOUBLE, ZREG_R0, ZREG_R1 } | jmp >3 |.code } else { if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && opline->result_type != IS_UNUSED) { - | ZVAL_COPY_VALUE res_addr, op1_addr, MAY_BE_LONG, ZREG_R0, ZREG_R1 + | ZVAL_COPY_VALUE res_addr, op1_def_addr, MAY_BE_LONG, ZREG_R0, ZREG_R1 } } if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { @@ -3819,10 +3837,10 @@ static int zend_jit_cmp_long_long(dasm_State **Dst, const zend_op *opline, int b | LONG_OP cmp, Ra(Z_REG(op2_addr)), op1_addr swap = 1; } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) != IS_CONST_ZVAL) { - | LONG_OP_MEM_CONST cmp, op2_addr, Z_LVAL_P(Z_ZV(op1_addr)) + | LONG_OP_WITH_CONST cmp, op2_addr, Z_LVAL_P(Z_ZV(op1_addr)) swap = 1; } else if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_MODE(op1_addr) != IS_CONST_ZVAL) { - | LONG_OP_MEM_CONST cmp, op1_addr, Z_LVAL_P(Z_ZV(op2_addr)) + | LONG_OP_WITH_CONST cmp, op1_addr, Z_LVAL_P(Z_ZV(op2_addr)) } else { | GET_ZVAL_LVAL ZREG_R0, op1_addr | LONG_OP cmp, r0, op2_addr @@ -4346,7 +4364,7 @@ static int zend_jit_cmp_slow(dasm_State **Dst, const zend_op *opline, int b, zen unsigned int target_label; zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1); - | LONG_OP_MEM_CONST cmp, res_addr, Z_L(0) + | LONG_OP_WITH_CONST cmp, res_addr, Z_L(0) if (((opline+1)->opcode == ZEND_JMPZ_EX || (opline+1)->opcode == ZEND_JMPNZ_EX) && (opline+1)->op1_type == IS_TMP_VAR && @@ -5210,7 +5228,7 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, int b, if (Z_MODE(op1_addr) == IS_REG) { | test Ra(Z_REG(op1_addr)), Ra(Z_REG(op1_addr)) } else { - | LONG_OP_MEM_CONST, cmp, op1_addr, Z_L(0) + | LONG_OP_WITH_CONST, cmp, op1_addr, Z_L(0) } if (set_bool) { | setne al @@ -8055,7 +8073,6 @@ static zend_bool zend_jit_opline_supports_reg(zend_op_array *op_array, zend_ssa switch (opline->opcode) { case ZEND_QM_ASSIGN: return 1; -#if 0 case ZEND_PRE_INC: case ZEND_PRE_DEC: case ZEND_POST_INC: @@ -8063,7 +8080,6 @@ static zend_bool zend_jit_opline_supports_reg(zend_op_array *op_array, zend_ssa op1_info = OP1_INFO(); return !(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG))); -#endif case ZEND_ADD: case ZEND_SUB: case ZEND_MUL: From 262bc0aca65c2d193bb038997ba9dc376b4d12ff Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 27 Mar 2017 22:03:19 +0300 Subject: [PATCH 390/569] Fixed 64-bit build --- ext/opcache/jit/zend_jit_x86.dasc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 0ad5aedb1ff10..39a2e1329100c 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -641,7 +641,7 @@ static void* dasm_labels[zend_lb_MAX]; | mov64 tmp_reg, ((uintptr_t)Z_LVAL_P(zv)) | SET_ZVAL_LVAL dst_addr, tmp_reg | SET_ZVAL_LVAL res_addr, tmp_reg -| } +|| } || } else { | SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv) | SET_ZVAL_LVAL res_addr, Z_LVAL_P(zv) From f842945dca80d46aefd433de34fbc8648a8355b7 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 27 Mar 2017 22:29:18 +0300 Subject: [PATCH 391/569] Temporary disable usage of preserved registers. Their values have to be stored in prologie and restored in epilogue. --- ext/opcache/jit/zend_jit.c | 5 +++++ ext/opcache/jit/zend_jit_x86.h | 6 +++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 0eea57f3dbe34..bbeaa40791311 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -1383,6 +1383,11 @@ static int zend_jit_try_allocate_free_reg(zend_op_array *op_array, zend_ssa *ssa available = ZEND_REGSET_INTERSECTION(available, ZEND_REGSET_GP); } + /* TODO: Allow usage of preserved registers ??? + * Their values have to be stored in prologuee and restored in epilogue + */ + available = ZEND_REGSET_DIFFERENCE(available, ZEND_REGSET_PRESERVED); + if (ZEND_REGSET_IS_EMPTY(available)) { return 0; } diff --git a/ext/opcache/jit/zend_jit_x86.h b/ext/opcache/jit/zend_jit_x86.h index ab16e762e70b8..d83c3d8eda3ad 100644 --- a/ext/opcache/jit/zend_jit_x86.h +++ b/ext/opcache/jit/zend_jit_x86.h @@ -129,7 +129,9 @@ typedef uint32_t zend_regset; # define ZEND_REGSET_FP \ ZEND_REGSET_DIFFERENCE(ZEND_REGSET_INTERVAL(ZREG_XMM0, ZREG_XMM15), ZEND_REGSET_FIXED) # define ZEND_REGSET_SCRATCH \ - (ZEND_REGSET(ZREG_R0) | ZEND_REGSET(ZREG_R1) | ZEND_REGSET(ZREG_R2) | ZEND_REGSET(ZREG_R3) | ZEND_REGSET(ZREG_R12) | ZEND_REGSET_FP) + (ZEND_REGSET(ZREG_RAX) | ZEND_REGSET(ZREG_RDI) | ZEND_REGSET(ZREG_RSI) | ZEND_REGSET(ZREG_RDX) | ZEND_REGSET(ZREG_RCX) | ZEND_REGSET_INTERVAL(ZREG_R8, ZREG_R11) | ZEND_REGSET_FP) +# define ZEND_REGSET_PRESERVED \ + (ZEND_REGSET(ZREG_RBX) | ZEND_REGSET(ZREG_RBP) | ZEND_REGSET(ZREG_R12) | ZEND_REGSET(ZREG_R13)) #else # define ZEND_REGSET_FIXED \ (ZEND_REGSET(ZREG_RSP) | ZEND_REGSET(ZREG_RSI) | ZEND_REGSET(ZREG_RDI)) @@ -139,6 +141,8 @@ typedef uint32_t zend_regset; ZEND_REGSET_DIFFERENCE(ZEND_REGSET_INTERVAL(ZREG_XMM0, ZREG_XMM7), ZEND_REGSET_FIXED) # define ZEND_REGSET_SCRATCH \ (ZEND_REGSET(ZREG_RAX) | ZEND_REGSET(ZREG_RCX) | ZEND_REGSET(ZREG_RDX) | ZEND_REGSET_FP) +# define ZEND_REGSET_PRESERVED \ + (ZEND_REGSET(ZREG_RBX) | ZEND_REGSET(ZREG_RBP)) #endif typedef uintptr_t zend_jit_addr; From 2d341b95053184b28245969fa20862ff25a22dd0 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 27 Mar 2017 22:57:36 +0300 Subject: [PATCH 392/569] Enable register allocation for SEND_VAR --- ext/opcache/jit/zend_jit.c | 2 +- ext/opcache/jit/zend_jit_x86.dasc | 16 +++++++++++++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index bbeaa40791311..e3f4412ff0f7b 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -1953,7 +1953,7 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa, const zend_op *rt_op case ZEND_SEND_VAR_EX: case ZEND_SEND_VAR_NO_REF: case ZEND_SEND_VAR_NO_REF_EX: - if (!zend_jit_send_var(&dasm_state, opline, op_array, ssa)) { + if (!zend_jit_send_var(&dasm_state, opline, op_array, ssa, ra)) { goto jit_failure; } goto done; diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 39a2e1329100c..5f0fa3db09fda 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -6393,20 +6393,19 @@ static int zend_jit_send_ref(dasm_State **Dst, const zend_op *opline, zend_op_ar return 1; } -static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) +static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, zend_lifetime_interval **ra) { uint32_t op1_info; uint32_t arg_num = opline->op2.num; zend_jit_addr op1_addr, arg_addr; - if ((opline->opcode == ZEND_SEND_VAR_EX || opline->opcode == ZEND_SEND_VAR_NO_REF_EX) && arg_num > MAX_ARG_FLAG_NUM) { goto fallback; } - op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, NULL, -1); + op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].op1_use : -1); arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var); op1_info = OP1_INFO(); @@ -6544,6 +6543,16 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, zend_op_ar |2: } } else { + if (ra && ssa->ops[opline - op_array->opcodes].op1_def >= 0) { + zend_jit_addr op1_def_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].op1_def : -1); + + if (op1_addr != op1_def_addr) { + | ZVAL_COPY_VALUE op1_def_addr, op1_addr, MAY_BE_LONG, ZREG_R0, ZREG_R1 + if (Z_MODE(op1_def_addr) == IS_REG && Z_MODE(op1_addr) != IS_REG) { + op1_addr= op1_def_addr; + } + } + } | ZVAL_COPY_VALUE arg_addr, op1_addr, op1_info, ZREG_R0, ZREG_R2 if (opline->op1_type == IS_CV) { | TRY_ADDREF op1_info, ah, r2 @@ -8072,6 +8081,7 @@ static zend_bool zend_jit_opline_supports_reg(zend_op_array *op_array, zend_ssa switch (opline->opcode) { case ZEND_QM_ASSIGN: + case ZEND_SEND_VAR: return 1; case ZEND_PRE_INC: case ZEND_PRE_DEC: From cf78a53167d9ac17790e9e05790348f99fee5c58 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 28 Mar 2017 13:51:31 +0300 Subject: [PATCH 393/569] Reorder basic blocks according to dominators tree; enable inner block register allocation --- ext/opcache/jit/zend_jit.c | 129 ++++++++++++++++++++++++------ ext/opcache/jit/zend_jit_x86.dasc | 7 -- 2 files changed, 106 insertions(+), 30 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index e3f4412ff0f7b..0b88fb4e0c152 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -988,29 +988,66 @@ static int zend_jit_add_range(zend_lifetime_interval **intervals, int var, uint3 ival->range.start = from; ival->range.end = to; ival->range.next = range; + } else if (ival->range.start == to + 1) { + ival->range.start = from; } else { - ZEND_ASSERT(from <= ival->range.start); - ZEND_ASSERT(to <= ival->range.end); - if (from < ival->range.start) { - ival->range.start = from; - } - if (to > ival->range.end) { - ival->range.end = to; + zend_life_range *range = &ival->range; + zend_life_range *last = NULL; + + do { + if (range->start > to + 1) { + break; + } else if (range->end + 1 >= from) { + if (range->start > from) { + range->start = from; + } + last = range; + range = range->next; + while (range) { + if (range->start > to + 1) { + break; + } + range = range->next; + last->next = range; + } + if (to > last->end) { + last->end = to; + } + return SUCCESS; + } + last = range; + range = range->next; + } while (range); + + range = zend_arena_alloc(&CG(arena), sizeof(zend_life_range)); + if (!range) { + return FAILURE; } + range->start = from; + range->end = to; + range->next = last->next; + last->next = range; } + return SUCCESS; } static int zend_jit_begin_range(zend_lifetime_interval **intervals, int var, uint32_t from) { - if (!intervals[var] || intervals[var]->range.start > from) { - // dead store - return zend_jit_add_range(intervals, var, from, from); - } + if (intervals[var]) { + zend_life_range *range = &intervals[var]->range; - intervals[var]->range.start = from; + do { + if (from >= range->start && from <= range->end) { + range->start = from; + return SUCCESS; + } + range = range->next; + } while (range); + } - return SUCCESS; + // dead store + return zend_jit_add_range(intervals, var, from, from); } static void zend_jit_insert_interval(zend_lifetime_interval **list, zend_lifetime_interval *ival) @@ -1147,14 +1184,42 @@ static void zend_jit_print_regset(zend_regset regset) } #endif +static int *zend_jit_compute_block_order_int(zend_ssa *ssa, int n, int *block_order) +{ + zend_basic_block *b = ssa->cfg.blocks + n; + +tail_call: + *block_order = n; + block_order++; + + n = b->children; + while (n >= 0) { + b = ssa->cfg.blocks + n; + if (b->next_child < 0) { + goto tail_call; + } + block_order = zend_jit_compute_block_order_int(ssa, n, block_order); + n = b->next_child; + } + + return block_order; +} + +static void zend_jit_compute_block_order(zend_ssa *ssa, int *block_order) +{ + int *end = zend_jit_compute_block_order_int(ssa, 0, block_order); + ZEND_ASSERT(end - block_order == ssa->cfg.blocks_count); +} + /* See "Linear Scan Register Allocation on SSA Form", Christian Wimmer and Michael Franz, CGO'10 (2010), Figure 4. */ static int zend_jit_compute_liveness(zend_op_array *op_array, zend_ssa *ssa, zend_bitset candidates, zend_lifetime_interval **list) { - int set_size, i, j, k; + int set_size, i, j, k, l; uint32_t n; zend_bitset live, live_in, pi_vars; - uint32_t *loop_end; + uint32_t *loop_start, *loop_end; + int *block_order; zend_ssa_phi *phi; zend_lifetime_interval **intervals; ALLOCA_FLAG(use_heap); @@ -1163,7 +1228,9 @@ static int zend_jit_compute_liveness(zend_op_array *op_array, zend_ssa *ssa, zen intervals = do_alloca( ZEND_MM_ALIGNED_SIZE(ssa->vars_count * sizeof(zend_lifetime_interval*)) + ZEND_MM_ALIGNED_SIZE((set_size * ssa->cfg.blocks_count + 2) * ZEND_BITSET_ELM_SIZE) + - ZEND_MM_ALIGNED_SIZE(ssa->cfg.blocks_count * sizeof(uint32_t)), + ZEND_MM_ALIGNED_SIZE(ssa->cfg.blocks_count * sizeof(uint32_t)) + + ZEND_MM_ALIGNED_SIZE(ssa->cfg.blocks_count * sizeof(uint32_t)) + + ZEND_MM_ALIGNED_SIZE(ssa->cfg.blocks_count * sizeof(int)), use_heap); if (!intervals) { @@ -1174,18 +1241,26 @@ static int zend_jit_compute_liveness(zend_op_array *op_array, zend_ssa *ssa, zen live_in = (zend_bitset)((char*)intervals + ZEND_MM_ALIGNED_SIZE(ssa->vars_count * sizeof(zend_lifetime_interval*))); live = (zend_bitset)((char*)live_in + ZEND_MM_ALIGNED_SIZE((set_size * ssa->cfg.blocks_count) * ZEND_BITSET_ELM_SIZE)); pi_vars = (zend_bitset)((char*)live + ZEND_MM_ALIGNED_SIZE(set_size * ZEND_BITSET_ELM_SIZE)); - loop_end = (uint32_t*)((char*)pi_vars + ZEND_MM_ALIGNED_SIZE(set_size * ZEND_BITSET_ELM_SIZE)); + loop_start = (uint32_t*)((char*)pi_vars + ZEND_MM_ALIGNED_SIZE(set_size * ZEND_BITSET_ELM_SIZE)); + loop_end = (uint32_t*)((char*)loop_start + ZEND_MM_ALIGNED_SIZE(ssa->cfg.blocks_count * sizeof(uint32_t))); + block_order = (int*)((char*)loop_end + ZEND_MM_ALIGNED_SIZE(ssa->cfg.blocks_count * sizeof(uint32_t))); + + zend_jit_compute_block_order(ssa, block_order); memset(intervals, 0, ssa->vars_count * sizeof(zend_lifetime_interval*)); zend_bitset_clear(live_in, set_size * ssa->cfg.blocks_count); + memset(loop_start, 0, ssa->cfg.blocks_count * sizeof(uint32_t)); memset(loop_end, 0, ssa->cfg.blocks_count * sizeof(uint32_t)); /* TODO: Provide a linear block order where all dominators of a block * are before this block, and where all blocks belonging to the same loop * are contiguous ??? */ - for (i = ssa->cfg.blocks_count - 1; i >= 0; i--) { - zend_basic_block *b = ssa->cfg.blocks + i; + for (l = ssa->cfg.blocks_count - 1; l >= 0; l--) { + zend_basic_block *b; + + i = block_order[l]; + b = ssa->cfg.blocks + i; /* live = UNION of successor.liveIn for each successor of b */ /* live.add(phi.inputOf(b)) for each phi of successors of b */ @@ -1291,18 +1366,26 @@ static int zend_jit_compute_liveness(zend_op_array *op_array, zend_ssa *ssa, zen zend_bitset_excl(live, phi->ssa_var); } - if (b->loop_header >= 0 && !loop_end[b->loop_header]) { - loop_end[b->loop_header] = b->start + b->len; + if (b->loop_header >= 0) { + if (!loop_start[b->loop_header] || b->start < loop_start[b->loop_header]) { + loop_start[b->loop_header] = b->start; + } + if (b->start + b->len > loop_end[b->loop_header]) { + loop_end[b->loop_header] = b->start + b->len; + } } /* if b is loop header */ if (b->flags & ZEND_BB_LOOP_HEADER) { - if (!loop_end[i]) { + if (!loop_start[i] || b->start < loop_start[i]) { + loop_start[i] = b->start; + } + if (b->start + b->len > loop_end[i]) { loop_end[i] = b->start + b->len; } ZEND_BITSET_FOREACH(live, set_size, j) { if (zend_bitset_in(candidates, j)) { - if (zend_jit_add_range(intervals, j, b->start, loop_end[i]) != SUCCESS) { + if (zend_jit_add_range(intervals, j, loop_start[i], loop_end[i]) != SUCCESS) { goto failure; } } diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 5f0fa3db09fda..ef39d366a5f65 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -8137,10 +8137,8 @@ static zend_bool zend_jit_may_be_in_reg(zend_op_array *op_array, zend_ssa *ssa, if (((ssa->var_info[var].type & MAY_BE_ANY) == MAY_BE_DOUBLE) || ((ssa->var_info[var].type & MAY_BE_ANY) == MAY_BE_LONG)) { - int definition_block = -1; if (ssa->vars[var].definition >= 0) { - definition_block = ssa->cfg.map[ssa->vars[var].definition]; if (!zend_jit_opline_supports_reg(op_array, ssa, op_array->opcodes + ssa->vars[var].definition, var)) { return 0; } @@ -8149,11 +8147,6 @@ static zend_bool zend_jit_may_be_in_reg(zend_op_array *op_array, zend_ssa *ssa, int use = ssa->vars[var].use_chain; do { - if (ssa->cfg.map[use] != definition_block) { - // TODO: enable global register allocation ??? - // Incorrect basic block oreder breaks register allocation. - return 0; - } if (!zend_jit_opline_supports_reg(op_array, ssa, op_array->opcodes + use, var)) { return 0; } From 2f727867e0f7149f435620bef2f9e032169eea1b Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 28 Mar 2017 17:13:22 +0300 Subject: [PATCH 394/569] Accurate handling life-time intervals of loop variables --- ext/opcache/jit/zend_jit.c | 95 +++++++++++++++++++++++++------------- 1 file changed, 62 insertions(+), 33 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 0b88fb4e0c152..6d9406fe21242 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -1211,14 +1211,49 @@ static void zend_jit_compute_block_order(zend_ssa *ssa, int *block_order) ZEND_ASSERT(end - block_order == ssa->cfg.blocks_count); } +static zend_bool zend_jit_in_loop(zend_ssa *ssa, int header, zend_basic_block *b) +{ + while (b->loop_header >= 0) { + if (b->loop_header == header) { + return 1; + } + b = ssa->cfg.blocks + b->loop_header; + } + return 0; +} + +static void zend_jit_compute_loop_body(zend_ssa *ssa, int header, int n, zend_bitset loop_body) +{ + zend_basic_block *b = ssa->cfg.blocks + n; + uint32_t i; + +tail_call: + if (b->len) { + for (i = b->start; i < b->start + b->len; i++) { + zend_bitset_incl(loop_body, i); + } + } + + n = b->children; + while (n >= 0) { + b = ssa->cfg.blocks + n; + if (zend_jit_in_loop(ssa, header, b)) { + if (b->next_child < 0) { + goto tail_call; + } + zend_jit_compute_loop_body(ssa, header, n, loop_body); + } + n = b->next_child; + } +} + /* See "Linear Scan Register Allocation on SSA Form", Christian Wimmer and Michael Franz, CGO'10 (2010), Figure 4. */ static int zend_jit_compute_liveness(zend_op_array *op_array, zend_ssa *ssa, zend_bitset candidates, zend_lifetime_interval **list) { int set_size, i, j, k, l; uint32_t n; - zend_bitset live, live_in, pi_vars; - uint32_t *loop_start, *loop_end; + zend_bitset live, live_in, pi_vars, loop_body; int *block_order; zend_ssa_phi *phi; zend_lifetime_interval **intervals; @@ -1227,9 +1262,8 @@ static int zend_jit_compute_liveness(zend_op_array *op_array, zend_ssa *ssa, zen set_size = zend_bitset_len(ssa->vars_count); intervals = do_alloca( ZEND_MM_ALIGNED_SIZE(ssa->vars_count * sizeof(zend_lifetime_interval*)) + - ZEND_MM_ALIGNED_SIZE((set_size * ssa->cfg.blocks_count + 2) * ZEND_BITSET_ELM_SIZE) + - ZEND_MM_ALIGNED_SIZE(ssa->cfg.blocks_count * sizeof(uint32_t)) + - ZEND_MM_ALIGNED_SIZE(ssa->cfg.blocks_count * sizeof(uint32_t)) + + ZEND_MM_ALIGNED_SIZE((set_size * (ssa->cfg.blocks_count + 2)) * ZEND_BITSET_ELM_SIZE) + + ZEND_MM_ALIGNED_SIZE(zend_bitset_len(op_array->last) * ZEND_BITSET_ELM_SIZE) + ZEND_MM_ALIGNED_SIZE(ssa->cfg.blocks_count * sizeof(int)), use_heap); @@ -1241,16 +1275,12 @@ static int zend_jit_compute_liveness(zend_op_array *op_array, zend_ssa *ssa, zen live_in = (zend_bitset)((char*)intervals + ZEND_MM_ALIGNED_SIZE(ssa->vars_count * sizeof(zend_lifetime_interval*))); live = (zend_bitset)((char*)live_in + ZEND_MM_ALIGNED_SIZE((set_size * ssa->cfg.blocks_count) * ZEND_BITSET_ELM_SIZE)); pi_vars = (zend_bitset)((char*)live + ZEND_MM_ALIGNED_SIZE(set_size * ZEND_BITSET_ELM_SIZE)); - loop_start = (uint32_t*)((char*)pi_vars + ZEND_MM_ALIGNED_SIZE(set_size * ZEND_BITSET_ELM_SIZE)); - loop_end = (uint32_t*)((char*)loop_start + ZEND_MM_ALIGNED_SIZE(ssa->cfg.blocks_count * sizeof(uint32_t))); - block_order = (int*)((char*)loop_end + ZEND_MM_ALIGNED_SIZE(ssa->cfg.blocks_count * sizeof(uint32_t))); - - zend_jit_compute_block_order(ssa, block_order); + loop_body = (zend_bitset)((char*)pi_vars + ZEND_MM_ALIGNED_SIZE(set_size * ZEND_BITSET_ELM_SIZE)); + block_order = (int*)((char*)loop_body + ZEND_MM_ALIGNED_SIZE(zend_bitset_len(op_array->last) * ZEND_BITSET_ELM_SIZE)); memset(intervals, 0, ssa->vars_count * sizeof(zend_lifetime_interval*)); zend_bitset_clear(live_in, set_size * ssa->cfg.blocks_count); - memset(loop_start, 0, ssa->cfg.blocks_count * sizeof(uint32_t)); - memset(loop_end, 0, ssa->cfg.blocks_count * sizeof(uint32_t)); + zend_jit_compute_block_order(ssa, block_order); /* TODO: Provide a linear block order where all dominators of a block * are before this block, and where all blocks belonging to the same loop @@ -1366,30 +1396,29 @@ static int zend_jit_compute_liveness(zend_op_array *op_array, zend_ssa *ssa, zen zend_bitset_excl(live, phi->ssa_var); } - if (b->loop_header >= 0) { - if (!loop_start[b->loop_header] || b->start < loop_start[b->loop_header]) { - loop_start[b->loop_header] = b->start; - } - if (b->start + b->len > loop_end[b->loop_header]) { - loop_end[b->loop_header] = b->start + b->len; - } - } - /* if b is loop header */ - if (b->flags & ZEND_BB_LOOP_HEADER) { - if (!loop_start[i] || b->start < loop_start[i]) { - loop_start[i] = b->start; - } - if (b->start + b->len > loop_end[i]) { - loop_end[i] = b->start + b->len; - } - ZEND_BITSET_FOREACH(live, set_size, j) { - if (zend_bitset_in(candidates, j)) { - if (zend_jit_add_range(intervals, j, loop_start[i], loop_end[i]) != SUCCESS) { + if ((b->flags & ZEND_BB_LOOP_HEADER) && + !zend_bitset_empty(live, set_size)) { + uint32_t set_size2 = zend_bitset_len(op_array->last); + + zend_bitset_clear(loop_body, set_size2); + zend_jit_compute_loop_body(ssa, i, i, loop_body); + while (!zend_bitset_empty(loop_body, set_size2)) { + uint32_t from = zend_bitset_first(loop_body, set_size2); + uint32_t to = from; + + do { + zend_bitset_excl(loop_body, to); + to++; + } while (zend_bitset_in(loop_body, to)); + to--; + + ZEND_BITSET_FOREACH(live, set_size, j) { + if (zend_jit_add_range(intervals, j, from, to) != SUCCESS) { goto failure; } - } - } ZEND_BITSET_FOREACH_END(); + } ZEND_BITSET_FOREACH_END(); + } } /* b.liveIn = live */ From 2428521eb9048079b6da6a1bc5cea5c0fd0ad2ff Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 29 Mar 2017 11:51:47 +0300 Subject: [PATCH 395/569] cleanup --- ext/opcache/jit/zend_jit.c | 4 +--- ext/opcache/jit/zend_jit_x86.dasc | 38 +++++++++++++++++-------------- 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 6d9406fe21242..5dfbb8b13b526 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -1756,9 +1756,7 @@ static zend_lifetime_interval** zend_jit_allocate_registers(zend_op_array *op_ar candidates_count = 0; zend_bitset_clear(candidates, set_size); for (i = 0; i < ssa->vars_count; i++) { - if (/*!ssa->vars[i].no_val - && */!(ssa->var_info[i].type & MAY_BE_REF) - && zend_jit_may_be_in_reg(op_array, ssa, i)) { + if (zend_jit_may_be_in_reg(op_array, ssa, i)) { zend_bitset_incl(candidates, i); candidates_count++; } diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index ef39d366a5f65..e33a499258682 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -8132,30 +8132,34 @@ static zend_bool zend_jit_may_be_in_reg(zend_op_array *op_array, zend_ssa *ssa, } if (ssa->vars[var].no_val) { + /* we don't need the value */ return 0; } - if (((ssa->var_info[var].type & MAY_BE_ANY) == MAY_BE_DOUBLE) || - ((ssa->var_info[var].type & MAY_BE_ANY) == MAY_BE_LONG)) { + if (((ssa->var_info[var].type & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_DOUBLE) && + ((ssa->var_info[var].type & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_LONG)) { + /* bad type */ + return 0; + } - if (ssa->vars[var].definition >= 0) { - if (!zend_jit_opline_supports_reg(op_array, ssa, op_array->opcodes + ssa->vars[var].definition, var)) { - return 0; - } + if (ssa->vars[var].definition >= 0) { + if (!zend_jit_opline_supports_reg(op_array, ssa, op_array->opcodes + ssa->vars[var].definition, var)) { + return 0; } - if (ssa->vars[var].use_chain >= 0) { - int use = ssa->vars[var].use_chain; + } - do { - if (!zend_jit_opline_supports_reg(op_array, ssa, op_array->opcodes + use, var)) { - return 0; - } - use = zend_ssa_next_use(ssa->ops, var, use); - } while (use >= 0); - } - return 1; + if (ssa->vars[var].use_chain >= 0) { + int use = ssa->vars[var].use_chain; + + do { + if (!zend_jit_opline_supports_reg(op_array, ssa, op_array->opcodes + use, var)) { + return 0; + } + use = zend_ssa_next_use(ssa->ops, var, use); + } while (use >= 0); } - return 0; + + return 1; } static zend_regset zend_jit_get_scratch_regset(zend_op_array *op_array, zend_ssa *ssa, zend_op *opline) From 8c5a3396502155b60a6a1a1a0958ec695e71a6ef Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 29 Mar 2017 15:58:34 +0300 Subject: [PATCH 396/569] More accurate register reuse --- ext/opcache/jit/zend_jit.c | 41 +++++++++++++++++-------------- ext/opcache/jit/zend_jit_x86.dasc | 28 +++++++++++++++++++++ 2 files changed, 50 insertions(+), 19 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 5dfbb8b13b526..b7d624aa896cc 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -1487,6 +1487,7 @@ static int zend_jit_try_allocate_free_reg(zend_op_array *op_array, zend_ssa *ssa uint32_t freeUntilPos[ZREG_NUM]; uint32_t pos; zend_reg i, reg; + zend_reg hint = ZREG_NONE; zend_life_range *range; if ((ssa->var_info[current->ssa_var].type & MAY_BE_ANY) == MAY_BE_DOUBLE) { @@ -1512,9 +1513,22 @@ static int zend_jit_try_allocate_free_reg(zend_op_array *op_array, zend_ssa *ssa /* for each interval it in active do */ /* freeUntilPos[it.reg] = 0 */ it = active; - while (it) { - freeUntilPos[it->reg] = 0; - it = it->list_next; + if (ssa->vars[current->ssa_var].definition == current->range.start) { + while (it) { + if (current->range.start != zend_interval_end(it)) { + freeUntilPos[it->reg] = 0; + } else if (zend_jit_may_reuse_reg(op_array, ssa, current->range.start, current->ssa_var, it->ssa_var)) { + hint = it->reg; + } else { + freeUntilPos[it->reg] = 0; + } + it = it->list_next; + } + } else { + while (it) { + freeUntilPos[it->reg] = 0; + it = it->list_next; + } } /* See "Linear Scan Register Allocation on SSA Form", Christian Wimmer and @@ -1592,6 +1606,11 @@ static int zend_jit_try_allocate_free_reg(zend_op_array *op_array, zend_ssa *ssa } #endif + if (hint != ZREG_NONE && freeUntilPos[hint] > zend_interval_end(current)) { + current->reg = hint; + return 1; + } + pos = 0; reg = ZREG_NONE; for (i = 0; i < ZREG_NUM; i++) { if (ZEND_REGSET_IN(available, i) && freeUntilPos[i] > pos) { @@ -1652,26 +1671,10 @@ static zend_lifetime_interval* zend_jit_linear_scan(zend_op_array *op_array, zen q = *p; if (end < position) { /* move ival from active to handled */ -end_interval: ZEND_REGSET_INCL(available, q->reg); *p = q->list_next; q->list_next = handled; handled = q; - } else if (end == position) { - /* In some cases, we may (and should) reuse operand - registers for result (coalesce) */ - switch (op_array->opcodes[position].opcode) { - case ZEND_QM_ASSIGN: - case ZEND_ADD: - case ZEND_MUL: - if (q->ssa_var == ssa->ops[position].op1_use) { - goto end_interval; - } - break; - default: - break; - } - p = &q->list_next; } else if (!zend_interval_covers(q, position)) { /* move ival from active to inactive */ ZEND_REGSET_INCL(available, q->reg); diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index e33a499258682..0d2f55e3bd652 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -8075,6 +8075,34 @@ static int zend_jit_free(dasm_State **Dst, const zend_op *opline, zend_op_array } #ifdef REG_ALLOC +static zend_bool zend_jit_may_reuse_reg(zend_op_array *op_array, zend_ssa *ssa, uint32_t position, int def_var, int use_var) +{ + if (ssa->var_info[def_var].type != ssa->var_info[use_var].type) { + return 0; + } + + switch (op_array->opcodes[position].opcode) { + case ZEND_QM_ASSIGN: + case ZEND_SEND_VAR: + case ZEND_PRE_INC: + case ZEND_PRE_DEC: + case ZEND_POST_INC: + case ZEND_POST_DEC: + return 1; + case ZEND_ADD: + case ZEND_SUB: + case ZEND_MUL: + if (def_var == ssa->ops[position].result_def && + use_var == ssa->ops[position].op1_use) { + return 1; + } + break; + default: + break; + } + return 0; +} + static zend_bool zend_jit_opline_supports_reg(zend_op_array *op_array, zend_ssa *ssa, zend_op *opline, int var) { uint32_t op1_info, op2_info; From 26ec007df54d466986562367c6991a6474a3a1d5 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 29 Mar 2017 17:07:06 +0300 Subject: [PATCH 397/569] Reuse operand register for temporary result --- ext/opcache/jit/zend_jit.c | 7 ++++ ext/opcache/jit/zend_jit_x86.dasc | 69 ++++++++++++++++++++++--------- 2 files changed, 57 insertions(+), 19 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index b7d624aa896cc..af1048c81815b 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -82,6 +82,13 @@ static void **dasm_ptr = NULL; static int zend_may_throw(const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa); static int zend_may_overflow(const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa); +static zend_bool zend_ssa_is_last_use(const zend_ssa *ssa, int var, int use) +{ + return + !ssa->vars[var].phi_use_chain && + zend_ssa_next_use(ssa->ops, var, use) < 0; +} + #include "dynasm/dasm_x86.h" #include "jit/zend_jit_x86.h" #include "jit/zend_jit_helpers.c" diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 0d2f55e3bd652..73870e0c46844 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -2085,17 +2085,24 @@ fallback: } static int zend_jit_math_long_long(dasm_State **Dst, - const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, - zend_jit_addr op1_addr, - zend_jit_addr op2_addr, - zend_jit_addr res_addr, - uint32_t res_info) + const zend_op *opline, + zend_jit_addr op1_addr, + zend_jit_addr op2_addr, + zend_jit_addr res_addr, + uint32_t res_info) { zend_bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); - zend_reg result_reg = - (Z_MODE(res_addr) == IS_REG) ? Z_REG(res_addr) : ZREG_R0; + zend_reg result_reg; + + if (Z_MODE(res_addr) == IS_REG) { + result_reg = Z_REG(res_addr); + } else if (Z_MODE(op1_addr) == IS_REG && zend_ssa_is_last_use(ssa, ssa->ops[opline-op_array->opcodes].op1_use, opline-op_array->opcodes)) { + result_reg = Z_REG(op1_addr); + } else { + result_reg = ZREG_R0; + } if (opline->opcode == ZEND_MUL && ((Z_MODE(op2_addr) == IS_CONST_ZVAL && @@ -2161,6 +2168,8 @@ static int zend_jit_math_long_long(dasm_State **Dst, } static int zend_jit_math_long_double(dasm_State **Dst, + zend_op_array *op_array, + zend_ssa *ssa, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, @@ -2187,21 +2196,34 @@ static int zend_jit_math_long_double(dasm_State **Dst, } static int zend_jit_math_double_long(dasm_State **Dst, + zend_op_array *op_array, + zend_ssa *ssa, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr) { |.if SSE - zend_reg result_reg = - (Z_MODE(res_addr) == IS_REG) ? Z_REG(res_addr) : ZREG_XMM0; + zend_reg result_reg; if (opline->opcode == ZEND_ADD || opline->opcode == ZEND_MUL || opline->opcode == ZEND_ASSIGN_ADD || opline->opcode == ZEND_ASSIGN_MUL) { + if (Z_MODE(res_addr) == IS_REG) { + result_reg = Z_REG(res_addr); + } else { + result_reg = ZREG_XMM0; + } | SSE_GET_ZVAL_LVAL result_reg, op2_addr | SSE_MATH opline->opcode, result_reg, op1_addr } else { zend_reg tmp_reg = ZREG_XMM1; + if (Z_MODE(res_addr) == IS_REG) { + result_reg = Z_REG(res_addr); + } else if (Z_MODE(op1_addr) == IS_REG && zend_ssa_is_last_use(ssa, ssa->ops[opline-op_array->opcodes].op1_use, opline-op_array->opcodes)) { + result_reg = Z_REG(op1_addr); + } else { + result_reg = ZREG_XMM0; + } | SSE_GET_ZVAL_DVAL result_reg, op1_addr | SSE_GET_ZVAL_LVAL tmp_reg, op2_addr | SSE_MATH_REG opline->opcode, result_reg, tmp_reg @@ -2228,6 +2250,8 @@ static int zend_jit_math_double_long(dasm_State **Dst, } static int zend_jit_math_double_double(dasm_State **Dst, + zend_op_array *op_array, + zend_ssa *ssa, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, @@ -2236,8 +2260,15 @@ static int zend_jit_math_double_double(dasm_State **Dst, zend_bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); |.if SSE - zend_reg result_reg = - (Z_MODE(res_addr) == IS_REG) ? Z_REG(res_addr) : ZREG_XMM0; + zend_reg result_reg; + + if (Z_MODE(res_addr) == IS_REG) { + result_reg = Z_REG(res_addr); + } else if (Z_MODE(op1_addr) == IS_REG && zend_ssa_is_last_use(ssa, ssa->ops[opline-op_array->opcodes].op1_use, opline-op_array->opcodes)) { + result_reg = Z_REG(op1_addr); + } else { + result_reg = ZREG_XMM0; + } | SSE_GET_ZVAL_DVAL result_reg, op1_addr if (same_ops) { @@ -2362,7 +2393,7 @@ static int zend_jit_math_helper(dasm_State **Dst, if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >6 } - if (!zend_jit_math_long_double(Dst, opline, op1_addr, op2_addr, res_addr)) { + if (!zend_jit_math_long_double(Dst, op_array, ssa, opline, op1_addr, op2_addr, res_addr)) { return 0; } | jmp >5 @@ -2371,7 +2402,7 @@ static int zend_jit_math_helper(dasm_State **Dst, | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6 } } - if (!zend_jit_math_long_long(Dst, opline, op_array, ssa, op1_addr, op2_addr, res_addr, res_info)) { + if (!zend_jit_math_long_long(Dst, op_array, ssa, opline, op1_addr, op2_addr, res_addr, res_info)) { return 0; } if (op1_info & MAY_BE_DOUBLE) { @@ -2388,7 +2419,7 @@ static int zend_jit_math_helper(dasm_State **Dst, | IF_NOT_ZVAL_TYPE, op2_addr, IS_DOUBLE, >6 } } - if (!zend_jit_math_double_double(Dst, opline, op1_addr, op2_addr, res_addr)) { + if (!zend_jit_math_double_double(Dst, op_array, ssa, opline, op1_addr, op2_addr, res_addr)) { return 0; } | jmp >5 @@ -2398,7 +2429,7 @@ static int zend_jit_math_helper(dasm_State **Dst, if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6 } - if (!zend_jit_math_double_long(Dst, opline, op1_addr, op2_addr, res_addr)) { + if (!zend_jit_math_double_long(Dst, op_array, ssa, opline, op1_addr, op2_addr, res_addr)) { return 0; } | jmp >5 @@ -2419,7 +2450,7 @@ static int zend_jit_math_helper(dasm_State **Dst, | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >6 } } - if (!zend_jit_math_double_double(Dst, opline, op1_addr, op2_addr, res_addr)) { + if (!zend_jit_math_double_double(Dst, op_array, ssa, opline, op1_addr, op2_addr, res_addr)) { return 0; } } @@ -2431,7 +2462,7 @@ static int zend_jit_math_helper(dasm_State **Dst, if (op2_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) { | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >5 } - if (!zend_jit_math_double_long(Dst, opline, op1_addr, op2_addr, res_addr)) { + if (!zend_jit_math_double_long(Dst, op_array, ssa, opline, op1_addr, op2_addr, res_addr)) { return 0; } if (op2_info & MAY_BE_DOUBLE) { @@ -2453,7 +2484,7 @@ static int zend_jit_math_helper(dasm_State **Dst, | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >6 } } - if (!zend_jit_math_double_double(Dst, opline, op1_addr, op2_addr, res_addr)) { + if (!zend_jit_math_double_double(Dst, op_array, ssa, opline, op1_addr, op2_addr, res_addr)) { return 0; } } @@ -2465,7 +2496,7 @@ static int zend_jit_math_helper(dasm_State **Dst, if (op1_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) { | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6 } - if (!zend_jit_math_long_double(Dst, opline, op1_addr, op2_addr, res_addr)) { + if (!zend_jit_math_long_double(Dst, op_array, ssa, opline, op1_addr, op2_addr, res_addr)) { return 0; } if (op1_info & MAY_BE_DOUBLE) { From c7802fc1331fffe0649326c75a241403aefdb4f4 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 30 Mar 2017 16:14:44 +0300 Subject: [PATCH 398/569] Enable register allocation for SEND_VAL/SEND_VAL_EX --- ext/opcache/jit/zend_jit.c | 2 +- ext/opcache/jit/zend_jit_x86.dasc | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index af1048c81815b..f8d81b84db37e 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -2060,7 +2060,7 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa, const zend_op *rt_op goto done; case ZEND_SEND_VAL: case ZEND_SEND_VAL_EX: - if (!zend_jit_send_val(&dasm_state, opline, op_array, ssa)) { + if (!zend_jit_send_val(&dasm_state, opline, op_array, ssa, ra)) { goto jit_failure; } goto done; diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 73870e0c46844..c39f513e70a63 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -6255,7 +6255,7 @@ fallback: } } -static int zend_jit_send_val(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) +static int zend_jit_send_val(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, zend_lifetime_interval **ra) { uint32_t op1_info; uint32_t arg_num = opline->op2.num; @@ -6290,7 +6290,7 @@ static int zend_jit_send_val(dasm_State **Dst, const zend_op *opline, zend_op_ar |.code } - op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, NULL, -1); + op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].op1_use : -1); arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var); if (opline->op1_type == IS_CONST) { @@ -8141,6 +8141,8 @@ static zend_bool zend_jit_opline_supports_reg(zend_op_array *op_array, zend_ssa switch (opline->opcode) { case ZEND_QM_ASSIGN: case ZEND_SEND_VAR: + case ZEND_SEND_VAL: + case ZEND_SEND_VAL_EX: return 1; case ZEND_PRE_INC: case ZEND_PRE_DEC: @@ -8233,6 +8235,8 @@ static zend_regset zend_jit_get_scratch_regset(zend_op_array *op_array, zend_ssa regset = ZEND_REGSET_EMPTY; break; case ZEND_QM_ASSIGN: + case ZEND_SEND_VAL: + case ZEND_SEND_VAL_EX: op1_info = OP1_INFO(); if (!(op1_info & MAY_BE_UNDEF)) { res_info = RES_INFO(); From 5dd41052f2662ed1b17a53c801ae836163a36f72 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 30 Mar 2017 16:23:06 +0300 Subject: [PATCH 399/569] INC/DEC use standard handler for IS_VAR operand --- ext/opcache/jit/zend_jit_x86.dasc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index c39f513e70a63..755925f81345f 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -8254,7 +8254,8 @@ static zend_regset zend_jit_get_scratch_regset(zend_op_array *op_array, zend_ssa case ZEND_POST_INC: case ZEND_POST_DEC: op1_info = OP1_INFO(); - if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG))) { + if (opline->op1_type == IS_CV + && !(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG))) { regset = ZEND_REGSET_EMPTY; if (opline->result_type != IS_UNUSED) { ZEND_REGSET_INCL(regset, ZREG_R0); From d0f5afbefabe646e0161a91447679d6ce43de118 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 30 Mar 2017 17:51:17 +0300 Subject: [PATCH 400/569] Register allocation support for ASSIGN (incomplete) --- ext/opcache/jit/zend_jit.c | 2 +- ext/opcache/jit/zend_jit_x86.dasc | 39 +++++++++++++++++++++++++++---- 2 files changed, 36 insertions(+), 5 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index f8d81b84db37e..0b498b0bc2039 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -2043,7 +2043,7 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa, const zend_op *rt_op } goto done; case ZEND_ASSIGN: - if (!zend_jit_assign(&dasm_state, opline, op_array, ssa)) { + if (!zend_jit_assign(&dasm_state, opline, op_array, ssa, ra)) { goto jit_failure; } goto done; diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 755925f81345f..5dffcaaae8ab0 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -5488,7 +5488,7 @@ static int zend_jit_qm_assign(dasm_State **Dst, const zend_op *opline, zend_op_a return zend_jit_simple_assign(Dst, opline, op_array, ssa, res_addr, -1, opline->op1_type, opline->op1, op1_addr, op1_info, 0, 0); } -static int zend_jit_assign(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) +static int zend_jit_assign(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, zend_lifetime_interval **ra) { uint32_t op1_info, op2_info; zend_jit_addr op1_addr, op2_addr, res_addr; @@ -5500,12 +5500,20 @@ static int zend_jit_assign(dasm_State **Dst, const zend_op *opline, zend_op_arra op1_info = OP1_INFO(); op2_info = OP2_INFO(); - op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, NULL, -1); - op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2, opline, NULL, -1); + op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].op1_use : -1); + op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].op2_use : -1); if (opline->result_type == IS_UNUSED) { res_addr = 0; } else { - res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1); + res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].result_def : -1); + } + + if (ra + && ssa->ops[opline - op_array->opcodes].op2_def >= 0 + && !ssa->vars[ssa->ops[opline - op_array->opcodes].op2_def].no_val + && !zend_jit_update_regs(Dst, op2_info, op2_addr, + zend_jit_decode_op(op_array, opline->op2_type, opline->op2, opline, ra, ssa->ops[opline - op_array->opcodes].op2_def))) { + return 0; } if (op1_info & MAY_BE_REF) { @@ -8115,6 +8123,7 @@ static zend_bool zend_jit_may_reuse_reg(zend_op_array *op_array, zend_ssa *ssa, switch (op_array->opcodes[position].opcode) { case ZEND_QM_ASSIGN: case ZEND_SEND_VAR: + case ZEND_ASSIGN: case ZEND_PRE_INC: case ZEND_PRE_DEC: case ZEND_POST_INC: @@ -8144,12 +8153,20 @@ static zend_bool zend_jit_opline_supports_reg(zend_op_array *op_array, zend_ssa case ZEND_SEND_VAL: case ZEND_SEND_VAL_EX: return 1; + case ZEND_ASSIGN: + op1_info = OP1_INFO(); + op2_info = OP2_INFO(); + return + opline->op1_type == IS_CV && + !(op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_RESOURCE|MAY_BE_REF)) && + !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))); case ZEND_PRE_INC: case ZEND_PRE_DEC: case ZEND_POST_INC: case ZEND_POST_DEC: op1_info = OP1_INFO(); return + opline->op1_type == IS_CV && !(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG))); case ZEND_ADD: case ZEND_SUB: @@ -8249,6 +8266,20 @@ static zend_regset zend_jit_get_scratch_regset(zend_op_array *op_array, zend_ssa } } break; + case ZEND_ASSIGN: + op1_info = OP1_INFO(); + op2_info = OP2_INFO(); + if (opline->op1_type == IS_CV + && !(op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_RESOURCE|MAY_BE_REF))) { + if ((op2_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) { + regset = ZEND_REGSET(ZREG_XMM0); + } else if ((op2_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) { + regset = ZEND_REGSET(ZREG_R0); + } else { + regset = ZEND_REGSET_UNION(ZEND_REGSET(ZREG_R0), ZEND_REGSET(ZREG_R2)); + } + } + break; case ZEND_PRE_INC: case ZEND_PRE_DEC: case ZEND_POST_INC: From 726c32c75ab6682eeaf20a9dbed406eaae35b241 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 30 Mar 2017 20:31:07 +0300 Subject: [PATCH 401/569] Eliminate useless type assignemtns --- ext/opcache/jit/zend_jit.c | 4 +- ext/opcache/jit/zend_jit_x86.dasc | 136 ++++++++++++++++++------------ 2 files changed, 82 insertions(+), 58 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 0b498b0bc2039..2faccabadd31c 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -896,7 +896,7 @@ static int zend_may_overflow(const zend_op *opline, zend_op_array *op_array, zen static int zend_jit_build_cfg(zend_op_array *op_array, zend_cfg *cfg, uint32_t *flags) { - if (zend_build_cfg(&CG(arena), op_array, ZEND_CFG_STACKLESS | ZEND_CFG_RECV_ENTRY | ZEND_RT_CONSTANTS | ZEND_CFG_SPLIT_AT_LIVE_RANGES | ZEND_CFG_NO_ENTRY_PREDECESSORS | ZEND_SSA_RC_INFERENCE_FLAG, cfg, flags) != SUCCESS) { + if (zend_build_cfg(&CG(arena), op_array, ZEND_CFG_STACKLESS | ZEND_CFG_RECV_ENTRY | ZEND_RT_CONSTANTS | ZEND_CFG_SPLIT_AT_LIVE_RANGES | ZEND_CFG_NO_ENTRY_PREDECESSORS | ZEND_SSA_RC_INFERENCE_FLAG | ZEND_SSA_USE_CV_RESULTS, cfg, flags) != SUCCESS) { return FAILURE; } @@ -928,7 +928,7 @@ static int zend_jit_op_array_analyze1(zend_op_array *op_array, zend_script *scri && op_array->last_try_catch == 0 && !(op_array->fn_flags & ZEND_ACC_GENERATOR) && !(*flags & ZEND_FUNC_INDIRECT_VAR_ACCESS)) { - if (zend_build_ssa(&CG(arena), script, op_array, ZEND_RT_CONSTANTS | ZEND_SSA_RC_INFERENCE, ssa, flags) != SUCCESS) { + if (zend_build_ssa(&CG(arena), script, op_array, ZEND_RT_CONSTANTS | ZEND_SSA_RC_INFERENCE | ZEND_SSA_USE_CV_RESULTS, ssa, flags) != SUCCESS) { return FAILURE; } diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 5dffcaaae8ab0..6f133b1e1057d 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -586,7 +586,7 @@ static void* dasm_labels[zend_lb_MAX]; || } || } || if (Z_MODE(dst_addr) == IS_MEM_ZVAL) { -|| if (((dst_info & MAY_BE_ANY) != (1<op1_type, opline->op1, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].op1_use : -1); op1_def_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].op1_def : -1); if (opline->result_type != IS_UNUSED) { + res_use_info = RES_USE_INFO(); res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].result_def : -1); } @@ -1970,10 +1975,10 @@ static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, zend_op_arr } if ((opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) && opline->result_type != IS_UNUSED) { - | ZVAL_COPY_VALUE res_addr, op1_addr, MAY_BE_LONG, ZREG_R0, ZREG_R1 + | ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, MAY_BE_LONG, ZREG_R0, ZREG_R1 } if (op1_addr != op1_def_addr) { - | ZVAL_COPY_VALUE op1_def_addr, op1_addr, MAY_BE_LONG, ZREG_R0, ZREG_R1 + | ZVAL_COPY_VALUE op1_def_addr, op1_info, op1_addr, MAY_BE_LONG, ZREG_R0, ZREG_R1 } if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { | LONG_OP_WITH_CONST add, op1_def_addr, Z_L(1) @@ -1986,7 +1991,7 @@ static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, zend_op_arr | jo >1 if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && opline->result_type != IS_UNUSED) { - | ZVAL_COPY_VALUE res_addr, op1_def_addr, MAY_BE_LONG, ZREG_R0, ZREG_R1 + | ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_LONG, ZREG_R0, ZREG_R1 } |.cold_code |1: @@ -2010,14 +2015,14 @@ static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, zend_op_arr | SET_ZVAL_TYPE_INFO op1_addr, IS_DOUBLE if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && opline->result_type != IS_UNUSED) { - | ZVAL_COPY_VALUE res_addr, op1_def_addr, MAY_BE_DOUBLE, ZREG_R0, ZREG_R1 + | ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_DOUBLE, ZREG_R0, ZREG_R1 } | jmp >3 |.code } else { if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && opline->result_type != IS_UNUSED) { - | ZVAL_COPY_VALUE res_addr, op1_def_addr, MAY_BE_LONG, ZREG_R0, ZREG_R1 + | ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_LONG, ZREG_R0, ZREG_R1 } } if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { @@ -2040,7 +2045,7 @@ static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, zend_op_arr if (opline->result_type != IS_UNUSED) { zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); - | ZVAL_COPY_VALUE res_addr, val_addr, op1_info, ZREG_R0, ZREG_R2 + | ZVAL_COPY_VALUE res_addr, res_use_info, val_addr, op1_info, ZREG_R0, ZREG_R2 | //ZVAL_COPY_CTOR op1_info, ah, r2, op_array->filename, opline->lineno if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY)) { @@ -2070,7 +2075,7 @@ static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, zend_op_arr } if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && opline->result_type != IS_UNUSED) { - | ZVAL_COPY_VALUE res_addr, op1_addr, op1_def_info, ZREG_R0, ZREG_R1 + | ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, op1_def_info, ZREG_R0, ZREG_R1 | TRY_ADDREF op1_def_info, ah, r1 } | jmp >3 @@ -2091,7 +2096,8 @@ static int zend_jit_math_long_long(dasm_State **Dst, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, - uint32_t res_info) + uint32_t res_info, + uint32_t res_use_info) { zend_bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); zend_reg result_reg; @@ -2149,18 +2155,24 @@ static int zend_jit_math_long_long(dasm_State **Dst, | FPU_MATH_REG opline->opcode, st1 | FPU_SET_ZVAL_DVAL res_addr |.endif - | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE + if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_DOUBLE) { + | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE + } | jmp >2 |.code | SET_ZVAL_LVAL res_addr, Ra(result_reg) if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { - | SET_ZVAL_TYPE_INFO res_addr, IS_LONG + if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_LONG) { + | SET_ZVAL_TYPE_INFO res_addr, IS_LONG + } } |2: } else if (Z_MODE(res_addr) == IS_MEM_ZVAL) { | SET_ZVAL_LVAL res_addr, Ra(result_reg) if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { - | SET_ZVAL_TYPE_INFO res_addr, IS_LONG + if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_LONG) { + | SET_ZVAL_TYPE_INFO res_addr, IS_LONG + } } } @@ -2173,7 +2185,8 @@ static int zend_jit_math_long_double(dasm_State **Dst, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, - zend_jit_addr res_addr) + zend_jit_addr res_addr, + uint32_t res_use_info) { |.if SSE zend_reg result_reg = @@ -2189,7 +2202,9 @@ static int zend_jit_math_long_double(dasm_State **Dst, |.endif if (Z_MODE(res_addr) == IS_MEM_ZVAL) { - | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE + if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_DOUBLE) { + | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE + } } return 1; @@ -2201,7 +2216,8 @@ static int zend_jit_math_double_long(dasm_State **Dst, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, - zend_jit_addr res_addr) + zend_jit_addr res_addr, + uint32_t res_use_info) { |.if SSE zend_reg result_reg; @@ -2242,7 +2258,9 @@ static int zend_jit_math_double_long(dasm_State **Dst, if (Z_MODE(res_addr) == IS_MEM_ZVAL) { if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { - | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE + if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_LONG) { + | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE + } } } @@ -2255,7 +2273,8 @@ static int zend_jit_math_double_double(dasm_State **Dst, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, - zend_jit_addr res_addr) + zend_jit_addr res_addr, + uint32_t res_use_info) { zend_bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); @@ -2285,7 +2304,9 @@ static int zend_jit_math_double_double(dasm_State **Dst, if (Z_MODE(res_addr) == IS_MEM_ZVAL) { if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { - | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE + if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_LONG) { + | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE + } } } @@ -2372,6 +2393,7 @@ static int zend_jit_math_helper(dasm_State **Dst, uint32_t op2_info, zend_jit_addr res_addr, uint32_t res_info, + uint32_t res_use_info, zend_bool separate_op1) /* Labels: 1,2,3,4,5,6 */ { @@ -2393,7 +2415,7 @@ static int zend_jit_math_helper(dasm_State **Dst, if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >6 } - if (!zend_jit_math_long_double(Dst, op_array, ssa, opline, op1_addr, op2_addr, res_addr)) { + if (!zend_jit_math_long_double(Dst, op_array, ssa, opline, op1_addr, op2_addr, res_addr, res_use_info)) { return 0; } | jmp >5 @@ -2402,7 +2424,7 @@ static int zend_jit_math_helper(dasm_State **Dst, | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6 } } - if (!zend_jit_math_long_long(Dst, op_array, ssa, opline, op1_addr, op2_addr, res_addr, res_info)) { + if (!zend_jit_math_long_long(Dst, op_array, ssa, opline, op1_addr, op2_addr, res_addr, res_info, res_use_info)) { return 0; } if (op1_info & MAY_BE_DOUBLE) { @@ -2419,7 +2441,7 @@ static int zend_jit_math_helper(dasm_State **Dst, | IF_NOT_ZVAL_TYPE, op2_addr, IS_DOUBLE, >6 } } - if (!zend_jit_math_double_double(Dst, op_array, ssa, opline, op1_addr, op2_addr, res_addr)) { + if (!zend_jit_math_double_double(Dst, op_array, ssa, opline, op1_addr, op2_addr, res_addr, res_use_info)) { return 0; } | jmp >5 @@ -2429,7 +2451,7 @@ static int zend_jit_math_helper(dasm_State **Dst, if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6 } - if (!zend_jit_math_double_long(Dst, op_array, ssa, opline, op1_addr, op2_addr, res_addr)) { + if (!zend_jit_math_double_long(Dst, op_array, ssa, opline, op1_addr, op2_addr, res_addr, res_use_info)) { return 0; } | jmp >5 @@ -2450,7 +2472,7 @@ static int zend_jit_math_helper(dasm_State **Dst, | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >6 } } - if (!zend_jit_math_double_double(Dst, op_array, ssa, opline, op1_addr, op2_addr, res_addr)) { + if (!zend_jit_math_double_double(Dst, op_array, ssa, opline, op1_addr, op2_addr, res_addr, res_use_info)) { return 0; } } @@ -2462,7 +2484,7 @@ static int zend_jit_math_helper(dasm_State **Dst, if (op2_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) { | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >5 } - if (!zend_jit_math_double_long(Dst, op_array, ssa, opline, op1_addr, op2_addr, res_addr)) { + if (!zend_jit_math_double_long(Dst, op_array, ssa, opline, op1_addr, op2_addr, res_addr, res_use_info)) { return 0; } if (op2_info & MAY_BE_DOUBLE) { @@ -2484,7 +2506,7 @@ static int zend_jit_math_helper(dasm_State **Dst, | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >6 } } - if (!zend_jit_math_double_double(Dst, op_array, ssa, opline, op1_addr, op2_addr, res_addr)) { + if (!zend_jit_math_double_double(Dst, op_array, ssa, opline, op1_addr, op2_addr, res_addr, res_use_info)) { return 0; } } @@ -2496,7 +2518,7 @@ static int zend_jit_math_helper(dasm_State **Dst, if (op1_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) { | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6 } - if (!zend_jit_math_long_double(Dst, op_array, ssa, opline, op1_addr, op2_addr, res_addr)) { + if (!zend_jit_math_long_double(Dst, op_array, ssa, opline, op1_addr, op2_addr, res_addr, res_use_info)) { return 0; } if (op1_info & MAY_BE_DOUBLE) { @@ -2556,7 +2578,7 @@ static int zend_jit_math_helper(dasm_State **Dst, static int zend_jit_math(dasm_State **Dst, const zend_op *opline, int *opnum, zend_op_array *op_array, zend_ssa *ssa, zend_lifetime_interval **ra) { - uint32_t op1_info, op2_info; + uint32_t op1_info, op2_info, res_use_info; zend_jit_addr op1_addr, op2_addr, res_addr; if (!ssa->ops || !ssa->var_info) { @@ -2565,6 +2587,7 @@ static int zend_jit_math(dasm_State **Dst, const zend_op *opline, int *opnum, ze op1_info = OP1_INFO(); op2_info = OP2_INFO(); + res_use_info = RES_USE_INFO(); if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) { goto fallback; @@ -2590,11 +2613,12 @@ static int zend_jit_math(dasm_State **Dst, const zend_op *opline, int *opnum, ze | mov RX, EX->call } res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var); + res_use_info = -1; } else { res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, ra, ssa->ops[opline - op_array->opcodes].result_def); } - return zend_jit_math_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, res_addr, RES_INFO(), 0); + return zend_jit_math_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, res_addr, RES_INFO(), res_use_info, 0); fallback: /* fallback to subroutine threading */ @@ -3137,9 +3161,9 @@ static int zend_jit_simple_assign(dasm_State **Dst, | // ZVAL_COPY_VALUE(return_value, &ref->value); ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R2, 8); if (!res_addr) { - | ZVAL_COPY_VALUE var_addr, ref_addr, val_info, ZREG_R2, ZREG_R0 + | ZVAL_COPY_VALUE var_addr, var_info, ref_addr, val_info, ZREG_R2, ZREG_R0 } else { - | ZVAL_COPY_VALUE_2 var_addr, res_addr, ref_addr, val_info, ZREG_R2, ZREG_R0 + | ZVAL_COPY_VALUE_2 var_addr, var_info, res_addr, ref_addr, val_info, ZREG_R2, ZREG_R0 } | je >2 | IF_NOT_REFCOUNTED dh, >3 @@ -3166,9 +3190,9 @@ static int zend_jit_simple_assign(dasm_State **Dst, } if (!res_addr) { - | ZVAL_COPY_VALUE var_addr, val_addr, val_info, ZREG_R2, ZREG_R0 + | ZVAL_COPY_VALUE var_addr, var_info, val_addr, val_info, ZREG_R2, ZREG_R0 } else { - | ZVAL_COPY_VALUE_2 var_addr, res_addr, val_addr, val_info, ZREG_R2, ZREG_R0 + | ZVAL_COPY_VALUE_2 var_addr, var_info, res_addr, val_addr, val_info, ZREG_R2, ZREG_R0 } if (val_type == IS_CV) { @@ -3655,7 +3679,7 @@ static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, zend_ case ZEND_ASSIGN_SUB: case ZEND_ASSIGN_MUL: case ZEND_ASSIGN_DIV: - if (!zend_jit_math_helper(Dst, opline, op_array, ssa, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, op3_addr, OP1_DATA_INFO(), var_addr, OP1_DEF_INFO(), 1)) { + if (!zend_jit_math_helper(Dst, opline, op_array, ssa, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, op3_addr, OP1_DATA_INFO(), var_addr, OP1_DEF_INFO(), var_info, 1)) { return 0; } break; @@ -3845,7 +3869,7 @@ static int zend_jit_assign_op(dasm_State **Dst, const zend_op *opline, zend_op_a case ZEND_ASSIGN_SUB: case ZEND_ASSIGN_MUL: case ZEND_ASSIGN_DIV: - return zend_jit_math_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, op1_addr, OP1_DEF_INFO(), 1); + return zend_jit_math_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, op1_addr, OP1_DEF_INFO(), op1_info, 1); case ZEND_ASSIGN_CONCAT: return zend_jit_concat_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, op1_addr, OP1_DEF_INFO()); default: @@ -6309,7 +6333,7 @@ static int zend_jit_send_val(dasm_State **Dst, const zend_op *opline, zend_op_ar | ADDREF_CONST zv, r0 } } else { - | ZVAL_COPY_VALUE arg_addr, op1_addr, op1_info, ZREG_R0, ZREG_R2 + | ZVAL_COPY_VALUE arg_addr, -1, op1_addr, op1_info, ZREG_R0, ZREG_R2 } return 1; @@ -6413,11 +6437,11 @@ static int zend_jit_send_ref(dasm_State **Dst, const zend_op *opline, zend_op_ar zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R1, 0); | mov r1, aword [r4] // restore - | ZVAL_COPY_VALUE ref_addr, val_addr, op1_info, ZREG_R2, ZREG_R2 + | ZVAL_COPY_VALUE ref_addr, -1, val_addr, op1_info, ZREG_R2, ZREG_R2 | SET_ZVAL_PTR val_addr, r0 | SET_ZVAL_TYPE_INFO val_addr, IS_REFERENCE_EX } else { - | ZVAL_COPY_VALUE ref_addr, op1_addr, op1_info, ZREG_R1, ZREG_R2 + | ZVAL_COPY_VALUE ref_addr, -1, op1_addr, op1_info, ZREG_R1, ZREG_R2 | SET_ZVAL_PTR op1_addr, r0 | SET_ZVAL_TYPE_INFO op1_addr, IS_REFERENCE_EX } @@ -6481,7 +6505,7 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, zend_op_ar } else if (opline->opcode == ZEND_SEND_VAR_NO_REF_EX) { mask = ZEND_SEND_PREFER_REF << ((arg_num + 3) * 2); - | ZVAL_COPY_VALUE arg_addr, op1_addr, op1_info, ZREG_R1, ZREG_R2 + | ZVAL_COPY_VALUE arg_addr, -1, op1_addr, op1_info, ZREG_R1, ZREG_R2 if (op1_info & MAY_BE_REF) { | cmp cl, IS_REFERENCE | je >7 @@ -6530,7 +6554,7 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, zend_op_ar } if (opline->opcode == ZEND_SEND_VAR_NO_REF) { - | ZVAL_COPY_VALUE arg_addr, op1_addr, op1_info, ZREG_R1, ZREG_R2 + | ZVAL_COPY_VALUE arg_addr, -1, op1_addr, op1_info, ZREG_R1, ZREG_R2 if (op1_info & MAY_BE_REF) { | cmp cl, IS_REFERENCE | je >7 @@ -6557,7 +6581,7 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, zend_op_ar | LOAD_ZVAL_ADDR FCARG1a, op1_addr | ZVAL_DEREF FCARG1a, op1_info - | ZVAL_COPY_VALUE arg_addr, val_addr, op1_info, ZREG_R0, ZREG_R2 + | ZVAL_COPY_VALUE arg_addr, -1, val_addr, op1_info, ZREG_R0, ZREG_R2 | TRY_ADDREF op1_info, ah, r2 } else { zend_jit_addr ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 8); @@ -6568,7 +6592,7 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, zend_op_ar | // zend_refcounted *ref = Z_COUNTED_P(retval_ptr); | GET_ZVAL_PTR FCARG1a, op1_addr | // ZVAL_COPY_VALUE(return_value, &ref->value); - | ZVAL_COPY_VALUE arg_addr, ref_addr, op1_info, ZREG_R0, ZREG_R2 + | ZVAL_COPY_VALUE arg_addr, -1, ref_addr, op1_info, ZREG_R0, ZREG_R2 | GC_DELREF FCARG1a | je >1 | IF_NOT_REFCOUNTED ah, >2 @@ -6578,7 +6602,7 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, zend_op_ar | EFREE_REG_24 op_array, opline | jmp >2 |.code - | ZVAL_COPY_VALUE arg_addr, op1_addr, op1_info, ZREG_R0, ZREG_R2 + | ZVAL_COPY_VALUE arg_addr, -1, op1_addr, op1_info, ZREG_R0, ZREG_R2 |2: } } else { @@ -6586,13 +6610,13 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, zend_op_ar zend_jit_addr op1_def_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].op1_def : -1); if (op1_addr != op1_def_addr) { - | ZVAL_COPY_VALUE op1_def_addr, op1_addr, MAY_BE_LONG, ZREG_R0, ZREG_R1 + | ZVAL_COPY_VALUE op1_def_addr, op1_info, op1_addr, op1_info, ZREG_R0, ZREG_R1 if (Z_MODE(op1_def_addr) == IS_REG && Z_MODE(op1_addr) != IS_REG) { op1_addr= op1_def_addr; } } } - | ZVAL_COPY_VALUE arg_addr, op1_addr, op1_info, ZREG_R0, ZREG_R2 + | ZVAL_COPY_VALUE arg_addr, -1, op1_addr, op1_info, ZREG_R0, ZREG_R2 if (opline->op1_type == IS_CV) { | TRY_ADDREF op1_info, ah, r2 } @@ -7204,14 +7228,14 @@ static int zend_jit_return(dasm_State **Dst, const zend_op *opline, zend_op_arra | ADDREF_CONST zv, r0 } } else if (opline->op1_type == IS_TMP_VAR) { - | ZVAL_COPY_VALUE ret_addr, op1_addr, op1_info, ZREG_R0, ZREG_R2 + | ZVAL_COPY_VALUE ret_addr, -1, op1_addr, op1_info, ZREG_R0, ZREG_R2 } else if (opline->op1_type == IS_CV) { if (op1_info & MAY_BE_REF) { | LOAD_ZVAL_ADDR r0, op1_addr | ZVAL_DEREF r0, op1_info op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); } - | ZVAL_COPY_VALUE ret_addr, op1_addr, op1_info, ZREG_R0, ZREG_R2 + | ZVAL_COPY_VALUE ret_addr, -1, op1_addr, op1_info, ZREG_R0, ZREG_R2 | // TODO: JIT: if (EXPECTED(!(EX_CALL_INFO() & ZEND_CALL_CODE))) ZVAL_NULL(retval_ptr); ??? | TRY_ADDREF op1_info, ah, r2 } else { @@ -7224,7 +7248,7 @@ static int zend_jit_return(dasm_State **Dst, const zend_op *opline, zend_op_arra | // zend_refcounted *ref = Z_COUNTED_P(retval_ptr); | GET_ZVAL_PTR r0, op1_addr | // ZVAL_COPY_VALUE(return_value, &ref->value); - | ZVAL_COPY_VALUE ret_addr, ref_addr, op1_info, ZREG_R2, ZREG_R2 + | ZVAL_COPY_VALUE ret_addr, -1, ref_addr, op1_info, ZREG_R2, ZREG_R2 | GC_DELREF r0 | je >2 | // if (IS_REFCOUNTED()) @@ -7250,7 +7274,7 @@ static int zend_jit_return(dasm_State **Dst, const zend_op *opline, zend_op_arra } |.code } - | ZVAL_COPY_VALUE ret_addr, op1_addr, op1_info, ZREG_R0, ZREG_R2 + | ZVAL_COPY_VALUE ret_addr, -1, op1_addr, op1_info, ZREG_R0, ZREG_R2 } |9: @@ -7403,10 +7427,10 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, const zend_op *opline, zend |1: | GC_ADDREF r1 |2: - | ZVAL_COPY_VALUE res_addr, val_addr, MAY_BE_ANY, ZREG_R1, ZREG_R2 + | ZVAL_COPY_VALUE res_addr, -1, val_addr, MAY_BE_ANY, ZREG_R1, ZREG_R2 } else { | // ZVAL_COPY - | ZVAL_COPY_VALUE res_addr, val_addr, MAY_BE_ANY, ZREG_R1, ZREG_R2 + | ZVAL_COPY_VALUE res_addr, -1, val_addr, MAY_BE_ANY, ZREG_R1, ZREG_R2 | TRY_ADDREF res_info, ch, r2 } } From 1f165843ac07c3a719cade058034efed8c7ff805 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 30 Mar 2017 22:16:50 +0300 Subject: [PATCH 402/569] Ignore "no_val" dependencies --- ext/opcache/jit/zend_jit.c | 14 +++++++++++--- ext/opcache/jit/zend_jit_x86.dasc | 19 ++++++++++++++----- 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 2faccabadd31c..8d21a6f464d4a 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -84,9 +84,17 @@ static int zend_may_overflow(const zend_op *opline, zend_op_array *op_array, zen static zend_bool zend_ssa_is_last_use(const zend_ssa *ssa, int var, int use) { - return - !ssa->vars[var].phi_use_chain && - zend_ssa_next_use(ssa->ops, var, use) < 0; + if (ssa->vars[var].phi_use_chain) { + zend_ssa_phi *phi = ssa->vars[var].phi_use_chain; + do { + if (!ssa->vars[phi->ssa_var].no_val) { + return 0; + } + phi = zend_ssa_next_use_phi(ssa, var, phi); + } while (phi); + } + + return zend_ssa_next_use(ssa->ops, var, use) < 0; } #include "dynasm/dasm_x86.h" diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 6f133b1e1057d..da54162ddc5eb 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -8227,16 +8227,25 @@ static zend_bool zend_jit_opline_supports_reg(zend_op_array *op_array, zend_ssa static zend_bool zend_jit_may_be_in_reg(zend_op_array *op_array, zend_ssa *ssa, int var) { - // TODO: enable global register allocation ??? - // SSA resolution is not implemented yet. - if (ssa->vars[var].definition_phi || ssa->vars[var].phi_use_chain) { + if (ssa->vars[var].no_val) { + /* we don't need the value */ return 0; } - if (ssa->vars[var].no_val) { - /* we don't need the value */ + // TODO: enable global register allocation ??? + // SSA resolution is not implemented yet. + if (ssa->vars[var].definition_phi) { return 0; } + if (ssa->vars[var].phi_use_chain) { + zend_ssa_phi *phi = ssa->vars[var].phi_use_chain; + do { + if (!ssa->vars[phi->ssa_var].no_val) { + return 0; + } + phi = zend_ssa_next_use_phi(ssa, var, phi); + } while (phi); + } if (((ssa->var_info[var].type & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_DOUBLE) && ((ssa->var_info[var].type & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_LONG)) { From 3d8532dbe321f22befd11d34f5f697a866c6f2c0 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 30 Mar 2017 22:34:47 +0300 Subject: [PATCH 403/569] typo --- ext/opcache/jit/zend_jit_x86.dasc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index da54162ddc5eb..897a281d85f16 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -2258,7 +2258,7 @@ static int zend_jit_math_double_long(dasm_State **Dst, if (Z_MODE(res_addr) == IS_MEM_ZVAL) { if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { - if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_LONG) { + if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_DOUBLE) { | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE } } @@ -2304,7 +2304,7 @@ static int zend_jit_math_double_double(dasm_State **Dst, if (Z_MODE(res_addr) == IS_MEM_ZVAL) { if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { - if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_LONG) { + if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_DOUBLE) { | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE } } From 9577dca9ef4d02567294008bb0451b4e141f7ee7 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 30 Mar 2017 23:30:49 +0300 Subject: [PATCH 404/569] Optimistic register allocation for ADD/SUB/MUL and spilling/reloaing on slow path --- ext/opcache/jit/zend_jit_x86.dasc | 83 ++++++++++++++++++++++++++----- 1 file changed, 71 insertions(+), 12 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 897a281d85f16..207558e4c6412 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -1953,6 +1953,46 @@ static int zend_jit_new(dasm_State **Dst, const zend_op *opline, int *opnum, zen return 1; } +static int zend_jit_spill(dasm_State **Dst, zend_jit_addr src, zend_jit_addr dst, uint32_t info) +{ + ZEND_ASSERT(Z_MODE(src) == IS_REG); + ZEND_ASSERT(Z_MODE(dst) == IS_MEM_ZVAL); + + if ((info & MAY_BE_ANY) == MAY_BE_LONG) { + | SET_ZVAL_LVAL dst, Ra(Z_REG(src)) + | SET_ZVAL_TYPE_INFO dst, IS_LONG + } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) { + | .if SSE + | SSE_SET_ZVAL_DVAL dst, Z_REG(src) + | .else + || ZEND_ASSERT(0); + | .endif + | SET_ZVAL_TYPE_INFO dst, IS_DOUBLE + } else { + ZEND_ASSERT(0); + } + return 1; +} + +static int zend_jit_load_reg(dasm_State **Dst, zend_jit_addr src, zend_jit_addr dst, uint32_t info) +{ + ZEND_ASSERT(Z_MODE(src) == IS_MEM_ZVAL); + ZEND_ASSERT(Z_MODE(dst) == IS_REG); + + if ((info & MAY_BE_ANY) == MAY_BE_LONG) { + | GET_ZVAL_LVAL Z_REG(dst), src + } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) { + | .if SSE + | SSE_GET_ZVAL_DVAL Z_REG(dst), src + | .else + || ZEND_ASSERT(0); + | .endif + } else { + ZEND_ASSERT(0); + } + return 1; +} + static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, zend_lifetime_interval **ra) { uint32_t op1_info, op1_def_info, res_use_info; @@ -2391,6 +2431,7 @@ static int zend_jit_math_helper(dasm_State **Dst, znode_op op2, zend_jit_addr op2_addr, uint32_t op2_info, + uint32_t res_var, zend_jit_addr res_addr, uint32_t res_info, uint32_t res_use_info, @@ -2541,10 +2582,27 @@ static int zend_jit_math_helper(dasm_State **Dst, if (separate_op1) { | SEPARATE_ZVAL_NOREF op1_addr, op1_info, 0, op_array->filename, opline->lineno } - if (Z_REG(res_addr) != ZREG_FCARG1a || Z_OFFSET(res_addr) != 0) { + if (Z_MODE(res_addr) == IS_REG) { + zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); + | LOAD_ZVAL_ADDR FCARG1a, real_addr + } else if (Z_REG(res_addr) != ZREG_FCARG1a || Z_OFFSET(res_addr) != 0) { | LOAD_ZVAL_ADDR FCARG1a, res_addr } + if (Z_MODE(op1_addr) == IS_REG) { + zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var); + if (!zend_jit_spill(Dst, op1_addr, real_addr, op1_info)) { + return 0; + } + op1_addr = real_addr; + } | LOAD_ZVAL_ADDR FCARG2a, op1_addr + if (Z_MODE(op2_addr) == IS_REG) { + zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op2.var); + if (!zend_jit_spill(Dst, op2_addr, real_addr, op2_info)) { + return 0; + } + op2_addr = real_addr; + } |.if X64 | LOAD_ZVAL_ADDR CARG3, op2_addr |.else @@ -2566,6 +2624,12 @@ static int zend_jit_math_helper(dasm_State **Dst, if (zend_may_throw(opline, op_array, ssa)) { zend_jit_check_exception(Dst); } + if (Z_MODE(res_addr) == IS_REG) { + zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); + if (!zend_jit_load_reg(Dst, real_addr, res_addr, res_info)) { + return 0; + } + } if ((op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { | jmp <5 @@ -2618,7 +2682,7 @@ static int zend_jit_math(dasm_State **Dst, const zend_op *opline, int *opnum, ze res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, ra, ssa->ops[opline - op_array->opcodes].result_def); } - return zend_jit_math_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, res_addr, RES_INFO(), res_use_info, 0); + return zend_jit_math_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, opline->result.var, res_addr, RES_INFO(), res_use_info, 0); fallback: /* fallback to subroutine threading */ @@ -3679,7 +3743,7 @@ static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, zend_ case ZEND_ASSIGN_SUB: case ZEND_ASSIGN_MUL: case ZEND_ASSIGN_DIV: - if (!zend_jit_math_helper(Dst, opline, op_array, ssa, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, op3_addr, OP1_DATA_INFO(), var_addr, OP1_DEF_INFO(), var_info, 1)) { + if (!zend_jit_math_helper(Dst, opline, op_array, ssa, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, op3_addr, OP1_DATA_INFO(), 0, var_addr, OP1_DEF_INFO(), var_info, 1)) { return 0; } break; @@ -3869,7 +3933,7 @@ static int zend_jit_assign_op(dasm_State **Dst, const zend_op *opline, zend_op_a case ZEND_ASSIGN_SUB: case ZEND_ASSIGN_MUL: case ZEND_ASSIGN_DIV: - return zend_jit_math_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, op1_addr, OP1_DEF_INFO(), op1_info, 1); + return zend_jit_math_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, opline->op1.var, op1_addr, OP1_DEF_INFO(), op1_info, 1); case ZEND_ASSIGN_CONCAT: return zend_jit_concat_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, op1_addr, OP1_DEF_INFO()); default: @@ -8176,6 +8240,9 @@ static zend_bool zend_jit_opline_supports_reg(zend_op_array *op_array, zend_ssa case ZEND_SEND_VAR: case ZEND_SEND_VAL: case ZEND_SEND_VAL_EX: + case ZEND_ADD: + case ZEND_SUB: + case ZEND_MUL: return 1; case ZEND_ASSIGN: op1_info = OP1_INFO(); @@ -8192,14 +8259,6 @@ static zend_bool zend_jit_opline_supports_reg(zend_op_array *op_array, zend_ssa return opline->op1_type == IS_CV && !(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG))); - case ZEND_ADD: - case ZEND_SUB: - case ZEND_MUL: - op1_info = OP1_INFO(); - op2_info = OP2_INFO(); - return - !(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) && - !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))); case ZEND_IS_SMALLER: case ZEND_IS_SMALLER_OR_EQUAL: case ZEND_IS_EQUAL: From c70457acd2e4b6d82b65cdd5c357632e12ff36c8 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 30 Mar 2017 23:44:20 +0300 Subject: [PATCH 405/569] ptimistic register allocation for comparison opcodes and spilling/reloaing on slow path --- ext/opcache/jit/zend_jit_x86.dasc | 33 ++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 207558e4c6412..27aff5a8c3470 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -4757,6 +4757,20 @@ static int zend_jit_cmp(dasm_State **Dst, const zend_op *opline, int b, int *opn |9: } | SAVE_VALID_OPLINE opline + if (Z_MODE(op1_addr) == IS_REG) { + zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); + if (!zend_jit_spill(Dst, op1_addr, real_addr, op1_info)) { + return 0; + } + op1_addr = real_addr; + } + if (Z_MODE(op2_addr) == IS_REG) { + zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var); + if (!zend_jit_spill(Dst, op2_addr, real_addr, op2_info)) { + return 0; + } + op2_addr = real_addr; + } | LOAD_ZVAL_ADDR FCARG2a, op1_addr if (opline->op1_type == IS_CV && (op1_info & MAY_BE_UNDEF)) { | IF_NOT_Z_TYPE FCARG2a, IS_UNDEF, >1 @@ -8243,6 +8257,13 @@ static zend_bool zend_jit_opline_supports_reg(zend_op_array *op_array, zend_ssa case ZEND_ADD: case ZEND_SUB: case ZEND_MUL: + case ZEND_IS_SMALLER: + case ZEND_IS_SMALLER_OR_EQUAL: + case ZEND_IS_EQUAL: + case ZEND_IS_NOT_EQUAL: + case ZEND_IS_IDENTICAL: + case ZEND_IS_NOT_IDENTICAL: + case ZEND_CASE: return 1; case ZEND_ASSIGN: op1_info = OP1_INFO(); @@ -8259,17 +8280,6 @@ static zend_bool zend_jit_opline_supports_reg(zend_op_array *op_array, zend_ssa return opline->op1_type == IS_CV && !(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG))); - case ZEND_IS_SMALLER: - case ZEND_IS_SMALLER_OR_EQUAL: - case ZEND_IS_EQUAL: - case ZEND_IS_NOT_EQUAL: - case ZEND_IS_IDENTICAL: - case ZEND_IS_NOT_IDENTICAL: - op1_info = OP1_INFO(); - op2_info = OP2_INFO(); - return - !(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) && - !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))); case ZEND_BOOL: case ZEND_BOOL_NOT: case ZEND_JMPZ: @@ -8411,6 +8421,7 @@ static zend_regset zend_jit_get_scratch_regset(zend_op_array *op_array, zend_ssa case ZEND_IS_NOT_EQUAL: case ZEND_IS_IDENTICAL: case ZEND_IS_NOT_IDENTICAL: + case ZEND_CASE: op1_info = OP1_INFO(); op2_info = OP1_INFO(); if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) && From 9831a5b6644251e3adc28f19be546cf02a5e6a75 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 31 Mar 2017 15:20:21 +0300 Subject: [PATCH 406/569] Unify spilling store/load --- ext/opcache/jit/zend_jit_x86.dasc | 122 ++++++++++++++++-------------- 1 file changed, 64 insertions(+), 58 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 27aff5a8c3470..63bfd2037eb7c 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -1953,7 +1953,7 @@ static int zend_jit_new(dasm_State **Dst, const zend_op *opline, int *opnum, zen return 1; } -static int zend_jit_spill(dasm_State **Dst, zend_jit_addr src, zend_jit_addr dst, uint32_t info) +static int zend_jit_spill_store(dasm_State **Dst, zend_jit_addr src, zend_jit_addr dst, uint32_t info) { ZEND_ASSERT(Z_MODE(src) == IS_REG); ZEND_ASSERT(Z_MODE(dst) == IS_MEM_ZVAL); @@ -1993,6 +1993,44 @@ static int zend_jit_load_reg(dasm_State **Dst, zend_jit_addr src, zend_jit_addr return 1; } +static int zend_jit_update_regs(dasm_State **Dst, zend_jit_addr src, zend_jit_addr dst, uint32_t info) +{ + if (src != dst) { + if (Z_MODE(src) == IS_REG) { + if (Z_MODE(dst) == IS_REG) { + if ((info & MAY_BE_ANY) == MAY_BE_LONG) { + | mov Ra(Z_REG(dst)), Ra(Z_REG(src)) + } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) { + | .if SSE + | movsd xmm(Z_REG(dst)-ZREG_XMM0), xmm(Z_REG(src)-ZREG_XMM0) + | .else + || ZEND_ASSERT(0); + | .endif + } else { + ZEND_ASSERT(0); + } + } else if (Z_MODE(dst) == IS_MEM_ZVAL) { + if (!zend_jit_spill_store(Dst, src, dst, info)) { + return 0; + } + } else { + ZEND_ASSERT(0); + } + } else if (Z_MODE(src) == IS_MEM_ZVAL) { + if (Z_MODE(dst) == IS_REG) { + if (!zend_jit_load_reg(Dst, src, dst, info)) { + return 0; + } + } else { + ZEND_ASSERT(0); + } + } else { + ZEND_ASSERT(0); + } + } + return 1; +} + static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, zend_lifetime_interval **ra) { uint32_t op1_info, op1_def_info, res_use_info; @@ -2590,7 +2628,7 @@ static int zend_jit_math_helper(dasm_State **Dst, } if (Z_MODE(op1_addr) == IS_REG) { zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var); - if (!zend_jit_spill(Dst, op1_addr, real_addr, op1_info)) { + if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info)) { return 0; } op1_addr = real_addr; @@ -2598,7 +2636,7 @@ static int zend_jit_math_helper(dasm_State **Dst, | LOAD_ZVAL_ADDR FCARG2a, op1_addr if (Z_MODE(op2_addr) == IS_REG) { zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op2.var); - if (!zend_jit_spill(Dst, op2_addr, real_addr, op2_info)) { + if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info)) { return 0; } op2_addr = real_addr; @@ -4759,14 +4797,14 @@ static int zend_jit_cmp(dasm_State **Dst, const zend_op *opline, int b, int *opn | SAVE_VALID_OPLINE opline if (Z_MODE(op1_addr) == IS_REG) { zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); - if (!zend_jit_spill(Dst, op1_addr, real_addr, op1_info)) { + if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info)) { return 0; } op1_addr = real_addr; } if (Z_MODE(op2_addr) == IS_REG) { zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var); - if (!zend_jit_spill(Dst, op2_addr, real_addr, op2_info)) { + if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info)) { return 0; } op2_addr = real_addr; @@ -5533,44 +5571,6 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, int b, return 1; } -static int zend_jit_update_regs(dasm_State **Dst, uint32_t info, zend_jit_addr src_addr, zend_jit_addr dst_addr) -{ - if (Z_MODE(src_addr) == IS_REG) { - if ((info & MAY_BE_ANY) == MAY_BE_LONG) { - if (Z_MODE(dst_addr) != IS_REG || Z_REG(src_addr) != Z_REG(dst_addr)) { - | SET_ZVAL_LVAL dst_addr, Ra(Z_REG(src_addr)) - } - if (Z_MODE(dst_addr) == IS_MEM_ZVAL) { - | SET_ZVAL_TYPE_INFO dst_addr, IS_LONG - } - } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) { - |.if SSE - | SSE_SET_ZVAL_DVAL dst_addr, Z_REG(src_addr) - |.else - || ZEND_ASSERT(0); - |.endif - if (Z_MODE(dst_addr) == IS_MEM_ZVAL) { - | SET_ZVAL_TYPE_INFO dst_addr, IS_DOUBLE - } - } else { - ZEND_ASSERT(0); - } - } else if (Z_MODE(dst_addr) == IS_REG) { - if ((info & MAY_BE_ANY) == MAY_BE_LONG) { - | GET_ZVAL_LVAL Z_REG(dst_addr), src_addr - } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) { - |.if SSE - | SSE_GET_ZVAL_DVAL Z_REG(dst_addr), src_addr - |.else - || ZEND_ASSERT(0); - |.endif - } else { - ZEND_ASSERT(0); - } - } - return 1; -} - static int zend_jit_qm_assign(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, zend_lifetime_interval **ra) { uint32_t op1_info = OP1_INFO(); @@ -5581,10 +5581,12 @@ static int zend_jit_qm_assign(dasm_State **Dst, const zend_op *opline, zend_op_a if (ra && ssa->ops[opline - op_array->opcodes].op1_def >= 0 - && !ssa->vars[ssa->ops[opline - op_array->opcodes].op1_def].no_val - && !zend_jit_update_regs(Dst, op1_info, op1_addr, - zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, ra, ssa->ops[opline - op_array->opcodes].op1_def))) { - return 0; + && !ssa->vars[ssa->ops[opline - op_array->opcodes].op1_def].no_val) { + zend_jit_addr op1_def_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, ra, ssa->ops[opline - op_array->opcodes].op1_def); + + if (!zend_jit_update_regs(Dst, op1_addr, op1_def_addr, op1_info)) { + return 0; + } } return zend_jit_simple_assign(Dst, opline, op_array, ssa, res_addr, -1, opline->op1_type, opline->op1, op1_addr, op1_info, 0, 0); @@ -5612,10 +5614,12 @@ static int zend_jit_assign(dasm_State **Dst, const zend_op *opline, zend_op_arra if (ra && ssa->ops[opline - op_array->opcodes].op2_def >= 0 - && !ssa->vars[ssa->ops[opline - op_array->opcodes].op2_def].no_val - && !zend_jit_update_regs(Dst, op2_info, op2_addr, - zend_jit_decode_op(op_array, opline->op2_type, opline->op2, opline, ra, ssa->ops[opline - op_array->opcodes].op2_def))) { - return 0; + && !ssa->vars[ssa->ops[opline - op_array->opcodes].op2_def].no_val) { + zend_jit_addr op2_def_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2, opline, ra, ssa->ops[opline - op_array->opcodes].op2_def); + + if (!zend_jit_update_regs(Dst, op2_addr, op2_def_addr, op2_info)) { + return 0; + } } if (op1_info & MAY_BE_REF) { @@ -6684,14 +6688,16 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, zend_op_ar |2: } } else { - if (ra && ssa->ops[opline - op_array->opcodes].op1_def >= 0) { - zend_jit_addr op1_def_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].op1_def : -1); + if (ra + && ssa->ops[opline - op_array->opcodes].op1_def >= 0 + && !ssa->vars[ssa->ops[opline - op_array->opcodes].op1_def].no_val) { + zend_jit_addr op1_def_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, ra, ssa->ops[opline - op_array->opcodes].op1_def); - if (op1_addr != op1_def_addr) { - | ZVAL_COPY_VALUE op1_def_addr, op1_info, op1_addr, op1_info, ZREG_R0, ZREG_R1 - if (Z_MODE(op1_def_addr) == IS_REG && Z_MODE(op1_addr) != IS_REG) { - op1_addr= op1_def_addr; - } + if (!zend_jit_update_regs(Dst, op1_addr, op1_def_addr, op1_info)) { + return 0; + } + if (Z_MODE(op1_def_addr) == IS_REG && Z_MODE(op1_addr) != IS_REG) { + op1_addr= op1_def_addr; } } | ZVAL_COPY_VALUE arg_addr, -1, op1_addr, op1_info, ZREG_R0, ZREG_R2 From 87f1ebd557b3110af5dd62457c20d08135c600ee Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 31 Mar 2017 15:26:54 +0300 Subject: [PATCH 407/569] Use opcache.jit_debug (mask 0x4) INI directive to enable/disable register allocation debug logs --- ext/opcache/jit/zend_jit.c | 55 ++++++++++++++++++-------------------- ext/opcache/jit/zend_jit.h | 1 + 2 files changed, 27 insertions(+), 29 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 8d21a6f464d4a..805ea9324036c 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -35,7 +35,6 @@ #include "Optimizer/zend_dump.h" //#define REG_ALLOC -//#define DEBUG_REG_ALLOC //#define CONTEXT_THREADED_JIT #define PREFER_MAP_32BIT //#define ZEND_JIT_RECORD @@ -1180,7 +1179,6 @@ static zend_lifetime_interval *zend_jit_sort_intervals(zend_lifetime_interval ** return list; } -#ifdef DEBUG_REG_ALLOC static void zend_jit_print_regset(zend_regset regset) { zend_reg reg; @@ -1197,7 +1195,6 @@ static void zend_jit_print_regset(zend_regset regset) } } } -#endif static int *zend_jit_compute_block_order_int(zend_ssa *ssa, int n, int *block_order) { @@ -1792,30 +1789,8 @@ static zend_lifetime_interval** zend_jit_allocate_registers(zend_op_array *op_ar } if (list) { -#ifdef DEBUG_REG_ALLOC - fprintf(stderr, "Live Ranges \"%s\"\n", op_array->function_name ? ZSTR_VAL(op_array->function_name) : "[main]"); - ival = list; - while (ival) { - zend_life_range *range; - - fprintf(stderr, "#%d: %u-%u", ival->ssa_var, ival->range.start, ival->range.end); - range = ival->range.next; - while (range) { - fprintf(stderr, ", %u-%u", range->start, range->end); - range = range->next; - } - fprintf(stderr, "\n"); - ival = ival->list_next; - } -#endif - - /* Linear Scan Register Allocation */ - list = zend_jit_linear_scan(op_array, ssa, list); - - if (list) { - -#ifdef DEBUG_REG_ALLOC - fprintf(stderr, "Allocated Live Ranges \"%s\"\n", op_array->function_name ? ZSTR_VAL(op_array->function_name) : "[main]"); + if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_REG_ALLOC) { + fprintf(stderr, "Live Ranges \"%s\"\n", op_array->function_name ? ZSTR_VAL(op_array->function_name) : "[main]"); ival = list; while (ival) { zend_life_range *range; @@ -1826,10 +1801,32 @@ static zend_lifetime_interval** zend_jit_allocate_registers(zend_op_array *op_ar fprintf(stderr, ", %u-%u", range->start, range->end); range = range->next; } - fprintf(stderr, " (%s) \n", zend_reg_name[ival->reg]); + fprintf(stderr, "\n"); ival = ival->list_next; } -#endif + } + + /* Linear Scan Register Allocation */ + list = zend_jit_linear_scan(op_array, ssa, list); + + if (list) { + + if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_REG_ALLOC) { + fprintf(stderr, "Allocated Live Ranges \"%s\"\n", op_array->function_name ? ZSTR_VAL(op_array->function_name) : "[main]"); + ival = list; + while (ival) { + zend_life_range *range; + + fprintf(stderr, "#%d: %u-%u", ival->ssa_var, ival->range.start, ival->range.end); + range = ival->range.next; + while (range) { + fprintf(stderr, ", %u-%u", range->start, range->end); + range = range->next; + } + fprintf(stderr, " (%s) \n", zend_reg_name[ival->reg]); + ival = ival->list_next; + } + } intervals = zend_arena_calloc(&CG(arena), ssa->vars_count, sizeof(zend_lifetime_interval*)); if (!intervals) { diff --git a/ext/opcache/jit/zend_jit.h b/ext/opcache/jit/zend_jit.h index 176a06732aa3e..7fbbbf5facd5f 100644 --- a/ext/opcache/jit/zend_jit.h +++ b/ext/opcache/jit/zend_jit.h @@ -55,6 +55,7 @@ #define ZEND_JIT_DEBUG_ASM (1<<0) #define ZEND_JIT_DEBUG_SSA (1<<1) +#define ZEND_JIT_DEBUG_REG_ALLOC (1<<2) #define ZEND_JIT_DEBUG_GDB (1<<4) #define ZEND_JIT_DEBUG_PERF (1<<5) From b6df9289f1f7c13ba1be0f23971c529d99cb8f99 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 31 Mar 2017 16:08:49 +0300 Subject: [PATCH 408/569] Use opcache.jit (deciaml mask 100) INI directive to enable/disable register allocation --- ext/opcache/jit/zend_jit.c | 30 ++++++++++++++---------------- ext/opcache/jit/zend_jit.h | 6 ++++++ ext/opcache/jit/zend_jit_x86.dasc | 2 -- 3 files changed, 20 insertions(+), 18 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 805ea9324036c..3ed9044719a81 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -34,7 +34,6 @@ #include "Optimizer/zend_call_graph.h" #include "Optimizer/zend_dump.h" -//#define REG_ALLOC //#define CONTEXT_THREADED_JIT #define PREFER_MAP_32BIT //#define ZEND_JIT_RECORD @@ -68,6 +67,7 @@ typedef struct _zend_jit_stub { static zend_uchar zend_jit_level = 0; static zend_uchar zend_jit_trigger = 0; +static zend_uchar zend_jit_reg_alloc = 0; zend_ulong zend_jit_profile_counter = 0; int zend_jit_profile_counter_rid = -1; @@ -125,6 +125,7 @@ ZEND_API void zend_jit_status(zval *ret) array_init(&stats); add_assoc_long(&stats, "level", zend_jit_level); add_assoc_long(&stats, "trigger", zend_jit_trigger); + add_assoc_long(&stats, "reg-alloc", zend_jit_reg_alloc); if (dasm_buf) { add_assoc_long(&stats, "buffer_size", (char*)dasm_end - (char*)dasm_buf); add_assoc_long(&stats, "buffer_free", (char*)dasm_end - (char*)*dasm_ptr); @@ -973,7 +974,6 @@ static int zend_jit_op_array_analyze2(zend_op_array *op_array, zend_script *scri return SUCCESS; } -#ifdef REG_ALLOC static int zend_jit_add_range(zend_lifetime_interval **intervals, int var, uint32_t from, uint32_t to) { zend_lifetime_interval *ival = intervals[var]; @@ -1852,7 +1852,6 @@ static zend_lifetime_interval** zend_jit_allocate_registers(zend_op_array *op_ar free_alloca(candidates, use_heap); return NULL; } -#endif static int zend_jit(zend_op_array *op_array, zend_ssa *ssa, const zend_op *rt_opline) { @@ -1861,9 +1860,7 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa, const zend_op *rt_op dasm_State* dasm_state = NULL; void *handler; int call_level = 0; -#ifdef REG_ALLOC - void *checkpoint; -#endif + void *checkpoint = NULL; zend_lifetime_interval **ra = NULL; #ifdef ZEND_JIT_FILTER @@ -1895,10 +1892,10 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa, const zend_op *rt_op pass: #endif -#ifdef REG_ALLOC - checkpoint = zend_arena_checkpoint(CG(arena)); - ra = zend_jit_allocate_registers(op_array, ssa); -#endif + if (zend_jit_reg_alloc) { + checkpoint = zend_arena_checkpoint(CG(arena)); + ra = zend_jit_allocate_registers(op_array, ssa); + } /* mark hidden branch targets */ for (b = 0; b < ssa->cfg.blocks_count; b++) { @@ -2345,18 +2342,18 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa, const zend_op *rt_op } dasm_free(&dasm_state); -#ifdef REG_ALLOC - zend_arena_release(&CG(arena), checkpoint); -#endif + if (zend_jit_reg_alloc) { + zend_arena_release(&CG(arena), checkpoint); + } return SUCCESS; jit_failure: if (dasm_state) { dasm_free(&dasm_state); } -#ifdef REG_ALLOC - zend_arena_release(&CG(arena), checkpoint); -#endif + if (zend_jit_reg_alloc) { + zend_arena_release(&CG(arena), checkpoint); + } return FAILURE; } @@ -2792,6 +2789,7 @@ ZEND_API int zend_jit_startup(zend_long jit, size_t size) zend_jit_level = ZEND_JIT_LEVEL(jit); zend_jit_trigger = ZEND_JIT_TRIGGER(jit); + zend_jit_reg_alloc = ZEND_JIT_REG_ALLOC(jit); if (zend_jit_trigger == ZEND_JIT_ON_PROF_REQUEST) { zend_extension dummy; diff --git a/ext/opcache/jit/zend_jit.h b/ext/opcache/jit/zend_jit.h index 7fbbbf5facd5f..7727b5197f8f8 100644 --- a/ext/opcache/jit/zend_jit.h +++ b/ext/opcache/jit/zend_jit.h @@ -38,8 +38,14 @@ #define ZEND_JIT_TRIGGER(n) (((n) / 10) % 10) +#define ZEND_JIT_REG_ALLOC_NONE 0 /* no register allocation */ +#define ZEND_JIT_REG_ALLOC_ENABLED 1 /* linear scan register allocation */ + +#define ZEND_JIT_REG_ALLOC(n) (((n) / 100) % 10) + #define ZEND_JIT_DEFAULT "5" + /* Makes profile based JIT (opcache.jit=2*) to generate code only for most * offten called functions (above the threshold). * TODO: this setting should be configurable diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 63bfd2037eb7c..a9903f2c8a215 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -8221,7 +8221,6 @@ static int zend_jit_free(dasm_State **Dst, const zend_op *opline, zend_op_array return 1; } -#ifdef REG_ALLOC static zend_bool zend_jit_may_reuse_reg(zend_op_array *op_array, zend_ssa *ssa, uint32_t position, int def_var, int use_var) { if (ssa->var_info[def_var].type != ssa->var_info[use_var].type) { @@ -8475,7 +8474,6 @@ static zend_regset zend_jit_get_scratch_regset(zend_op_array *op_array, zend_ssa return regset; } -#endif #ifdef ZEND_JIT_RECORD static int zend_jit_func_header(dasm_State **Dst, zend_op_array *op_array) From a8dd92e0e22a9e3f814cde196d93dd3f440d2594 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 3 Apr 2017 16:58:54 +0300 Subject: [PATCH 409/569] Take into account no_val usages of SSA variables during lifetime intervals construction and register allocation. More accureate scratch registers calculation. --- ext/opcache/jit/zend_jit.c | 21 +++++++++++++++------ ext/opcache/jit/zend_jit_x86.dasc | 12 +++++++----- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 3ed9044719a81..c8277f8b24c00 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -81,7 +81,7 @@ static void **dasm_ptr = NULL; static int zend_may_throw(const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa); static int zend_may_overflow(const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa); -static zend_bool zend_ssa_is_last_use(const zend_ssa *ssa, int var, int use) +static zend_bool zend_ssa_is_last_use(zend_op_array *op_array, const zend_ssa *ssa, int var, int use) { if (ssa->vars[var].phi_use_chain) { zend_ssa_phi *phi = ssa->vars[var].phi_use_chain; @@ -93,7 +93,8 @@ static zend_bool zend_ssa_is_last_use(const zend_ssa *ssa, int var, int use) } while (phi); } - return zend_ssa_next_use(ssa->ops, var, use) < 0; + use = zend_ssa_next_use(ssa->ops, var, use); + return use < 0 || zend_ssa_is_no_val_use(op_array->opcodes + use, ssa->ops + use, var); } #include "dynasm/dasm_x86.h" @@ -1347,12 +1348,14 @@ static int zend_jit_compute_liveness(zend_op_array *op_array, zend_ssa *ssa, zen /* for each operation op of b in reverse order */ for (n = b->start + b->len; n > b->start;) { zend_ssa_op *op; + const zend_op *opline; uint32_t num; n--; op = ssa->ops + n; + opline = op_array->opcodes + n; - if (UNEXPECTED(op_array->opcodes[n].opcode == ZEND_OP_DATA)) { + if (UNEXPECTED(opline->opcode == ZEND_OP_DATA)) { num = n - 1; } else { num = n; @@ -1383,19 +1386,25 @@ static int zend_jit_compute_liveness(zend_op_array *op_array, zend_ssa *ssa, zen /* for each input operand opd of op do */ /* live.add(opd) */ /* addRange(opd, b.from, op) */ - if (op->op1_use >= 0 && zend_bitset_in(candidates, op->op1_use)) { + if (op->op1_use >= 0 + && zend_bitset_in(candidates, op->op1_use) + && !zend_ssa_is_no_val_use(opline, op, op->op1_use)) { zend_bitset_incl(live, op->op1_use); if (zend_jit_add_range(intervals, op->op1_use, b->start, num) != SUCCESS) { goto failure; } } - if (op->op2_use >= 0 && zend_bitset_in(candidates, op->op2_use)) { + if (op->op2_use >= 0 + && zend_bitset_in(candidates, op->op2_use) + && !zend_ssa_is_no_val_use(opline, op, op->op2_use)) { zend_bitset_incl(live, op->op2_use); if (zend_jit_add_range(intervals, op->op2_use, b->start, num) != SUCCESS) { goto failure; } } - if (op->result_use >= 0 && zend_bitset_in(candidates, op->result_use)) { + if (op->result_use >= 0 + && zend_bitset_in(candidates, op->result_use) + && !zend_ssa_is_no_val_use(opline, op, op->result_use)) { zend_bitset_incl(live, op->result_use); if (zend_jit_add_range(intervals, op->result_use, b->start, num) != SUCCESS) { goto failure; diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index a9903f2c8a215..09de42c098a03 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -2182,7 +2182,7 @@ static int zend_jit_math_long_long(dasm_State **Dst, if (Z_MODE(res_addr) == IS_REG) { result_reg = Z_REG(res_addr); - } else if (Z_MODE(op1_addr) == IS_REG && zend_ssa_is_last_use(ssa, ssa->ops[opline-op_array->opcodes].op1_use, opline-op_array->opcodes)) { + } else if (Z_MODE(op1_addr) == IS_REG && zend_ssa_is_last_use(op_array, ssa, ssa->ops[opline-op_array->opcodes].op1_use, opline-op_array->opcodes)) { result_reg = Z_REG(op1_addr); } else { result_reg = ZREG_R0; @@ -2313,7 +2313,7 @@ static int zend_jit_math_double_long(dasm_State **Dst, if (Z_MODE(res_addr) == IS_REG) { result_reg = Z_REG(res_addr); - } else if (Z_MODE(op1_addr) == IS_REG && zend_ssa_is_last_use(ssa, ssa->ops[opline-op_array->opcodes].op1_use, opline-op_array->opcodes)) { + } else if (Z_MODE(op1_addr) == IS_REG && zend_ssa_is_last_use(op_array, ssa, ssa->ops[opline-op_array->opcodes].op1_use, opline-op_array->opcodes)) { result_reg = Z_REG(op1_addr); } else { result_reg = ZREG_XMM0; @@ -2361,7 +2361,7 @@ static int zend_jit_math_double_double(dasm_State **Dst, if (Z_MODE(res_addr) == IS_REG) { result_reg = Z_REG(res_addr); - } else if (Z_MODE(op1_addr) == IS_REG && zend_ssa_is_last_use(ssa, ssa->ops[opline-op_array->opcodes].op1_use, opline-op_array->opcodes)) { + } else if (Z_MODE(op1_addr) == IS_REG && zend_ssa_is_last_use(op_array, ssa, ssa->ops[opline-op_array->opcodes].op1_use, opline-op_array->opcodes)) { result_reg = Z_REG(op1_addr); } else { result_reg = ZREG_XMM0; @@ -8416,7 +8416,10 @@ static zend_regset zend_jit_get_scratch_regset(zend_op_array *op_array, zend_ssa ZEND_REGSET_INCL(regset, ZREG_XMM1); } } else { - regset = ZEND_REGSET_UNION(ZEND_REGSET(ZREG_XMM0), ZEND_REGSET(ZREG_XMM1)); + regset = ZEND_REGSET(ZREG_XMM0); + if (opline->opcode == ZEND_SUB && (op2_info & MAY_BE_LONG)) { + ZEND_REGSET_INCL(regset, ZREG_XMM1); + } } } break; @@ -8434,7 +8437,6 @@ static zend_regset zend_jit_get_scratch_regset(zend_op_array *op_array, zend_ssa regset = ZEND_REGSET(ZREG_R0); if ((op1_info & MAY_BE_DOUBLE) || (op2_info & MAY_BE_DOUBLE)) { ZEND_REGSET_INCL(regset, ZREG_XMM0); - ZEND_REGSET_INCL(regset, ZREG_XMM1); } } break; From 18c10de0594c3ef9e50052fec00fae6e5c4683cb Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 4 Apr 2017 00:29:27 +0300 Subject: [PATCH 410/569] More accurate scratch registers calculation --- ext/opcache/jit/zend_jit.c | 2 +- ext/opcache/jit/zend_jit_x86.dasc | 32 ++++++++++++++++++++++++++----- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index c8277f8b24c00..68cb060c1570f 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -1581,7 +1581,7 @@ static int zend_jit_try_allocate_free_reg(zend_op_array *op_array, zend_ssa *ssa if (ssa->ops[line].op1_def != current->ssa_var && ssa->ops[line].op2_def != current->ssa_var && ssa->ops[line].result_def != current->ssa_var) { - regset = zend_jit_get_scratch_regset(op_array, ssa, op_array->opcodes + line); + regset = zend_jit_get_scratch_regset(op_array, ssa, line, current->ssa_var); if (!ZEND_REGSET_IS_EMPTY(regset)) { for (reg = 0; reg < ZREG_NUM; reg++) { if (ZEND_REGSET_IN(regset, reg)) { diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 09de42c098a03..643dd2c405c21 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -8347,8 +8347,9 @@ static zend_bool zend_jit_may_be_in_reg(zend_op_array *op_array, zend_ssa *ssa, return 1; } -static zend_regset zend_jit_get_scratch_regset(zend_op_array *op_array, zend_ssa *ssa, zend_op *opline) +static zend_regset zend_jit_get_scratch_regset(zend_op_array *op_array, zend_ssa *ssa, uint32_t line, int current_var) { + const zend_op *opline = op_array->opcodes + line; uint32_t op1_info, op2_info, res_info; zend_regset regset = ZEND_REGSET_SCRATCH; @@ -8363,22 +8364,43 @@ static zend_regset zend_jit_get_scratch_regset(zend_op_array *op_array, zend_ssa case ZEND_SEND_VAL_EX: op1_info = OP1_INFO(); if (!(op1_info & MAY_BE_UNDEF)) { - res_info = RES_INFO(); - if ((res_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) { + if (ssa->ops[line].op1_use == current_var) { + regset = ZEND_REGSET_EMPTY; + } else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) { regset = ZEND_REGSET(ZREG_XMM0); - } else if ((res_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) { + } else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) { regset = ZEND_REGSET(ZREG_R0); } else { regset = ZEND_REGSET_UNION(ZEND_REGSET(ZREG_R0), ZEND_REGSET(ZREG_R2)); } } break; + case ZEND_SEND_VAR: + op1_info = OP1_INFO(); + if (!(op1_info & MAY_BE_UNDEF)) { + if (ssa->ops[line].op1_use == current_var) { + regset = ZEND_REGSET_EMPTY; + } else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) { + regset = ZEND_REGSET(ZREG_XMM0); + } else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) { + regset = ZEND_REGSET(ZREG_R0); + } else { + regset = ZEND_REGSET_UNION(ZEND_REGSET(ZREG_R0), ZEND_REGSET(ZREG_R2)); + if (op1_info & MAY_BE_REF) { + ZEND_REGSET_INCL(regset, ZREG_R1); + } + } + } + break; case ZEND_ASSIGN: op1_info = OP1_INFO(); op2_info = OP2_INFO(); if (opline->op1_type == IS_CV + && !(op2_info & MAY_BE_UNDEF) && !(op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_RESOURCE|MAY_BE_REF))) { - if ((op2_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) { + if (ssa->ops[line].op2_use == current_var) { + regset = ZEND_REGSET_EMPTY; + } else if ((op2_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) { regset = ZEND_REGSET(ZREG_XMM0); } else if ((op2_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) { regset = ZEND_REGSET(ZREG_R0); From 3d9c41b5d98fff231458b65e6398a0ba1f110dfc Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 4 Apr 2017 10:54:57 +0300 Subject: [PATCH 411/569] Support register allocation for RETURN --- ext/opcache/jit/zend_jit.c | 2 +- ext/opcache/jit/zend_jit_x86.dasc | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 68cb060c1570f..d5855a51077ce 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -2122,7 +2122,7 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa, const zend_op *rt_op } goto done; case ZEND_RETURN: - if (!zend_jit_return(&dasm_state, opline, op_array, ssa)) { + if (!zend_jit_return(&dasm_state, opline, op_array, ssa, ra)) { goto jit_failure; } goto done; diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 643dd2c405c21..8ad13f6e7ac41 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -7243,7 +7243,7 @@ static int zend_jit_leave_func(dasm_State **Dst, const zend_op *opline, zend_op_ return 1; } -static int zend_jit_return(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) +static int zend_jit_return(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, zend_lifetime_interval **ra) { uint32_t op1_info; zend_jit_addr op1_addr, ret_addr; @@ -7254,7 +7254,7 @@ static int zend_jit_return(dasm_State **Dst, const zend_op *opline, zend_op_arra } op1_info = OP1_INFO(); - op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, NULL, -1); + op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].op1_use : -1); if (opline->op1_type == IS_CV && (op1_info & MAY_BE_UNDEF)) { // TODO: support for IS_UNDEF ??? return zend_jit_tail_handler(Dst, opline); @@ -8269,6 +8269,7 @@ static zend_bool zend_jit_opline_supports_reg(zend_op_array *op_array, zend_ssa case ZEND_IS_IDENTICAL: case ZEND_IS_NOT_IDENTICAL: case ZEND_CASE: + case ZEND_RETURN: return 1; case ZEND_ASSIGN: op1_info = OP1_INFO(); @@ -8359,6 +8360,9 @@ static zend_regset zend_jit_get_scratch_regset(zend_op_array *op_array, zend_ssa case ZEND_JMP: regset = ZEND_REGSET_EMPTY; break; + case ZEND_RETURN: + regset = ZEND_REGSET(ZREG_R1); + break; case ZEND_QM_ASSIGN: case ZEND_SEND_VAL: case ZEND_SEND_VAL_EX: From 95f38e1cf3b26c30bd121b12b57b7bd79cf1cad3 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 6 Apr 2017 12:23:51 +0300 Subject: [PATCH 412/569] Global register allocation (Incomplete. Seems to work, but needs tweaking). --- ext/opcache/jit/zend_jit.c | 236 +++++++++++++++++++++++++++--- ext/opcache/jit/zend_jit.h | 6 +- ext/opcache/jit/zend_jit_x86.dasc | 107 +++++++++++--- 3 files changed, 306 insertions(+), 43 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index d5855a51077ce..416f088d458f0 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -987,9 +987,12 @@ static int zend_jit_add_range(zend_lifetime_interval **intervals, int var, uint3 ival->ssa_var = var; ival->reg = ZREG_NONE; ival->split = 0; + ival->store = 0; + ival->load = 0; ival->range.start = from; ival->range.end = to; ival->range.next = NULL; + ival->hint = NULL; intervals[var] = ival; } else if (ival->range.start > to + 1) { zend_life_range *range = zend_arena_alloc(&CG(arena), sizeof(zend_life_range)); @@ -1098,9 +1101,14 @@ static int zend_jit_split_interval(zend_lifetime_interval *current, uint32_t pos } } + current->store = 1; + ival->ssa_var = current->ssa_var; ival->reg = ZREG_NONE; ival->split = 1; + ival->store = 0; + ival->load = 1; + ival->hint = NULL; do { if (pos >= range->start && pos <= range->end) { @@ -1260,6 +1268,31 @@ static void zend_jit_compute_loop_body(zend_ssa *ssa, int header, int n, zend_bi } } +static void zend_jit_add_hint(zend_lifetime_interval **intervals, int dst, int src) +{ + if (intervals[dst]->range.start < intervals[src]->range.start) { + int tmp = src; + src = dst; + dst = tmp; + } + while (1) { + if (intervals[dst]->hint) { + if (intervals[dst]->hint->range.start < intervals[src]->range.start) { + int tmp = src; + src = intervals[dst]->hint->ssa_var; + dst = tmp; + } else { + dst = intervals[dst]->hint->ssa_var; + } + } else { + if (dst != src) { + intervals[dst]->hint = intervals[src]; + } + return; + } + } +} + /* See "Linear Scan Register Allocation on SSA Form", Christian Wimmer and Michael Franz, CGO'10 (2010), Figure 4. */ static int zend_jit_compute_liveness(zend_op_array *op_array, zend_ssa *ssa, zend_bitset candidates, zend_lifetime_interval **list) @@ -1446,6 +1479,42 @@ static int zend_jit_compute_liveness(zend_op_array *op_array, zend_ssa *ssa, zen zend_bitset_copy(live_in + set_size * i, live, set_size); } + if (zend_jit_reg_alloc >= ZEND_JIT_REG_ALLOC_GLOBAL) { + /* Register hinting (a cheap way for register coalesing) */ + for (i = 0; i < ssa->vars_count; i++) { + if (intervals[i]) { + int var = intervals[i]->ssa_var; + int src; + + if (ssa->vars[var].definition_phi) { + zend_ssa_phi *phi = ssa->vars[var].definition_phi; + + if (phi->pi >= 0) { + src = phi->sources[0]; + if (intervals[src]) { + zend_jit_add_hint(intervals, i, src); + } + } else { + for (k = 0; k < ssa->cfg.blocks[phi->block].predecessors_count; k++) { + src = phi->sources[k]; + if (src >= 0) { + if (ssa->vars[src].definition_phi + && ssa->vars[src].definition_phi->pi >= 0 + && phi->block == ssa->vars[src].definition_phi->block) { + /* Skip zero-lenght interval for Pi variable */ + src = ssa->vars[src].definition_phi->sources[0]; + } + if (intervals[src]) { + zend_jit_add_hint(intervals, i, src); + } + } + } + } + } + } + } + } + *list = zend_jit_sort_intervals(intervals, ssa->vars_count); free_alloca(intervals, use_heap); return SUCCESS; @@ -1550,6 +1619,9 @@ static int zend_jit_try_allocate_free_reg(zend_op_array *op_array, zend_ssa *ssa freeUntilPos[it->reg] = 0; it = it->list_next; } + if (current->hint) { + hint = current->hint->reg; + } } /* See "Linear Scan Register Allocation on SSA Form", Christian Wimmer and @@ -1803,13 +1875,27 @@ static zend_lifetime_interval** zend_jit_allocate_registers(zend_op_array *op_ar ival = list; while (ival) { zend_life_range *range; + int var_num = ssa->vars[ival->ssa_var].var; - fprintf(stderr, "#%d: %u-%u", ival->ssa_var, ival->range.start, ival->range.end); + fprintf(stderr, "#%d.", ival->ssa_var); + zend_dump_var(op_array, (var_num < op_array->last_var ? IS_CV : 0), var_num); + fprintf(stderr, ": %u-%u", ival->range.start, ival->range.end); range = ival->range.next; while (range) { fprintf(stderr, ", %u-%u", range->start, range->end); range = range->next; } + if (ival->load) { + fprintf(stderr, " load"); + } + if (ival->store) { + fprintf(stderr, " store"); + } + if (ival->hint) { + var_num = ssa->vars[ival->hint->ssa_var].var; + fprintf(stderr, " hint=#%d.", ival->hint->ssa_var); + zend_dump_var(op_array, (var_num < op_array->last_var ? IS_CV : 0), var_num); + } fprintf(stderr, "\n"); ival = ival->list_next; } @@ -1819,24 +1905,6 @@ static zend_lifetime_interval** zend_jit_allocate_registers(zend_op_array *op_ar list = zend_jit_linear_scan(op_array, ssa, list); if (list) { - - if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_REG_ALLOC) { - fprintf(stderr, "Allocated Live Ranges \"%s\"\n", op_array->function_name ? ZSTR_VAL(op_array->function_name) : "[main]"); - ival = list; - while (ival) { - zend_life_range *range; - - fprintf(stderr, "#%d: %u-%u", ival->ssa_var, ival->range.start, ival->range.end); - range = ival->range.next; - while (range) { - fprintf(stderr, ", %u-%u", range->start, range->end); - range = range->next; - } - fprintf(stderr, " (%s) \n", zend_reg_name[ival->reg]); - ival = ival->list_next; - } - } - intervals = zend_arena_calloc(&CG(arena), ssa->vars_count, sizeof(zend_lifetime_interval*)); if (!intervals) { goto failure; @@ -1851,6 +1919,112 @@ static zend_lifetime_interval** zend_jit_allocate_registers(zend_op_array *op_ar ival = next; } + if (zend_jit_reg_alloc >= ZEND_JIT_REG_ALLOC_GLOBAL) { + /* Naive SSA resolution */ + for (i = 0; i < ssa->vars_count; i++) { + if (ssa->vars[i].definition_phi && !ssa->vars[i].no_val) { + zend_ssa_phi *phi = ssa->vars[i].definition_phi; + int k, src; + + if (phi->pi >= 0) { + if (!ssa->vars[i].phi_use_chain + || ssa->vars[i].phi_use_chain->block != phi->block) { + src = phi->sources[0]; + if (intervals[i]) { + if (!intervals[src]) { + intervals[i]->load = 1; + } else if (intervals[i]->reg != intervals[src]->reg) { + intervals[i]->load = 1; + intervals[src]->store = 1; + } + } else if (intervals[src]) { + intervals[src]->store = 1; + } + } + } else { + int need_move = 0; + + for (k = 0; k < ssa->cfg.blocks[phi->block].predecessors_count; k++) { + src = phi->sources[k]; + if (src >= 0) { + if (ssa->vars[src].definition_phi + && ssa->vars[src].definition_phi->pi >= 0 + && phi->block == ssa->vars[src].definition_phi->block) { + /* Skip zero-lenght interval for Pi variable */ + src = ssa->vars[src].definition_phi->sources[0]; + } + if (intervals[i]) { + if (!intervals[src]) { + need_move = 1; + } else if (intervals[i]->reg != intervals[src]->reg) { + need_move = 1; + } + } else if (intervals[src]) { + need_move = 1; + } + } + } + if (need_move) { + if (intervals[i]) { + intervals[i]->load = 1; + } + for (k = 0; k < ssa->cfg.blocks[phi->block].predecessors_count; k++) { + src = phi->sources[k]; + if (src >= 0) { + if (ssa->vars[src].definition_phi + && ssa->vars[src].definition_phi->pi >= 0 + && phi->block == ssa->vars[src].definition_phi->block) { + /* Skip zero-lenght interval for Pi variable */ + src = ssa->vars[src].definition_phi->sources[0]; + } + if (intervals[src]) { + intervals[src]->store = 1; + } + } + } + } + } + } + } + } + + if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_REG_ALLOC) { + fprintf(stderr, "Allocated Live Ranges \"%s\"\n", op_array->function_name ? ZSTR_VAL(op_array->function_name) : "[main]"); + for (i = 0; i < ssa->vars_count; i++) { + ival = intervals[i]; + while (ival) { + zend_life_range *range; + int var_num = ssa->vars[ival->ssa_var].var; + + fprintf(stderr, "#%d.", ival->ssa_var); + zend_dump_var(op_array, (var_num < op_array->last_var ? IS_CV : 0), var_num); + fprintf(stderr, ": %u-%u", ival->range.start, ival->range.end); + range = ival->range.next; + while (range) { + fprintf(stderr, ", %u-%u", range->start, range->end); + range = range->next; + } + fprintf(stderr, " (%s)", zend_reg_name[ival->reg]); + if (ival->load) { + fprintf(stderr, " load"); + } + if (ival->store) { + fprintf(stderr, " store"); + } + if (ival->hint) { + var_num = ssa->vars[ival->hint->ssa_var].var; + fprintf(stderr, " hint=#%d.", ival->hint->ssa_var); + zend_dump_var(op_array, (var_num < op_array->last_var ? IS_CV : 0), var_num); + if (ival->hint->reg != ZREG_NONE) { + fprintf(stderr, " (%s)", zend_reg_name[ival->hint->reg]); + } + } + fprintf(stderr, "\n"); + ival = ival->list_next; + } + } + } + free_alloca(candidates, use_heap); return intervals; } @@ -1994,6 +2168,30 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa, const zend_op *rt_op if (!ssa->cfg.blocks[b].len) { continue; } + if ((zend_jit_reg_alloc >= ZEND_JIT_REG_ALLOC_GLOBAL) && ra) { + zend_ssa_phi *phi = ssa->blocks[b].phis; + + while (phi) { + zend_lifetime_interval *ival = ra[phi->ssa_var]; + + if (ival) { + if (ival->load) { + ZEND_ASSERT(ival->reg != ZREG_NONE); + + if (!zend_jit_load_ssa_var(&dasm_state, ssa, phi->ssa_var, ival->reg)) { + goto jit_failure; + } + } else if (ival->store) { + ZEND_ASSERT(ival->reg != ZREG_NONE); + + if (!zend_jit_store_ssa_var(&dasm_state, ssa, phi->ssa_var, ival->reg)) { + goto jit_failure; + } + } + } + phi = phi->next; + } + } end = ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len - 1; for (i = ssa->cfg.blocks[b].start; i <= end; i++) { opline = op_array->opcodes + i; diff --git a/ext/opcache/jit/zend_jit.h b/ext/opcache/jit/zend_jit.h index 7727b5197f8f8..81b0f06b6639b 100644 --- a/ext/opcache/jit/zend_jit.h +++ b/ext/opcache/jit/zend_jit.h @@ -39,7 +39,8 @@ #define ZEND_JIT_TRIGGER(n) (((n) / 10) % 10) #define ZEND_JIT_REG_ALLOC_NONE 0 /* no register allocation */ -#define ZEND_JIT_REG_ALLOC_ENABLED 1 /* linear scan register allocation */ +#define ZEND_JIT_REG_ALLOC_ENABLED 1 /* local linear scan register allocation */ +#define ZEND_JIT_REG_ALLOC_GLOBAL 2 /* global linear scan register allocation */ #define ZEND_JIT_REG_ALLOC(n) (((n) / 100) % 10) @@ -92,7 +93,10 @@ struct _zend_lifetime_interval { int ssa_var; int8_t reg; zend_bool split; + zend_bool store; + zend_bool load; zend_life_range range; + zend_lifetime_interval *hint; zend_lifetime_interval *list_next; }; diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 8ad13f6e7ac41..5be30000f14af 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -1953,21 +1953,25 @@ static int zend_jit_new(dasm_State **Dst, const zend_op *opline, int *opnum, zen return 1; } -static int zend_jit_spill_store(dasm_State **Dst, zend_jit_addr src, zend_jit_addr dst, uint32_t info) +static int zend_jit_spill_store(dasm_State **Dst, zend_jit_addr src, zend_jit_addr dst, uint32_t info, zend_bool set_type) { ZEND_ASSERT(Z_MODE(src) == IS_REG); ZEND_ASSERT(Z_MODE(dst) == IS_MEM_ZVAL); if ((info & MAY_BE_ANY) == MAY_BE_LONG) { | SET_ZVAL_LVAL dst, Ra(Z_REG(src)) - | SET_ZVAL_TYPE_INFO dst, IS_LONG + if (set_type) { + | SET_ZVAL_TYPE_INFO dst, IS_LONG + } } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) { | .if SSE | SSE_SET_ZVAL_DVAL dst, Z_REG(src) | .else || ZEND_ASSERT(0); | .endif - | SET_ZVAL_TYPE_INFO dst, IS_DOUBLE + if (set_type) { + | SET_ZVAL_TYPE_INFO dst, IS_DOUBLE + } } else { ZEND_ASSERT(0); } @@ -1993,6 +1997,35 @@ static int zend_jit_load_reg(dasm_State **Dst, zend_jit_addr src, zend_jit_addr return 1; } +static int zend_jit_store_ssa_var(dasm_State **Dst, zend_ssa *ssa, int var, zend_reg reg) +{ + zend_jit_addr src = ZEND_ADDR_REG(reg); + zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, (uint32_t)(uintptr_t)ZEND_CALL_VAR_NUM(NULL, ssa->vars[var].var)); + uint32_t info = ssa->var_info[var].type; + + return zend_jit_spill_store(Dst, src, dst, info, 1); +} + +static int zend_jit_store_ssa_var_if_necessary(dasm_State **Dst, zend_ssa *ssa, zend_lifetime_interval **ra, zend_jit_addr src, int var) +{ + if (Z_MODE(src) == IS_REG && ra[var] && ra[var]->store) { + zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, (uint32_t)(uintptr_t)ZEND_CALL_VAR_NUM(NULL, ssa->vars[var].var)); + uint32_t info = ssa->var_info[var].type; + + return zend_jit_spill_store(Dst, src, dst, info, 1); + } + return 1; +} + +static int zend_jit_load_ssa_var(dasm_State **Dst, zend_ssa *ssa, int var, zend_reg reg) +{ + zend_jit_addr src = ZEND_ADDR_MEM_ZVAL(ZREG_FP, (uint32_t)(uintptr_t)ZEND_CALL_VAR_NUM(NULL, ssa->vars[var].var)); + zend_jit_addr dst = ZEND_ADDR_REG(reg); + uint32_t info = ssa->var_info[var].type; + + return zend_jit_load_reg(Dst, src, dst, info); +} + static int zend_jit_update_regs(dasm_State **Dst, zend_jit_addr src, zend_jit_addr dst, uint32_t info) { if (src != dst) { @@ -2010,7 +2043,7 @@ static int zend_jit_update_regs(dasm_State **Dst, zend_jit_addr src, zend_jit_ad ZEND_ASSERT(0); } } else if (Z_MODE(dst) == IS_MEM_ZVAL) { - if (!zend_jit_spill_store(Dst, src, dst, info)) { + if (!zend_jit_spill_store(Dst, src, dst, info, 1)) { return 0; } } else { @@ -2160,6 +2193,14 @@ static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, zend_op_arr |.code } |3: + if (ra && !zend_jit_store_ssa_var_if_necessary(Dst, ssa, ra, op1_def_addr, ssa->ops[opline - op_array->opcodes].op1_def)) { + return 0; + } + if (opline->result_type != IS_UNUSED) { + if (ra && !zend_jit_store_ssa_var_if_necessary(Dst, ssa, ra, res_addr, ssa->ops[opline - op_array->opcodes].result_def)) { + return 0; + } + } return 1; fallback: @@ -2628,7 +2669,7 @@ static int zend_jit_math_helper(dasm_State **Dst, } if (Z_MODE(op1_addr) == IS_REG) { zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var); - if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info)) { + if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { return 0; } op1_addr = real_addr; @@ -2636,7 +2677,7 @@ static int zend_jit_math_helper(dasm_State **Dst, | LOAD_ZVAL_ADDR FCARG2a, op1_addr if (Z_MODE(op2_addr) == IS_REG) { zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op2.var); - if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info)) { + if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) { return 0; } op2_addr = real_addr; @@ -2720,7 +2761,13 @@ static int zend_jit_math(dasm_State **Dst, const zend_op *opline, int *opnum, ze res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, ra, ssa->ops[opline - op_array->opcodes].result_def); } - return zend_jit_math_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, opline->result.var, res_addr, RES_INFO(), res_use_info, 0); + if (!zend_jit_math_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, opline->result.var, res_addr, RES_INFO(), res_use_info, 0)) { + return 0; + } + if (ra && !zend_jit_store_ssa_var_if_necessary(Dst, ssa, ra, res_addr, ssa->ops[opline - op_array->opcodes].result_def)) { + return 0; + } + return 1; fallback: /* fallback to subroutine threading */ @@ -4797,14 +4844,14 @@ static int zend_jit_cmp(dasm_State **Dst, const zend_op *opline, int b, int *opn | SAVE_VALID_OPLINE opline if (Z_MODE(op1_addr) == IS_REG) { zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); - if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info)) { + if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { return 0; } op1_addr = real_addr; } if (Z_MODE(op2_addr) == IS_REG) { zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var); - if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info)) { + if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) { return 0; } op2_addr = real_addr; @@ -5589,7 +5636,13 @@ static int zend_jit_qm_assign(dasm_State **Dst, const zend_op *opline, zend_op_a } } - return zend_jit_simple_assign(Dst, opline, op_array, ssa, res_addr, -1, opline->op1_type, opline->op1, op1_addr, op1_info, 0, 0); + if (!zend_jit_simple_assign(Dst, opline, op_array, ssa, res_addr, -1, opline->op1_type, opline->op1, op1_addr, op1_info, 0, 0)) { + return 0; + } + if (ra && !zend_jit_store_ssa_var_if_necessary(Dst, ssa, ra, res_addr, ssa->ops[opline - op_array->opcodes].result_def)) { + return 0; + } + return 1; } static int zend_jit_assign(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, zend_lifetime_interval **ra) @@ -8307,19 +8360,22 @@ static zend_bool zend_jit_may_be_in_reg(zend_op_array *op_array, zend_ssa *ssa, return 0; } - // TODO: enable global register allocation ??? - // SSA resolution is not implemented yet. - if (ssa->vars[var].definition_phi) { - return 0; - } - if (ssa->vars[var].phi_use_chain) { - zend_ssa_phi *phi = ssa->vars[var].phi_use_chain; - do { - if (!ssa->vars[phi->ssa_var].no_val) { - return 0; - } - phi = zend_ssa_next_use_phi(ssa, var, phi); - } while (phi); + if (zend_jit_reg_alloc < ZEND_JIT_REG_ALLOC_GLOBAL) { + /* Disable global register allocation, + * register allocation forSAA variables connected through Phi functions + */ + if (ssa->vars[var].definition_phi) { + return 0; + } + if (ssa->vars[var].phi_use_chain) { + zend_ssa_phi *phi = ssa->vars[var].phi_use_chain; + do { + if (!ssa->vars[phi->ssa_var].no_val) { + return 0; + } + phi = zend_ssa_next_use_phi(ssa, var, phi); + } while (phi); + } } if (((ssa->var_info[var].type & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_DOUBLE) && @@ -8446,6 +8502,11 @@ static zend_regset zend_jit_get_scratch_regset(zend_op_array *op_array, zend_ssa if (opline->opcode == ZEND_SUB && (op2_info & MAY_BE_LONG)) { ZEND_REGSET_INCL(regset, ZREG_XMM1); } + |.if X64 + || if (opline->op1_type == IS_CONST || opline->op2_type == IS_CONST) { + || ZEND_REGSET_INCL(regset, ZREG_R0); + || } + |.endif } } break; From 963a1c2f2f23cf2b3de703c9f4616483b890ab13 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 6 Apr 2017 16:48:21 +0300 Subject: [PATCH 413/569] Register allocation improvements and fixes --- ext/opcache/jit/zend_jit.c | 25 +++++++++++++------------ ext/opcache/jit/zend_jit_x86.dasc | 15 +++++++++++++-- 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 416f088d458f0..9d87307abf1d2 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -1649,23 +1649,24 @@ static int zend_jit_try_allocate_free_reg(zend_op_array *op_array, zend_ssa *ssa zend_regset regset; zend_reg reg; - do { - if (ssa->ops[line].op1_def != current->ssa_var && - ssa->ops[line].op2_def != current->ssa_var && - ssa->ops[line].result_def != current->ssa_var) { - regset = zend_jit_get_scratch_regset(op_array, ssa, line, current->ssa_var); - if (!ZEND_REGSET_IS_EMPTY(regset)) { - for (reg = 0; reg < ZREG_NUM; reg++) { - if (ZEND_REGSET_IN(regset, reg)) { - if (line < freeUntilPos[reg]) { - freeUntilPos[reg] = line; - } + if (ssa->ops[line].op1_def == current->ssa_var || + ssa->ops[line].op2_def == current->ssa_var || + ssa->ops[line].result_def == current->ssa_var) { + line++; + } + while (line <= range->end) { + regset = zend_jit_get_scratch_regset(op_array, ssa, line, current->ssa_var); + if (!ZEND_REGSET_IS_EMPTY(regset)) { + for (reg = 0; reg < ZREG_NUM; reg++) { + if (ZEND_REGSET_IN(regset, reg)) { + if (line < freeUntilPos[reg]) { + freeUntilPos[reg] = line; } } } } line++; - } while (line <= range->end); + } range = range->next; } while (range); diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 5be30000f14af..02dfcfdeba425 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -2602,7 +2602,7 @@ static int zend_jit_math_helper(dasm_State **Dst, } |1: if (op2_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) { - | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >5 + | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6 } if (!zend_jit_math_double_long(Dst, op_array, ssa, opline, op1_addr, op2_addr, res_addr, res_use_info)) { return 0; @@ -8521,9 +8521,20 @@ static zend_regset zend_jit_get_scratch_regset(zend_op_array *op_array, zend_ssa op2_info = OP1_INFO(); if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) && !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { - regset = ZEND_REGSET(ZREG_R0); + regset = ZEND_REGSET_EMPTY; + if ((op1_info & MAY_BE_LONG) && + (op2_info & MAY_BE_LONG) && + opline->op1_type != IS_CONST && + opline->op2_type != IS_CONST) { + ZEND_REGSET_INCL(regset, ZREG_R0); + } if ((op1_info & MAY_BE_DOUBLE) || (op2_info & MAY_BE_DOUBLE)) { ZEND_REGSET_INCL(regset, ZREG_XMM0); + |.if X64 + || if (opline->op1_type == IS_CONST || opline->op2_type == IS_CONST) { + || ZEND_REGSET_INCL(regset, ZREG_R0); + || } + |.endif } } break; From 4557e6dd883f4d4e284353c8c3b5e61797af5054 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 6 Apr 2017 17:28:46 +0300 Subject: [PATCH 414/569] Cheaper vm_interrupt check --- ext/opcache/jit/zend_jit_x86.dasc | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 02dfcfdeba425..f9d7f3180605f 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -1756,9 +1756,20 @@ static int zend_jit_set_valid_ip(dasm_State **Dst, const zend_op *opline) static int zend_jit_check_timeout(dasm_State **Dst, const zend_op *opline) { if (zend_interrupt_function) { +#if 0 if (!zend_jit_set_valid_ip(Dst, opline)) { return 0; } +#else + | cmp byte [&EG(vm_interrupt)], 0 + | jne >1 + |.cold_code + |1: + | LOAD_ADDR IP, opline + | jmp ->interrupt_handler + |.code + return 1; +#endif } | cmp byte [&EG(vm_interrupt)], 0 | jne ->interrupt_handler From d9474c784041745a455dd5be06b52c0029e40697 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 7 Apr 2017 13:21:21 +0300 Subject: [PATCH 415/569] Use AVX instructions (if enabled and available). This makes improvement for 32-bit build, but for some reason makes huge slowdown on x86_64 (has to be investigated and solved). --- ext/opcache/jit/zend_cpuid.h | 60 ++++++++++++++++++++++ ext/opcache/jit/zend_jit.c | 7 +++ ext/opcache/jit/zend_jit.h | 5 ++ ext/opcache/jit/zend_jit_x86.dasc | 84 +++++++++++++++++++++++++++++-- 4 files changed, 152 insertions(+), 4 deletions(-) create mode 100644 ext/opcache/jit/zend_cpuid.h diff --git a/ext/opcache/jit/zend_cpuid.h b/ext/opcache/jit/zend_cpuid.h new file mode 100644 index 0000000000000..f095afb028663 --- /dev/null +++ b/ext/opcache/jit/zend_cpuid.h @@ -0,0 +1,60 @@ +/* + +----------------------------------------------------------------------+ + | Zend JIT | + +----------------------------------------------------------------------+ + | Copyright (c) 1998-2016 The PHP Group | + +----------------------------------------------------------------------+ + | This source file is subject to version 3.01 of the PHP license, | + | that is bundled with this package in the file LICENSE, and is | + | available through the world-wide-web at the following url: | + | http://www.php.net/license/3_01.txt | + | If you did not receive a copy of the PHP license and are unable to | + | obtain it through the world-wide-web, please send a note to | + | license@php.net so we can mail you a copy immediately. | + +----------------------------------------------------------------------+ + | Authors: Dmitry Stogov | + +----------------------------------------------------------------------+ +*/ + +#ifndef ZEND_CPUID +#define ZEND_CPUID + +#define ZEND_CPUID_EDX_FPU (1<<0) +#define ZEND_CPUID_EDX_MMX (1<<23) +#define ZEND_CPUID_EDX_SSE (1<<25) +#define ZEND_CPUID_EDX_SSE2 (1<<26) + +#define ZEND_CPUID_ECX_SSE3 (1<<0) +#define ZEND_CPUID_ECX_SSSE3 (1<<9) +#define ZEND_CPUID_ECX_FMA3 (1<<12) +#define ZEND_CPUID_ECX_SSE41 (1<<19) +#define ZEND_CPUID_ECX_SSE42 (1<<20) +#define ZEND_CPUID_ECX_AES (1<<25) +#define ZEND_CPUID_ECX_AVX (1<<28) +#define ZEND_CPUID_ECX_RDRAND (1<<30) + +typedef struct _zend_cpuid_info { + uint32_t eax; + uint32_t ebx; + uint32_t ecx; + uint32_t edx; +} zend_cpuid_info; + +static inline void zend_cpuid(uint32_t func, uint32_t subfunc, zend_cpuid_info *info) +{ + __asm__ __volatile__ ( + "cpuid" + : "=a"(info->eax), "=b"(info->ebx), "=c"(info->ecx), "=d"(info->edx) + : "a"(func), "c"(subfunc) + ); +} + +#endif + +/* + * Local variables: + * tab-width: 4 + * c-basic-offset: 4 + * indent-tabs-mode: t + * End: + */ diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 9d87307abf1d2..a639e367046b6 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -68,6 +68,7 @@ typedef struct _zend_jit_stub { static zend_uchar zend_jit_level = 0; static zend_uchar zend_jit_trigger = 0; static zend_uchar zend_jit_reg_alloc = 0; +static zend_uchar zend_jit_cpu_flags = 0; zend_ulong zend_jit_profile_counter = 0; int zend_jit_profile_counter_rid = -1; @@ -2998,6 +2999,12 @@ ZEND_API int zend_jit_startup(zend_long jit, size_t size) zend_jit_level = ZEND_JIT_LEVEL(jit); zend_jit_trigger = ZEND_JIT_TRIGGER(jit); zend_jit_reg_alloc = ZEND_JIT_REG_ALLOC(jit); + zend_jit_cpu_flags = ZEND_JIT_CPU_FLAGS(jit); + + if (zend_jit_setup() != SUCCESS) { + // TODO: error reporting and cleanup ??? + return FAILURE; + } if (zend_jit_trigger == ZEND_JIT_ON_PROF_REQUEST) { zend_extension dummy; diff --git a/ext/opcache/jit/zend_jit.h b/ext/opcache/jit/zend_jit.h index 81b0f06b6639b..4702967941229 100644 --- a/ext/opcache/jit/zend_jit.h +++ b/ext/opcache/jit/zend_jit.h @@ -44,6 +44,11 @@ #define ZEND_JIT_REG_ALLOC(n) (((n) / 100) % 10) +#define ZEND_JIT_CPU_AVX 1 /* use AVX instructions, if available */ + +#define ZEND_JIT_CPU_FLAGS(n) (((n) / 1000) % 10) + + #define ZEND_JIT_DEFAULT "5" diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index f9d7f3180605f..0f3d1acab8179 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -62,6 +62,7 @@ |.endif #include "jit/zend_jit_x86.h" +#include "jit/zend_cpuid.h" const char* zend_reg_name[] = { #ifdef __x86_64__ @@ -75,6 +76,8 @@ const char* zend_reg_name[] = { #endif }; +static uint32_t zend_jit_x86_flags = 0; + |.type EX, zend_execute_data, FP |.type OP, zend_op, IP |.type ZVAL, zval @@ -374,6 +377,48 @@ static void* dasm_labels[zend_lb_MAX]; || } |.endmacro +|.macro AVX_OP, avx_ins, reg, op1_reg, addr +|| if (Z_MODE(addr) == IS_CONST_ZVAL) { +| .if X64 +|| if (IS_32BIT(Z_ZV(addr))) { +| avx_ins xmm(reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), qword [Z_ZV(addr)] +|| } else { +| LOAD_ADDR r0, Z_ZV(addr) +| avx_ins xmm(reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), qword [r0] +|| } +| .else +| avx_ins xmm(reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), qword [Z_ZV(addr)] +| .endif +|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { +| avx_ins xmm(reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), qword [Ra(Z_REG(addr))+Z_OFFSET(addr)] +|| } else if (Z_MODE(addr) == IS_REG) { +| avx_ins xmm(reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), xmm(Z_REG(addr)-ZREG_XMM0) +|| } else { +|| ZEND_ASSERT(0); +|| } +|.endmacro + +|.macro AVX_MATH, opcode, reg, op1_reg, addr +|| switch (opcode) { +|| case ZEND_ADD: +|| case ZEND_ASSIGN_ADD: +| AVX_OP vaddsd, reg, op1_reg, addr +|| break; +|| case ZEND_SUB: +|| case ZEND_ASSIGN_SUB: +| AVX_OP vsubsd, reg, op1_reg, addr +|| break; +|| case ZEND_MUL: +|| case ZEND_ASSIGN_MUL: +| AVX_OP vmulsd, reg, op1_reg, addr +|| break; +|| case ZEND_DIV: +|| case ZEND_ASSIGN_DIV: +| AVX_OP vdivsd, reg, op1_reg, addr +|| break; +|| } +|.endmacro + |.macro LONG_OP, long_ins, reg, addr || if (Z_MODE(addr) == IS_CONST_ZVAL) { | .if X64 @@ -1697,6 +1742,31 @@ static const zend_jit_stub zend_jit_stubs[] = { JIT_STUB(not_obj), }; +static int zend_jit_setup(void) +{ + zend_cpuid_info info; + + zend_cpuid(0, 0, &info); + if (info.eax < 1) { + return FAILURE; + } + zend_cpuid(1, 0, &info); + + |.if SSE + if (!(info.edx & ZEND_CPUID_EDX_SSE2)) { + zend_error(E_CORE_ERROR, "CPU doesn't support SSE2"); + return FAILURE; + } + if (zend_jit_cpu_flags & ZEND_JIT_CPU_AVX) { + if (info.ecx & ZEND_CPUID_ECX_AVX) { + zend_jit_x86_flags |= ZEND_JIT_CPU_AVX; + } + } + |.endif + + return SUCCESS; +} + static int zend_jit_align_func(dasm_State **Dst) { reuse_ip = 0; @@ -2419,11 +2489,17 @@ static int zend_jit_math_double_double(dasm_State **Dst, result_reg = ZREG_XMM0; } - | SSE_GET_ZVAL_DVAL result_reg, op1_addr - if (same_ops) { - | SSE_MATH_REG opline->opcode, result_reg, result_reg + if (Z_MODE(op1_addr) == IS_REG + && Z_REG(op1_addr) != result_reg + && (zend_jit_x86_flags & ZEND_JIT_CPU_AVX)) { + | AVX_MATH opline->opcode, result_reg, Z_REG(op1_addr), op2_addr } else { - | SSE_MATH opline->opcode, result_reg, op2_addr + | SSE_GET_ZVAL_DVAL result_reg, op1_addr + if (same_ops) { + | SSE_MATH_REG opline->opcode, result_reg, result_reg + } else { + | SSE_MATH opline->opcode, result_reg, op2_addr + } } | SSE_SET_ZVAL_DVAL res_addr, result_reg |.else From 8f77ccda7bfe6f382493b2dd1f62f60a2be3cfcb Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 7 Apr 2017 15:31:10 +0300 Subject: [PATCH 416/569] Avoid AVX/SSE transition penalties --- ext/opcache/jit/zend_jit_x86.dasc | 186 +++++++++++++++++++++++++----- 1 file changed, 154 insertions(+), 32 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 0f3d1acab8179..5be23872b6b2a 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -276,6 +276,14 @@ static void* dasm_labels[zend_lb_MAX]; | fstp qword [Ra(Z_REG(res_addr))+Z_OFFSET(res_addr)] |.endmacro +|.macro SSE_AVX_INS, sse_ins, avx_ins, op1, op2 +|| if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { +| avx_ins op1, op2 +|| } else { +| sse_ins op1, op2 +|| } +|.endmacro + |.macro SSE_OP, sse_ins, reg, addr || if (Z_MODE(addr) == IS_CONST_ZVAL) { | .if X64 @@ -283,7 +291,7 @@ static void* dasm_labels[zend_lb_MAX]; | sse_ins xmm(reg-ZREG_XMM0), qword [Z_ZV(addr)] || } else { | LOAD_ADDR r0, Z_ZV(addr) -| sse_ins xmm(reg-ZREG_XMM0), qword [r0] +| sse_ins, xmm(reg-ZREG_XMM0), qword [r0] || } | .else | sse_ins xmm(reg-ZREG_XMM0), qword [Z_ZV(addr)] @@ -297,22 +305,63 @@ static void* dasm_labels[zend_lb_MAX]; || } |.endmacro -|.macro SSE_GET_ZVAL_LVAL, reg, addr +|.macro SSE_AVX_OP, sse_ins, avx_ins, reg, addr || if (Z_MODE(addr) == IS_CONST_ZVAL) { | .if X64 || if (IS_32BIT(Z_ZV(addr))) { -| cvtsi2sd xmm(reg-ZREG_XMM0), aword [Z_ZV(addr)] +| SSE_AVX_INS sse_ins, avx_ins, xmm(reg-ZREG_XMM0), qword [Z_ZV(addr)] +|| } else { +| LOAD_ADDR r0, Z_ZV(addr) +| SSE_AVX_INS sse_ins, avx_ins, xmm(reg-ZREG_XMM0), qword [r0] +|| } +| .else +| SSE_AVX_INS sse_ins, avx_ins, xmm(reg-ZREG_XMM0), qword [Z_ZV(addr)] +| .endif +|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { +| SSE_AVX_INS sse_ins, avx_ins, xmm(reg-ZREG_XMM0), qword [Ra(Z_REG(addr))+Z_OFFSET(addr)] +|| } else if (Z_MODE(addr) == IS_REG) { +| SSE_AVX_INS sse_ins, avx_ins, xmm(reg-ZREG_XMM0), xmm(Z_REG(addr)-ZREG_XMM0) +|| } else { +|| ZEND_ASSERT(0); +|| } +|.endmacro + +|.macro SSE_GET_ZVAL_LVAL, reg, addr +|| if (Z_MODE(addr) == IS_CONST_ZVAL) { +| .if X64 +|| if (IS_32BIT(Z_ZV(addr))) { +|| if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { +| vcvtsi2sd xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), aword [Z_ZV(addr)] +|| } else { +| cvtsi2sd xmm(reg-ZREG_XMM0), aword [Z_ZV(addr)] +|| } || } else { | LOAD_ADDR r0, Z_ZV(addr) -| cvtsi2sd xmm(reg-ZREG_XMM0), aword [r0] +|| if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { +| vcvtsi2sd xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), aword [r0] +|| } else { +| cvtsi2sd xmm(reg-ZREG_XMM0), aword [r0] +|| } || } | .else -| cvtsi2sd xmm(reg-ZREG_XMM0), aword [Z_ZV(addr)] +|| if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { +| vcvtsi2sd xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), aword [Z_ZV(addr)] +|| } else { +| cvtsi2sd xmm(reg-ZREG_XMM0), aword [Z_ZV(addr)] +|| } | .endif || } else if (Z_MODE(addr) == IS_MEM_ZVAL) { -| cvtsi2sd xmm(reg-ZREG_XMM0), aword [Ra(Z_REG(addr))+Z_OFFSET(addr)] +|| if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { +| vcvtsi2sd xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), aword [Ra(Z_REG(addr))+Z_OFFSET(addr)] +|| } else { +| cvtsi2sd xmm(reg-ZREG_XMM0), aword [Ra(Z_REG(addr))+Z_OFFSET(addr)] +|| } || } else if (Z_MODE(addr) == IS_REG) { -| cvtsi2sd xmm(reg-ZREG_XMM0), Ra(Z_REG(addr)) +|| if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { +| vcvtsi2sd xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), Ra(Z_REG(addr)) +|| } else { +| cvtsi2sd xmm(reg-ZREG_XMM0), Ra(Z_REG(addr)) +|| } || } else { || ZEND_ASSERT(0); || } @@ -320,7 +369,24 @@ static void* dasm_labels[zend_lb_MAX]; |.macro SSE_GET_ZVAL_DVAL, reg, addr || if (Z_MODE(addr) != IS_REG || reg != Z_REG(addr)) { -| SSE_OP movsd, reg, addr +|| if (Z_MODE(addr) == IS_CONST_ZVAL) { +| .if X64 +|| if (IS_32BIT(Z_ZV(addr))) { +| SSE_AVX_INS movsd, vmovsd, xmm(reg-ZREG_XMM0), qword [Z_ZV(addr)] +|| } else { +| LOAD_ADDR r0, Z_ZV(addr) +| SSE_AVX_INS movsd, vmovsd, xmm(reg-ZREG_XMM0), qword [r0] +|| } +| .else +| SSE_AVX_INS movsd, vmovsd, xmm(reg-ZREG_XMM0), qword [Z_ZV(addr)] +| .endif +|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { +| SSE_AVX_INS movsd, vmovsd, xmm(reg-ZREG_XMM0), qword [Ra(Z_REG(addr))+Z_OFFSET(addr)] +|| } else if (Z_MODE(addr) == IS_REG) { +| SSE_AVX_INS movsd, vmovaps, xmm(reg-ZREG_XMM0), xmm(Z_REG(addr)-ZREG_XMM0) +|| } else { +|| ZEND_ASSERT(0); +|| } || } |.endmacro @@ -369,11 +435,11 @@ static void* dasm_labels[zend_lb_MAX]; |.macro SSE_SET_ZVAL_DVAL, addr, reg || if (Z_MODE(addr) == IS_REG) { || if (reg != Z_REG(addr)) { -| movsd xmm(Z_REG(addr)-ZREG_XMM0), xmm(reg-ZREG_XMM0) +| SSE_AVX_INS movsd, vmovaps, xmm(Z_REG(addr)-ZREG_XMM0), xmm(reg-ZREG_XMM0) || } || } else { || ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); -| movsd qword [Ra(Z_REG(addr))+Z_OFFSET(addr)], xmm(reg-ZREG_XMM0) +| SSE_AVX_INS movsd, vmovsd, qword [Ra(Z_REG(addr))+Z_OFFSET(addr)], xmm(reg-ZREG_XMM0) || } |.endmacro @@ -419,6 +485,27 @@ static void* dasm_labels[zend_lb_MAX]; || } |.endmacro +|.macro AVX_MATH_REG, opcode, dst_reg, op1_reg, src_reg +|| switch (opcode) { +|| case ZEND_ADD: +|| case ZEND_ASSIGN_ADD: +| vaddsd xmm(dst_reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0) +|| break; +|| case ZEND_SUB: +|| case ZEND_ASSIGN_SUB: +| vsubsd xmm(dst_reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0) +|| break; +|| case ZEND_MUL: +|| case ZEND_ASSIGN_MUL: +| vmulsd xmm(dst_reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0) +|| break; +|| case ZEND_DIV: +|| case ZEND_ASSIGN_DIV: +| vdivsd xmm(dst_reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0) +|| break; +|| } +|.endmacro + |.macro LONG_OP, long_ins, reg, addr || if (Z_MODE(addr) == IS_CONST_ZVAL) { | .if X64 @@ -593,14 +680,18 @@ static void* dasm_labels[zend_lb_MAX]; || zend_reg dst_reg = (Z_MODE(dst_addr) == IS_REG) ? Z_REG(dst_addr) : ZREG_XMM0; | .if X64 or SSE || if (Z_DVAL_P(zv) == 0.0 && !is_signed(Z_DVAL_P(zv))) { -| xorps xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0) +|| if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { +| vxorps xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0) +|| } else { +| xorps xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0) +|| } | .if X64 || } else if (!IS_32BIT(zv)) { | mov64 tmp_reg, ((uintptr_t)zv) -| movsd xmm(dst_reg-ZREG_XMM0), qword [tmp_reg] +| SSE_AVX_INS movsd, vmovsd, xmm(dst_reg-ZREG_XMM0), qword [tmp_reg] | .endif || } else { -| movsd xmm(dst_reg-ZREG_XMM0), qword [((uint32_t)(uintptr_t)zv)] +| SSE_AVX_INS movsd, vmovsd, xmm(dst_reg-ZREG_XMM0), qword [((uint32_t)(uintptr_t)zv)] || } | SSE_SET_ZVAL_DVAL dst_addr, dst_reg | .else @@ -644,14 +735,18 @@ static void* dasm_labels[zend_lb_MAX]; || Z_REG(dst_addr) : ((Z_MODE(dst_addr) == IS_REG) ? Z_MODE(res_addr) : ZREG_XMM0); | .if X64 or SSE || if (Z_DVAL_P(zv) == 0.0 && !is_signed(Z_DVAL_P(zv))) { -| xorps xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0) +|| if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { +| vxorps xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0) +|| } else { +| xorps xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0) +|| } | .if X64 || } else if (!IS_32BIT(zv)) { | mov64 tmp_reg, ((uintptr_t)zv) -| movsd xmm(dst_reg-ZREG_XMM0), qword [tmp_reg] +| SSE_AVX_INS movsd, vmovsd, xmm(dst_reg-ZREG_XMM0), qword [tmp_reg] | .endif || } else { -| movsd xmm(dst_reg-ZREG_XMM0), qword [((uint32_t)(uintptr_t)zv)] +| SSE_AVX_INS movsd, vmovsd, xmm(dst_reg-ZREG_XMM0), qword [((uint32_t)(uintptr_t)zv)] || } | SSE_SET_ZVAL_DVAL dst_addr, ZREG_XMM0 | SSE_SET_ZVAL_DVAL res_addr, ZREG_XMM0 @@ -2116,7 +2211,7 @@ static int zend_jit_update_regs(dasm_State **Dst, zend_jit_addr src, zend_jit_ad | mov Ra(Z_REG(dst)), Ra(Z_REG(src)) } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) { | .if SSE - | movsd xmm(Z_REG(dst)-ZREG_XMM0), xmm(Z_REG(src)-ZREG_XMM0) + | SSE_AVX_INS movsd, vmovaps, xmm(Z_REG(dst)-ZREG_XMM0), xmm(Z_REG(src)-ZREG_XMM0) | .else || ZEND_ASSERT(0); | .endif @@ -2347,7 +2442,11 @@ static int zend_jit_math_long_long(dasm_State **Dst, | SSE_GET_ZVAL_LVAL tmp_reg1, op1_addr | SSE_GET_ZVAL_LVAL tmp_reg2, op2_addr - | SSE_MATH_REG opline->opcode, tmp_reg1, tmp_reg2 + if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { + | AVX_MATH_REG opline->opcode, tmp_reg1, tmp_reg1, tmp_reg2 + } else { + | SSE_MATH_REG opline->opcode, tmp_reg1, tmp_reg2 + } | SSE_SET_ZVAL_DVAL res_addr, tmp_reg1 |.else | FPU_GET_ZVAL_LVAL op2_addr @@ -2440,9 +2539,22 @@ static int zend_jit_math_double_long(dasm_State **Dst, } else { result_reg = ZREG_XMM0; } - | SSE_GET_ZVAL_DVAL result_reg, op1_addr - | SSE_GET_ZVAL_LVAL tmp_reg, op2_addr - | SSE_MATH_REG opline->opcode, result_reg, tmp_reg + if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { + zend_reg op1_reg; + + if (Z_MODE(op1_addr) == IS_REG) { + op1_reg = Z_REG(op1_addr); + } else { + | SSE_GET_ZVAL_DVAL result_reg, op1_addr + op1_reg = result_reg; + } + | SSE_GET_ZVAL_LVAL tmp_reg, op2_addr + | AVX_MATH_REG opline->opcode, result_reg, op1_reg, tmp_reg + } else { + | SSE_GET_ZVAL_DVAL result_reg, op1_addr + | SSE_GET_ZVAL_LVAL tmp_reg, op2_addr + | SSE_MATH_REG opline->opcode, result_reg, tmp_reg + } } | SSE_SET_ZVAL_DVAL res_addr, result_reg |.else @@ -2489,10 +2601,16 @@ static int zend_jit_math_double_double(dasm_State **Dst, result_reg = ZREG_XMM0; } - if (Z_MODE(op1_addr) == IS_REG - && Z_REG(op1_addr) != result_reg - && (zend_jit_x86_flags & ZEND_JIT_CPU_AVX)) { - | AVX_MATH opline->opcode, result_reg, Z_REG(op1_addr), op2_addr + if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { + zend_reg op1_reg; + + if (Z_MODE(op1_addr) == IS_REG) { + op1_reg = Z_REG(op1_addr); + } else { + | SSE_GET_ZVAL_DVAL result_reg, op1_addr + op1_reg =result_reg; + } + | AVX_MATH opline->opcode, result_reg, op1_reg, op2_addr } else { | SSE_GET_ZVAL_DVAL result_reg, op1_addr if (same_ops) { @@ -4593,7 +4711,7 @@ static int zend_jit_cmp_long_double(dasm_State **Dst, const zend_op *opline, int zend_reg tmp_reg = ZREG_XMM0; | SSE_GET_ZVAL_LVAL tmp_reg, op1_addr - | SSE_OP ucomisd, tmp_reg, op2_addr + | SSE_AVX_OP ucomisd, vucomisd, tmp_reg, op2_addr |.else | FPU_GET_ZVAL_DVAL op2_addr | FPU_GET_ZVAL_LVAL op1_addr @@ -4612,7 +4730,7 @@ static int zend_jit_cmp_double_long(dasm_State **Dst, const zend_op *opline, int zend_reg tmp_reg = ZREG_XMM0; | SSE_GET_ZVAL_LVAL tmp_reg, op2_addr - | SSE_OP ucomisd, tmp_reg, op1_addr + | SSE_AVX_OP ucomisd, vucomisd, tmp_reg, op1_addr swap = 1; |.else | FPU_GET_ZVAL_LVAL op2_addr @@ -4630,15 +4748,15 @@ static int zend_jit_cmp_double_double(dasm_State **Dst, const zend_op *opline, i |.if SSE if (Z_MODE(op1_addr) == IS_REG) { - | SSE_OP ucomisd, Z_REG(op1_addr), op2_addr + | SSE_AVX_OP ucomisd, vucomisd, Z_REG(op1_addr), op2_addr } else if (Z_MODE(op2_addr) == IS_REG) { - | SSE_OP ucomisd, Z_REG(op2_addr), op1_addr + | SSE_AVX_OP ucomisd, vucomisd, Z_REG(op2_addr), op1_addr swap = 1; } else { zend_reg tmp_reg = ZREG_XMM0; | SSE_GET_ZVAL_DVAL tmp_reg, op1_addr - | SSE_OP ucomisd, tmp_reg, op2_addr + | SSE_AVX_OP ucomisd, vucomisd, tmp_reg, op2_addr } |.else | FPU_GET_ZVAL_DVAL op2_addr @@ -5560,8 +5678,12 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, int b, if ((op1_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { |.if SSE - | xorps xmm0, xmm0 - | SSE_OP ucomisd, ZREG_XMM0, op1_addr + if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { + | vxorps xmm0, xmm0, xmm0 + } else { + | xorps xmm0, xmm0 + } + | SSE_AVX_OP ucomisd, vucomisd, ZREG_XMM0, op1_addr |.else | FPU_GET_ZVAL_DVAL op1_addr | fldz From 83860d37ed8b5165290835c1eaf526b93e65e62d Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 7 Apr 2017 16:16:02 +0300 Subject: [PATCH 417/569] Use AVX instructions if available --- ext/opcache/jit/zend_jit_x86.dasc | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 5be23872b6b2a..e9dd2e4d74773 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -2492,7 +2492,11 @@ static int zend_jit_math_long_double(dasm_State **Dst, (Z_MODE(res_addr) == IS_REG) ? Z_REG(res_addr) : ZREG_XMM0; | SSE_GET_ZVAL_LVAL result_reg, op1_addr - | SSE_MATH opline->opcode, result_reg, op2_addr + if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { + | AVX_MATH opline->opcode, result_reg, result_reg, op2_addr + } else { + | SSE_MATH opline->opcode, result_reg, op2_addr + } | SSE_SET_ZVAL_DVAL res_addr, result_reg |.else | FPU_GET_ZVAL_LVAL op1_addr @@ -2528,7 +2532,11 @@ static int zend_jit_math_double_long(dasm_State **Dst, result_reg = ZREG_XMM0; } | SSE_GET_ZVAL_LVAL result_reg, op2_addr - | SSE_MATH opline->opcode, result_reg, op1_addr + if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { + | AVX_MATH opline->opcode, result_reg, result_reg, op1_addr + } else { + | SSE_MATH opline->opcode, result_reg, op1_addr + } } else { zend_reg tmp_reg = ZREG_XMM1; From 548873820c9b1447544b610e89f65d33434be1f7 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 10 Apr 2017 21:09:17 +0300 Subject: [PATCH 418/569] Micro optimization for commutative operators --- ext/opcache/jit/zend_jit.c | 9 +++++++++ ext/opcache/jit/zend_jit_x86.dasc | 12 +++++++++--- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index a639e367046b6..28f449f91e90d 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -98,6 +98,15 @@ static zend_bool zend_ssa_is_last_use(zend_op_array *op_array, const zend_ssa *s return use < 0 || zend_ssa_is_no_val_use(op_array->opcodes + use, ssa->ops + use, var); } +static zend_bool zend_is_commutative(zend_uchar opcode) +{ + return + opcode == ZEND_ADD || + opcode == ZEND_MUL || + opcode == ZEND_ASSIGN_ADD || + opcode == ZEND_ASSIGN_MUL; +} + #include "dynasm/dasm_x86.h" #include "jit/zend_jit_x86.h" #include "jit/zend_jit_helpers.c" diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index e9dd2e4d74773..0243d26011c77 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -2525,7 +2525,7 @@ static int zend_jit_math_double_long(dasm_State **Dst, |.if SSE zend_reg result_reg; - if (opline->opcode == ZEND_ADD || opline->opcode == ZEND_MUL || opline->opcode == ZEND_ASSIGN_ADD || opline->opcode == ZEND_ASSIGN_MUL) { + if (zend_is_commutative(opline->opcode)) { if (Z_MODE(res_addr) == IS_REG) { result_reg = Z_REG(res_addr); } else { @@ -2611,14 +2611,20 @@ static int zend_jit_math_double_double(dasm_State **Dst, if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { zend_reg op1_reg; + zend_jit_addr val_addr; if (Z_MODE(op1_addr) == IS_REG) { op1_reg = Z_REG(op1_addr); + val_addr = op2_addr; + } else if (Z_MODE(op2_addr) == IS_REG && zend_is_commutative(opline->opcode)) { + op1_reg = Z_REG(op2_addr); + val_addr = op1_addr; } else { | SSE_GET_ZVAL_DVAL result_reg, op1_addr - op1_reg =result_reg; + op1_reg = result_reg; + val_addr = op2_addr; } - | AVX_MATH opline->opcode, result_reg, op1_reg, op2_addr + | AVX_MATH opline->opcode, result_reg, op1_reg, val_addr } else { | SSE_GET_ZVAL_DVAL result_reg, op1_addr if (same_ops) { From 04a13ea503feaa064e0cb5b797d0c0f6abdba5f2 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 11 Apr 2017 00:21:40 +0300 Subject: [PATCH 419/569] Don't reuse registers used as "hint" if other registers are available --- ext/opcache/jit/zend_jit.c | 74 ++++++++++++++++++++++++++++++++------ ext/opcache/jit/zend_jit.h | 1 + 2 files changed, 65 insertions(+), 10 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 28f449f91e90d..8b3e18af9f24b 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -1003,6 +1003,7 @@ static int zend_jit_add_range(zend_lifetime_interval **intervals, int var, uint3 ival->range.end = to; ival->range.next = NULL; ival->hint = NULL; + ival->used_as_hint = NULL; intervals[var] = ival; } else if (ival->range.start > to + 1) { zend_life_range *range = zend_arena_alloc(&CG(arena), sizeof(zend_life_range)); @@ -1526,6 +1527,17 @@ static int zend_jit_compute_liveness(zend_op_array *op_array, zend_ssa *ssa, zen } *list = zend_jit_sort_intervals(intervals, ssa->vars_count); + + if (*list) { + zend_lifetime_interval *ival = *list; + while (ival) { + if (ival->hint) { + ival->hint->used_as_hint = ival; + } + ival = ival->list_next; + } + } + free_alloca(intervals, use_heap); return SUCCESS; @@ -1581,13 +1593,14 @@ static uint32_t zend_interval_intersection(zend_lifetime_interval *ival1, zend_l /* See "Optimized Interval Splitting in a Linear Scan Register Allocator", Christian Wimmer VEE'05 (2005), Figure 4. Allocation without spilling */ -static int zend_jit_try_allocate_free_reg(zend_op_array *op_array, zend_ssa *ssa, zend_lifetime_interval *current, zend_regset available, zend_lifetime_interval *active, zend_lifetime_interval *inactive, zend_lifetime_interval **list, zend_lifetime_interval **free) +static int zend_jit_try_allocate_free_reg(zend_op_array *op_array, zend_ssa *ssa, zend_lifetime_interval *current, zend_regset available, zend_regset *hints, zend_lifetime_interval *active, zend_lifetime_interval *inactive, zend_lifetime_interval **list, zend_lifetime_interval **free) { zend_lifetime_interval *it; uint32_t freeUntilPos[ZREG_NUM]; - uint32_t pos; - zend_reg i, reg; + uint32_t pos, pos2; + zend_reg i, reg, reg2; zend_reg hint = ZREG_NONE; + zend_regset low_priority_regs; zend_life_range *range; if ((ssa->var_info[current->ssa_var].type & MAY_BE_ANY) == MAY_BE_DOUBLE) { @@ -1618,7 +1631,12 @@ static int zend_jit_try_allocate_free_reg(zend_op_array *op_array, zend_ssa *ssa if (current->range.start != zend_interval_end(it)) { freeUntilPos[it->reg] = 0; } else if (zend_jit_may_reuse_reg(op_array, ssa, current->range.start, current->ssa_var, it->ssa_var)) { - hint = it->reg; + if (!ZEND_REGSET_IN(*hints, it->reg) && + /* TODO: Avoid most often scratch registers. Find a better way ??? */ + (!current->used_as_hint || + (it->reg != ZREG_R0 && it->reg != ZREG_R1 && it->reg != ZREG_XMM0 && it->reg != ZREG_XMM1))) { + hint = it->reg; + } } else { freeUntilPos[it->reg] = 0; } @@ -1629,8 +1647,11 @@ static int zend_jit_try_allocate_free_reg(zend_op_array *op_array, zend_ssa *ssa freeUntilPos[it->reg] = 0; it = it->list_next; } - if (current->hint) { - hint = current->hint->reg; + } + if (current->hint) { + hint = current->hint->reg; + if (current->hint->used_as_hint == current) { + ZEND_REGSET_EXCL(*hints, hint); } } @@ -1712,14 +1733,40 @@ static int zend_jit_try_allocate_free_reg(zend_op_array *op_array, zend_ssa *ssa if (hint != ZREG_NONE && freeUntilPos[hint] > zend_interval_end(current)) { current->reg = hint; + if (current->used_as_hint) { + ZEND_REGSET_INCL(*hints, hint); + } return 1; } pos = 0; reg = ZREG_NONE; + pos2 = 0; reg2 = ZREG_NONE; + low_priority_regs = *hints; + if (current->used_as_hint) { + /* TODO: Avoid most often scratch registers. Find a better way ??? */ + ZEND_REGSET_INCL(low_priority_regs, ZREG_R0); + ZEND_REGSET_INCL(low_priority_regs, ZREG_R1); + ZEND_REGSET_INCL(low_priority_regs, ZREG_XMM0); + ZEND_REGSET_INCL(low_priority_regs, ZREG_XMM1); + } for (i = 0; i < ZREG_NUM; i++) { - if (ZEND_REGSET_IN(available, i) && freeUntilPos[i] > pos) { - reg = i; - pos = freeUntilPos[i]; + if (ZEND_REGSET_IN(available, i)) { + if (ZEND_REGSET_IN(low_priority_regs, i)) { + if (freeUntilPos[i] > pos2) { + reg2 = i; + pos2 = freeUntilPos[i]; + } + } else if (freeUntilPos[i] > pos) { + reg = i; + pos = freeUntilPos[i]; + } + } + } + + if (reg == ZREG_NONE) { + if (reg2 != ZREG_NONE) { + reg = reg2; + pos = pos2; } } @@ -1729,6 +1776,9 @@ static int zend_jit_try_allocate_free_reg(zend_op_array *op_array, zend_ssa *ssa } else if (zend_interval_end(current) < pos) { /* register available for the whole interval */ current->reg = reg; + if (current->used_as_hint) { + ZEND_REGSET_INCL(*hints, reg); + } return 1; } else { /* TODO: enable interval splitting ??? */ @@ -1737,6 +1787,9 @@ static int zend_jit_try_allocate_free_reg(zend_op_array *op_array, zend_ssa *ssa return 0; } current->reg = reg; + if (current->used_as_hint) { + ZEND_REGSET_INCL(*hints, reg); + } return 1; } } @@ -1759,6 +1812,7 @@ static zend_lifetime_interval* zend_jit_linear_scan(zend_op_array *op_array, zen zend_lifetime_interval *current, **p, *q; uint32_t position; zend_regset available = ZEND_REGSET_UNION(ZEND_REGSET_GP, ZEND_REGSET_FP); + zend_regset hints = ZEND_REGSET_EMPTY; unhandled = list; /* active = inactive = handled = free = {} */ @@ -1811,7 +1865,7 @@ static zend_lifetime_interval* zend_jit_linear_scan(zend_op_array *op_array, zen } } - if (zend_jit_try_allocate_free_reg(op_array, ssa, current, available, active, inactive, &unhandled, &free) || + if (zend_jit_try_allocate_free_reg(op_array, ssa, current, available, &hints, active, inactive, &unhandled, &free) || zend_jit_allocate_blocked_reg()) { ZEND_REGSET_EXCL(available, current->reg); current->list_next = active; diff --git a/ext/opcache/jit/zend_jit.h b/ext/opcache/jit/zend_jit.h index 4702967941229..bb5cce2604b2f 100644 --- a/ext/opcache/jit/zend_jit.h +++ b/ext/opcache/jit/zend_jit.h @@ -102,6 +102,7 @@ struct _zend_lifetime_interval { zend_bool load; zend_life_range range; zend_lifetime_interval *hint; + zend_lifetime_interval *used_as_hint; zend_lifetime_interval *list_next; }; From 21652c0f8b233ba02b21872ddc994171643a336f Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 11 Apr 2017 01:13:11 +0300 Subject: [PATCH 420/569] Mimnimal JIT support for SWITCH_LONG and SWITCH_STRING (just ignore them for now) --- ext/opcache/jit/zend_jit.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 8b3e18af9f24b..7cda5df630a73 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -517,6 +517,8 @@ static int zend_may_throw(const zend_op *opline, zend_op_array *op_array, zend_s case ZEND_DEFINED: case ZEND_ISSET_ISEMPTY_THIS: case ZEND_COALESCE: + case ZEND_SWITCH_LONG: + case ZEND_SWITCH_STRING: return 0; case ZEND_INIT_FCALL: /* can't throw, because call is resolved at compile time */ @@ -1352,7 +1354,7 @@ static int zend_jit_compute_liveness(zend_op_array *op_array, zend_ssa *ssa, zen /* live = UNION of successor.liveIn for each successor of b */ /* live.add(phi.inputOf(b)) for each phi of successors of b */ zend_bitset_clear(live, set_size); - for (j = 0; j < 2 && b->successors[j] >= 0; j++) { + for (j = 0; j < b->successors_count; j++) { int succ = b->successors[j]; zend_bitset_union(live, live_in + set_size * succ, set_size); @@ -2486,6 +2488,8 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa, const zend_op *rt_op break; case ZEND_NOP: case ZEND_OP_DATA: + case ZEND_SWITCH_LONG: + case ZEND_SWITCH_STRING: break; case ZEND_JMP: if (!zend_jit_jmp(&dasm_state, ssa->cfg.blocks[b].successors[0])) { From a8fea9984c32249328441c11a6db14c0c09a7251 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 11 Apr 2017 02:20:56 +0300 Subject: [PATCH 421/569] Fixed lifetinme interval construction --- ext/opcache/jit/zend_jit.c | 1 + 1 file changed, 1 insertion(+) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 7cda5df630a73..eb8fac3b6e2c0 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -1038,6 +1038,7 @@ static int zend_jit_add_range(zend_lifetime_interval **intervals, int var, uint3 if (range->start > to + 1) { break; } + last->end = range->end; range = range->next; last->next = range; } From 12fa042f3db6ebb0f479fece1a5c9659a6449adb Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 11 Apr 2017 13:05:36 +0300 Subject: [PATCH 422/569] Micro optimizaton for commutative operaions Replace ($x * 2.0) by ($x + $x) --- ext/opcache/jit/zend_jit.c | 2 ++ ext/opcache/jit/zend_jit_x86.dasc | 22 +++++++++++++++++++--- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index eb8fac3b6e2c0..90243a194a522 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -1967,6 +1967,7 @@ static zend_lifetime_interval** zend_jit_allocate_registers(zend_op_array *op_ar fprintf(stderr, "\n"); ival = ival->list_next; } + fprintf(stderr, "\n"); } /* Linear Scan Register Allocation */ @@ -2091,6 +2092,7 @@ static zend_lifetime_interval** zend_jit_allocate_registers(zend_op_array *op_ar ival = ival->list_next; } } + fprintf(stderr, "\n"); } free_alloca(candidates, use_heap); diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 0243d26011c77..90d10afa69415 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -2624,13 +2624,29 @@ static int zend_jit_math_double_double(dasm_State **Dst, op1_reg = result_reg; val_addr = op2_addr; } - | AVX_MATH opline->opcode, result_reg, op1_reg, val_addr + if ((opline->opcode == ZEND_MUL || opline->opcode == ZEND_ASSIGN_MUL) && + Z_MODE(val_addr) == IS_CONST_ZVAL && Z_DVAL_P(Z_ZV(val_addr)) == 2.0) { + | AVX_MATH_REG ZEND_ADD, result_reg, op1_reg, op1_reg + } else { + | AVX_MATH opline->opcode, result_reg, op1_reg, val_addr + } } else { - | SSE_GET_ZVAL_DVAL result_reg, op1_addr + zend_jit_addr val_addr; + + if (Z_MODE(op1_addr) != IS_REG && Z_MODE(op2_addr) == IS_REG && zend_is_commutative(opline->opcode)) { + | SSE_GET_ZVAL_DVAL result_reg, op2_addr + val_addr = op1_addr; + } else { + | SSE_GET_ZVAL_DVAL result_reg, op1_addr + val_addr = op2_addr; + } if (same_ops) { | SSE_MATH_REG opline->opcode, result_reg, result_reg + } else if ((opline->opcode == ZEND_MUL || opline->opcode == ZEND_ASSIGN_MUL) && + Z_MODE(val_addr) == IS_CONST_ZVAL && Z_DVAL_P(Z_ZV(val_addr)) == 2.0) { + | SSE_MATH_REG ZEND_ADD, result_reg, result_reg } else { - | SSE_MATH opline->opcode, result_reg, op2_addr + | SSE_MATH opline->opcode, result_reg, val_addr } } | SSE_SET_ZVAL_DVAL res_addr, result_reg From 6ebd2847a5ffb9831e6a4a170cad9c3aaa96fdfd Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 11 Apr 2017 17:22:12 +0300 Subject: [PATCH 423/569] Reload elimination --- ext/opcache/jit/zend_jit.c | 21 +++++++++++++++++++++ ext/opcache/jit/zend_jit_x86.dasc | 6 +++++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 90243a194a522..1385869b34710 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -2055,6 +2055,27 @@ static zend_lifetime_interval** zend_jit_allocate_registers(zend_op_array *op_ar } } } + /* Load elimination */ + for (i = 0; i < ssa->vars_count; i++) { + if (intervals[i] && + intervals[i]->load && + ssa->vars[i].use_chain < 0) { + zend_bool may_remove = 1; + zend_ssa_phi *phi = ssa->vars[i].phi_use_chain; + + while (phi) { + if (intervals[phi->ssa_var] && + !intervals[phi->ssa_var]->load) { + may_remove = 0; + break; + } + phi = zend_ssa_next_use_phi(ssa, i, phi); + } + if (may_remove) { + intervals[i] = NULL; + } + } + } } if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_REG_ALLOC) { diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 90d10afa69415..bb25e8d16e968 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -2265,7 +2265,11 @@ static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, zend_op_arr | ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, MAY_BE_LONG, ZREG_R0, ZREG_R1 } if (op1_addr != op1_def_addr) { - | ZVAL_COPY_VALUE op1_def_addr, op1_info, op1_addr, MAY_BE_LONG, ZREG_R0, ZREG_R1 + if (Z_MODE(op1_addr) != IS_REG && Z_MODE(res_addr) == IS_REG) { + | ZVAL_COPY_VALUE op1_def_addr, op1_info, res_addr, MAY_BE_LONG, ZREG_R0, ZREG_R1 + } else { + | ZVAL_COPY_VALUE op1_def_addr, op1_info, op1_addr, MAY_BE_LONG, ZREG_R0, ZREG_R1 + } } if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { | LONG_OP_WITH_CONST add, op1_def_addr, Z_L(1) From efe918d8e6524cb169b3dac43415f155f41a029b Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 12 Apr 2017 12:48:44 +0300 Subject: [PATCH 424/569] Improve regset traversing --- ext/opcache/jit/zend_jit.c | 49 +++++++++++++++------------------- ext/opcache/jit/zend_jit_x86.h | 15 +++++++++++ 2 files changed, 36 insertions(+), 28 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 1385869b34710..bba3885fecb9a 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -1207,16 +1207,14 @@ static void zend_jit_print_regset(zend_regset regset) zend_reg reg; int first = 1; - for (reg = 0; reg < ZREG_NUM; reg++) { - if (ZEND_REGSET_IN(regset, reg)) { - if (first) { - first = 0; - fprintf(stderr, "%s", zend_reg_name[reg]); - } else { - fprintf(stderr, ", %s", zend_reg_name[reg]); - } + ZEND_REGSET_FOREACH(regset, reg) { + if (first) { + first = 0; + fprintf(stderr, "%s", zend_reg_name[reg]); + } else { + fprintf(stderr, ", %s", zend_reg_name[reg]); } - } + } ZEND_REGSET_FOREACH_END(); } static int *zend_jit_compute_block_order_int(zend_ssa *ssa, int n, int *block_order) @@ -1690,15 +1688,11 @@ static int zend_jit_try_allocate_free_reg(zend_op_array *op_array, zend_ssa *ssa } while (line <= range->end) { regset = zend_jit_get_scratch_regset(op_array, ssa, line, current->ssa_var); - if (!ZEND_REGSET_IS_EMPTY(regset)) { - for (reg = 0; reg < ZREG_NUM; reg++) { - if (ZEND_REGSET_IN(regset, reg)) { - if (line < freeUntilPos[reg]) { - freeUntilPos[reg] = line; - } - } + ZEND_REGSET_FOREACH(regset, reg) { + if (line < freeUntilPos[reg]) { + freeUntilPos[reg] = line; } - } + } ZEND_REGSET_FOREACH_END(); line++; } range = range->next; @@ -1752,19 +1746,18 @@ static int zend_jit_try_allocate_free_reg(zend_op_array *op_array, zend_ssa *ssa ZEND_REGSET_INCL(low_priority_regs, ZREG_XMM0); ZEND_REGSET_INCL(low_priority_regs, ZREG_XMM1); } - for (i = 0; i < ZREG_NUM; i++) { - if (ZEND_REGSET_IN(available, i)) { - if (ZEND_REGSET_IN(low_priority_regs, i)) { - if (freeUntilPos[i] > pos2) { - reg2 = i; - pos2 = freeUntilPos[i]; - } - } else if (freeUntilPos[i] > pos) { - reg = i; - pos = freeUntilPos[i]; + + ZEND_REGSET_FOREACH(available, i) { + if (ZEND_REGSET_IN(low_priority_regs, i)) { + if (freeUntilPos[i] > pos2) { + reg2 = i; + pos2 = freeUntilPos[i]; } + } else if (freeUntilPos[i] > pos) { + reg = i; + pos = freeUntilPos[i]; } - } + } ZEND_REGSET_FOREACH_END(); if (reg == ZREG_NONE) { if (reg2 != ZREG_NONE) { diff --git a/ext/opcache/jit/zend_jit_x86.h b/ext/opcache/jit/zend_jit_x86.h index d83c3d8eda3ad..3fe55e9349430 100644 --- a/ext/opcache/jit/zend_jit_x86.h +++ b/ext/opcache/jit/zend_jit_x86.h @@ -145,6 +145,21 @@ typedef uint32_t zend_regset; (ZEND_REGSET(ZREG_RBX) | ZEND_REGSET(ZREG_RBP)) #endif +#define ZEND_REGSET_FIRST(set) ((zend_reg)__builtin_ctz(set)) +#define ZEND_REGSET_LAST(set) ((zend_reg)(__builtin_clz(set)^31))) + +#define ZEND_REGSET_FOREACH(set, reg) \ + do { \ + zend_regset _tmp = (set); \ + while (!ZEND_REGSET_IS_EMPTY(_tmp)) { \ + zend_reg _reg = ZEND_REGSET_FIRST(_tmp); \ + ZEND_REGSET_EXCL(_tmp, _reg); \ + reg = _reg; \ + +#define ZEND_REGSET_FOREACH_END() \ + } \ + } while (0) + typedef uintptr_t zend_jit_addr; #define IS_CONST_ZVAL 0 From f6472e415de8f709798831a75c47d0885a2b87f0 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 12 Apr 2017 17:22:48 +0300 Subject: [PATCH 425/569] Store elimination --- ext/opcache/jit/zend_jit_x86.dasc | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index bb25e8d16e968..e0def4ab51f11 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -2202,7 +2202,7 @@ static int zend_jit_load_ssa_var(dasm_State **Dst, zend_ssa *ssa, int var, zend_ return zend_jit_load_reg(Dst, src, dst, info); } -static int zend_jit_update_regs(dasm_State **Dst, zend_jit_addr src, zend_jit_addr dst, uint32_t info) +static int zend_jit_update_regs(dasm_State **Dst, zend_jit_addr src, zend_jit_addr dst, uint32_t info, zend_lifetime_interval *src_ival) { if (src != dst) { if (Z_MODE(src) == IS_REG) { @@ -2219,8 +2219,10 @@ static int zend_jit_update_regs(dasm_State **Dst, zend_jit_addr src, zend_jit_ad ZEND_ASSERT(0); } } else if (Z_MODE(dst) == IS_MEM_ZVAL) { - if (!zend_jit_spill_store(Dst, src, dst, info, 1)) { - return 0; + if (!src_ival->load && !src_ival->store) { + if (!zend_jit_spill_store(Dst, src, dst, info, 1)) { + return 0; + } } } else { ZEND_ASSERT(0); @@ -5874,7 +5876,7 @@ static int zend_jit_qm_assign(dasm_State **Dst, const zend_op *opline, zend_op_a && !ssa->vars[ssa->ops[opline - op_array->opcodes].op1_def].no_val) { zend_jit_addr op1_def_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, ra, ssa->ops[opline - op_array->opcodes].op1_def); - if (!zend_jit_update_regs(Dst, op1_addr, op1_def_addr, op1_info)) { + if (!zend_jit_update_regs(Dst, op1_addr, op1_def_addr, op1_info, ra[ssa->ops[opline - op_array->opcodes].op1_use])) { return 0; } } @@ -5913,7 +5915,7 @@ static int zend_jit_assign(dasm_State **Dst, const zend_op *opline, zend_op_arra && !ssa->vars[ssa->ops[opline - op_array->opcodes].op2_def].no_val) { zend_jit_addr op2_def_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2, opline, ra, ssa->ops[opline - op_array->opcodes].op2_def); - if (!zend_jit_update_regs(Dst, op2_addr, op2_def_addr, op2_info)) { + if (!zend_jit_update_regs(Dst, op2_addr, op2_def_addr, op2_info, ra[ssa->ops[opline - op_array->opcodes].op2_use])) { return 0; } } @@ -6989,7 +6991,7 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, zend_op_ar && !ssa->vars[ssa->ops[opline - op_array->opcodes].op1_def].no_val) { zend_jit_addr op1_def_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, ra, ssa->ops[opline - op_array->opcodes].op1_def); - if (!zend_jit_update_regs(Dst, op1_addr, op1_def_addr, op1_info)) { + if (!zend_jit_update_regs(Dst, op1_addr, op1_def_addr, op1_info, ra[ssa->ops[opline - op_array->opcodes].op1_use])) { return 0; } if (Z_MODE(op1_def_addr) == IS_REG && Z_MODE(op1_addr) != IS_REG) { From 5573e08ba8281f56f5035712bb606dcb5fa2bc3c Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 12 Apr 2017 18:07:01 +0300 Subject: [PATCH 426/569] Use "xor reg, reg" instead of "mov $0, reg" --- ext/opcache/jit/zend_jit_x86.dasc | 52 ++++++++++++++++++++++++------- 1 file changed, 40 insertions(+), 12 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index e0def4ab51f11..5ff2f05b67924 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -557,15 +557,19 @@ static void* dasm_labels[zend_lb_MAX]; |.macro GET_ZVAL_LVAL, reg, addr || if (Z_MODE(addr) == IS_CONST_ZVAL) { -| .if X64 -|| if (!IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(addr)))) { -| mov64 Ra(reg), Z_LVAL_P(Z_ZV(addr)) -|| } else { +|| if (Z_LVAL_P(Z_ZV(addr)) == 0) { +| xor Ra(reg), Ra(reg) +|| } else { +| .if X64 +|| if (!IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(addr)))) { +| mov64 Ra(reg), Z_LVAL_P(Z_ZV(addr)) +|| } else { +| mov Ra(reg), Z_LVAL_P(Z_ZV(addr)) +|| } +| .else | mov Ra(reg), Z_LVAL_P(Z_ZV(addr)) -|| } -| .else -| mov Ra(reg), Z_LVAL_P(Z_ZV(addr)) -| .endif +| .endif +|| } || } else if (Z_MODE(addr) == IS_MEM_ZVAL) { | mov Ra(reg), aword [Ra(Z_REG(addr))+Z_OFFSET(addr)] || } else if (Z_MODE(addr) == IS_REG) { @@ -704,6 +708,8 @@ static void* dasm_labels[zend_lb_MAX]; || } | FPU_SET_ZVAL_DVAL dst_addr | .endif +|| } else if (Z_LVAL_P(zv) == 0 && Z_MODE(dst_addr) == IS_REG) { +| xor Ra(Z_REG(dst_addr)), Ra(Z_REG(dst_addr)) || } else { | .if X64 || if (!IS_SIGNED_32BIT(Z_LVAL_P(zv))) { @@ -768,6 +774,14 @@ static void* dasm_labels[zend_lb_MAX]; || } | FPU_SET_ZVAL_DVAL res_addr | .endif +|| } else if (Z_LVAL_P(zv) == 0 && (Z_MODE(dst_addr) == IS_REG || Z_MODE(res_addr) == IS_REG)) { +|| if (Z_MODE(dst_addr) == IS_REG) { +| xor Ra(Z_REG(dst_addr)), Ra(Z_REG(dst_addr)) +| SET_ZVAL_LVAL res_addr, Ra(Z_REG(dst_addr)) +|| } else { +| xor Ra(Z_REG(res_addr)), Ra(Z_REG(res_addr)) +| SET_ZVAL_LVAL dst_addr, Ra(Z_REG(res_addr)) +|| } || } else { | .if X64 || if (!IS_SIGNED_32BIT(Z_LVAL_P(zv))) { @@ -782,13 +796,27 @@ static void* dasm_labels[zend_lb_MAX]; | SET_ZVAL_LVAL dst_addr, tmp_reg | SET_ZVAL_LVAL res_addr, tmp_reg || } +|| } else if (Z_MODE(dst_addr) == IS_REG) { +| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv) +| SET_ZVAL_LVAL res_addr, Ra(Z_REG(dst_addr)) +|| } else if (Z_MODE(res_addr) == IS_REG) { +| SET_ZVAL_LVAL res_addr, Z_LVAL_P(zv) +| SET_ZVAL_LVAL dst_addr, Ra(Z_REG(res_addr)) || } else { | SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv) | SET_ZVAL_LVAL res_addr, Z_LVAL_P(zv) || } | .else -| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv) -| SET_ZVAL_LVAL res_addr, Z_LVAL_P(zv) +|| if (Z_MODE(dst_addr) == IS_REG) { +| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv) +| SET_ZVAL_LVAL res_addr, Ra(Z_REG(dst_addr)) +|| } else if (Z_MODE(res_addr) == IS_REG) { +| SET_ZVAL_LVAL res_addr, Z_LVAL_P(zv) +| SET_ZVAL_LVAL dst_addr, Ra(Z_REG(res_addr)) +|| } else { +| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv) +| SET_ZVAL_LVAL res_addr, Z_LVAL_P(zv) +|| } | .endif || } || } @@ -1093,7 +1121,7 @@ static void* dasm_labels[zend_lb_MAX]; || if (filename) { | LOAD_ADDR FCARG2a, ZSTR_VAL((zend_string*)filename) || } else { -| mov FCARG2a, 0 +| xor FCARG2a, FCARG2a || } | .if X64 | mov CARG3d, lineno @@ -1184,7 +1212,7 @@ static void* dasm_labels[zend_lb_MAX]; || if (filename) { | LOAD_ADDR FCARG2a, ZSTR_VAL((zend_string*)filename) || } else { -| mov FCARG2a, 0 +| xor FCARG2a, FCARG2a || } | .if X64 | mov CARG3d, lineno From 6db7fae04a9e5aa3f6f3689e727c61576fe3b6de Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 12 Apr 2017 18:31:24 +0300 Subject: [PATCH 427/569] Use "test reg, reg" instead of "cmp $0, reg" --- ext/opcache/jit/zend_jit_x86.dasc | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 5ff2f05b67924..c203f9c822e13 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -3191,7 +3191,11 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o | // if (EXPECTED((zend_ulong)(_h) < (zend_ulong)(_ht)->nNumUsed)) |.if X64 | movsxd r0, dword [FCARG1a + offsetof(zend_array, nNumUsed)] - | cmp r0, val + if (val == 0) { + | test r0, r0 + } else { + | cmp r0, val + } |.else | cmp dword [FCARG1a + offsetof(zend_array, nNumUsed)], val |.endif @@ -4305,9 +4309,17 @@ static int zend_jit_cmp_long_long(dasm_State **Dst, const zend_op *opline, int b zend_bool swap = 0; if (Z_MODE(op1_addr) == IS_REG) { - | LONG_OP cmp, Ra(Z_REG(op1_addr)), op2_addr + if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { + | test Ra(Z_REG(op1_addr)), Ra(Z_REG(op1_addr)) + } else { + | LONG_OP cmp, Ra(Z_REG(op1_addr)), op2_addr + } } else if (Z_MODE(op2_addr) == IS_REG) { - | LONG_OP cmp, Ra(Z_REG(op2_addr)), op1_addr + if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op1_addr)) == 0) { + | test Ra(Z_REG(op2_addr)), Ra(Z_REG(op2_addr)) + } else { + | LONG_OP cmp, Ra(Z_REG(op2_addr)), op1_addr + } swap = 1; } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) != IS_CONST_ZVAL) { | LONG_OP_WITH_CONST cmp, op2_addr, Z_LVAL_P(Z_ZV(op1_addr)) @@ -4316,7 +4328,11 @@ static int zend_jit_cmp_long_long(dasm_State **Dst, const zend_op *opline, int b | LONG_OP_WITH_CONST cmp, op1_addr, Z_LVAL_P(Z_ZV(op2_addr)) } else { | GET_ZVAL_LVAL ZREG_R0, op1_addr - | LONG_OP cmp, r0, op2_addr + if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { + | test r0, r0 + } else { + | LONG_OP cmp, r0, op2_addr + } } if (((opline+1)->opcode == ZEND_JMPZ_EX || (opline+1)->opcode == ZEND_JMPNZ_EX) && @@ -8086,7 +8102,7 @@ static int zend_jit_bind_global(dasm_State **Dst, const zend_op *opline, zend_op //value = Z_INDIRECT_P(value) | mov r0, [r0] | mov cl, byte [r0 + 8] - | cmp cl, IS_UNDEF + | test cl, cl // cmp cl, IS_UNDEF | jne >2 | SET_Z_TYPE_INFO r0, IS_NULL | jmp >8 From ad461c4e248c9e554a57bc4fdb025d16aea64e4f Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 13 Apr 2017 13:06:19 +0300 Subject: [PATCH 428/569] Eliminate useless Z_TYPE_P() writes --- ext/opcache/jit/zend_jit.c | 11 +++++++++++ ext/opcache/jit/zend_jit_x86.dasc | 24 +++++++++++++++++------- 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index bba3885fecb9a..eadcb6373b8b4 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -1763,6 +1763,7 @@ static int zend_jit_try_allocate_free_reg(zend_op_array *op_array, zend_ssa *ssa if (reg2 != ZREG_NONE) { reg = reg2; pos = pos2; + reg2 = ZREG_NONE; } } @@ -1776,6 +1777,16 @@ static int zend_jit_try_allocate_free_reg(zend_op_array *op_array, zend_ssa *ssa ZEND_REGSET_INCL(*hints, reg); } return 1; +#if 0 + // TODO: allow low prioirity register usage + } else if (reg2 != ZREG_NONE && zend_interval_end(current) < pos2) { + /* register available for the whole interval */ + current->reg = reg2; + if (current->used_as_hint) { + ZEND_REGSET_INCL(*hints, reg2); + } + return 1; +#endif } else { /* TODO: enable interval splitting ??? */ /* register available for the first part of the interval */ diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index c203f9c822e13..51c0a8db5ba1d 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -2210,13 +2210,23 @@ static int zend_jit_store_ssa_var(dasm_State **Dst, zend_ssa *ssa, int var, zend return zend_jit_spill_store(Dst, src, dst, info, 1); } -static int zend_jit_store_ssa_var_if_necessary(dasm_State **Dst, zend_ssa *ssa, zend_lifetime_interval **ra, zend_jit_addr src, int var) +static int zend_jit_store_ssa_var_if_necessary(dasm_State **Dst, zend_ssa *ssa, zend_lifetime_interval **ra, zend_jit_addr src, int var, int old_var) { if (Z_MODE(src) == IS_REG && ra[var] && ra[var]->store) { zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, (uint32_t)(uintptr_t)ZEND_CALL_VAR_NUM(NULL, ssa->vars[var].var)); uint32_t info = ssa->var_info[var].type; - - return zend_jit_spill_store(Dst, src, dst, info, 1); + zend_bool set_type = 1; + + if (old_var >= 0) { + uint32_t old_info = ssa->var_info[old_var].type; + if ((info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) == + (old_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF))) { + if (!ra[old_var] || ra[old_var]->load || ra[old_var]->store) { + set_type = 0; + } + } + } + return zend_jit_spill_store(Dst, src, dst, info, set_type); } return 1; } @@ -2403,11 +2413,11 @@ static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, zend_op_arr |.code } |3: - if (ra && !zend_jit_store_ssa_var_if_necessary(Dst, ssa, ra, op1_def_addr, ssa->ops[opline - op_array->opcodes].op1_def)) { + if (ra && !zend_jit_store_ssa_var_if_necessary(Dst, ssa, ra, op1_def_addr, ssa->ops[opline - op_array->opcodes].op1_def, ssa->ops[opline - op_array->opcodes].op1_use)) { return 0; } if (opline->result_type != IS_UNUSED) { - if (ra && !zend_jit_store_ssa_var_if_necessary(Dst, ssa, ra, res_addr, ssa->ops[opline - op_array->opcodes].result_def)) { + if (ra && !zend_jit_store_ssa_var_if_necessary(Dst, ssa, ra, res_addr, ssa->ops[opline - op_array->opcodes].result_def, ssa->ops[opline - op_array->opcodes].result_use)) { return 0; } } @@ -3033,7 +3043,7 @@ static int zend_jit_math(dasm_State **Dst, const zend_op *opline, int *opnum, ze if (!zend_jit_math_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, opline->result.var, res_addr, RES_INFO(), res_use_info, 0)) { return 0; } - if (ra && !zend_jit_store_ssa_var_if_necessary(Dst, ssa, ra, res_addr, ssa->ops[opline - op_array->opcodes].result_def)) { + if (ra && !zend_jit_store_ssa_var_if_necessary(Dst, ssa, ra, res_addr, ssa->ops[opline - op_array->opcodes].result_def, ssa->ops[opline - op_array->opcodes].result_use)) { return 0; } return 1; @@ -5928,7 +5938,7 @@ static int zend_jit_qm_assign(dasm_State **Dst, const zend_op *opline, zend_op_a if (!zend_jit_simple_assign(Dst, opline, op_array, ssa, res_addr, -1, opline->op1_type, opline->op1, op1_addr, op1_info, 0, 0)) { return 0; } - if (ra && !zend_jit_store_ssa_var_if_necessary(Dst, ssa, ra, res_addr, ssa->ops[opline - op_array->opcodes].result_def)) { + if (ra && !zend_jit_store_ssa_var_if_necessary(Dst, ssa, ra, res_addr, ssa->ops[opline - op_array->opcodes].result_def, ssa->ops[opline - op_array->opcodes].result_use)) { return 0; } return 1; From de44ac9c1ec7c2325d2730ebf52d357a6909bdf6 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 13 Apr 2017 16:43:46 +0300 Subject: [PATCH 429/569] Remove some useless (unused) allocations --- ext/opcache/jit/zend_jit.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index eadcb6373b8b4..5bca4322c6e21 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -2059,10 +2059,11 @@ static zend_lifetime_interval** zend_jit_allocate_registers(zend_op_array *op_ar } } } - /* Load elimination */ + /* Remove useless register allocation */ for (i = 0; i < ssa->vars_count; i++) { if (intervals[i] && - intervals[i]->load && + (intervals[i]->load || + intervals[i]->store && ssa->vars[i].definition >= 0) && ssa->vars[i].use_chain < 0) { zend_bool may_remove = 1; zend_ssa_phi *phi = ssa->vars[i].phi_use_chain; From 3e8958bf781be9b78e8aca3006b98474d8744387 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 14 Apr 2017 10:49:52 +0300 Subject: [PATCH 430/569] Store elimination --- ext/opcache/jit/zend_jit_x86.dasc | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 51c0a8db5ba1d..74d63d4c8f471 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -2305,10 +2305,15 @@ static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, zend_op_arr | ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, MAY_BE_LONG, ZREG_R0, ZREG_R1 } if (op1_addr != op1_def_addr) { - if (Z_MODE(op1_addr) != IS_REG && Z_MODE(res_addr) == IS_REG) { - | ZVAL_COPY_VALUE op1_def_addr, op1_info, res_addr, MAY_BE_LONG, ZREG_R0, ZREG_R1 + if (Z_MODE(op1_addr) != IS_REG && Z_MODE(res_addr) == IS_REG && + (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC)) { + if (!zend_jit_update_regs(Dst, res_addr, op1_def_addr, MAY_BE_LONG, ra[ssa->ops[opline - op_array->opcodes].op1_use])) { + return 0; + } } else { - | ZVAL_COPY_VALUE op1_def_addr, op1_info, op1_addr, MAY_BE_LONG, ZREG_R0, ZREG_R1 + if (!zend_jit_update_regs(Dst, op1_addr, op1_def_addr, MAY_BE_LONG, ra[ssa->ops[opline - op_array->opcodes].op1_use])) { + return 0; + } } } if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { From 47a0a4d30595fe571addc36352c0f758e764f570 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 14 Apr 2017 11:38:39 +0300 Subject: [PATCH 431/569] Added register hinting for implicit copy opearations --- ext/opcache/jit/zend_jit.c | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 5bca4322c6e21..e9d395946d1e9 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -1522,6 +1522,37 @@ static int zend_jit_compute_liveness(zend_op_array *op_array, zend_ssa *ssa, zen } } } + } else if (ssa->vars[var].definition >= 0) { + uint32_t line = ssa->vars[var].definition; + const zend_op *opline = op_array->opcodes + line; + + switch (opline->opcode) { + case ZEND_QM_ASSIGN: + case ZEND_SEND_VAR: + if (i == ssa->ops[line].op1_def && + ssa->ops[line].op1_use >= 0 && + intervals[ssa->ops[line].op1_use]) { + zend_jit_add_hint(intervals, i, ssa->ops[line].op1_use); + } + break; + case ZEND_PRE_INC: + case ZEND_PRE_DEC: + case ZEND_POST_INC: + case ZEND_POST_DEC: + if (i == ssa->ops[line].op1_def && + ssa->ops[line].op1_use >= 0 && + intervals[ssa->ops[line].op1_use]) { + zend_jit_add_hint(intervals, i, ssa->ops[line].op1_use); + } + break; + case ZEND_ASSIGN: + if (i == ssa->ops[line].op2_def && + ssa->ops[line].op2_use >= 0 && + intervals[ssa->ops[line].op2_use]) { + zend_jit_add_hint(intervals, i, ssa->ops[line].op2_use); + } + break; + } } } } From cc4d944cd326e08f8cb2543e60120ea2a0fb9424 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 14 Apr 2017 12:44:23 +0300 Subject: [PATCH 432/569] Added register hinting for implicit and explicit copy opearations --- ext/opcache/jit/zend_jit.c | 42 +++++++++++++++++++++++++++----------- 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index e9d395946d1e9..a51f25301309f 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -1522,33 +1522,51 @@ static int zend_jit_compute_liveness(zend_op_array *op_array, zend_ssa *ssa, zen } } } - } else if (ssa->vars[var].definition >= 0) { + } + } + } + for (i = 0; i < ssa->vars_count; i++) { + if (intervals[i] && !intervals[i]->hint) { + int var = intervals[i]->ssa_var; + + if (ssa->vars[var].definition >= 0) { uint32_t line = ssa->vars[var].definition; const zend_op *opline = op_array->opcodes + line; switch (opline->opcode) { case ZEND_QM_ASSIGN: - case ZEND_SEND_VAR: - if (i == ssa->ops[line].op1_def && - ssa->ops[line].op1_use >= 0 && - intervals[ssa->ops[line].op1_use]) { + case ZEND_POST_INC: + case ZEND_POST_DEC: + if (ssa->ops[line].op1_use >= 0 && + intervals[ssa->ops[line].op1_use] && + (var == ssa->ops[line].op1_def || + (var == ssa->ops[line].result_def && + (ssa->ops[line].op1_def < 0 || + !intervals[ssa->ops[line].op1_def])))) { zend_jit_add_hint(intervals, i, ssa->ops[line].op1_use); } break; + case ZEND_SEND_VAR: case ZEND_PRE_INC: case ZEND_PRE_DEC: - case ZEND_POST_INC: - case ZEND_POST_DEC: - if (i == ssa->ops[line].op1_def && + if (var == ssa->ops[line].op1_def && ssa->ops[line].op1_use >= 0 && intervals[ssa->ops[line].op1_use]) { zend_jit_add_hint(intervals, i, ssa->ops[line].op1_use); } break; case ZEND_ASSIGN: - if (i == ssa->ops[line].op2_def && - ssa->ops[line].op2_use >= 0 && - intervals[ssa->ops[line].op2_use]) { + if (ssa->ops[line].op2_use >= 0 && + intervals[ssa->ops[line].op2_use] && + (var == ssa->ops[line].op2_def || + (var == ssa->ops[line].op1_def && + (ssa->ops[line].op2_def < 0 || + !intervals[ssa->ops[line].op2_def])) || + (var == ssa->ops[line].result_def && + (ssa->ops[line].op2_def < 0 || + !intervals[ssa->ops[line].op2_def]) && + (ssa->ops[line].op1_def < 0 || + !intervals[ssa->ops[line].op1_def])))) { zend_jit_add_hint(intervals, i, ssa->ops[line].op2_use); } break; @@ -2094,7 +2112,7 @@ static zend_lifetime_interval** zend_jit_allocate_registers(zend_op_array *op_ar for (i = 0; i < ssa->vars_count; i++) { if (intervals[i] && (intervals[i]->load || - intervals[i]->store && ssa->vars[i].definition >= 0) && + (intervals[i]->store && ssa->vars[i].definition >= 0)) && ssa->vars[i].use_chain < 0) { zend_bool may_remove = 1; zend_ssa_phi *phi = ssa->vars[i].phi_use_chain; From 481c78c0ffd5e8b778790dd14c53fd3055baeafa Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 14 Apr 2017 12:45:02 +0300 Subject: [PATCH 433/569] Improved scratch registers assignment/reservation --- ext/opcache/jit/zend_jit_x86.dasc | 156 ++++++++++++++++++++++-------- 1 file changed, 113 insertions(+), 43 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 74d63d4c8f471..0dd8b558a9281 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -2587,14 +2587,17 @@ static int zend_jit_math_double_long(dasm_State **Dst, | SSE_MATH opline->opcode, result_reg, op1_addr } } else { - zend_reg tmp_reg = ZREG_XMM1; + zend_reg tmp_reg; if (Z_MODE(res_addr) == IS_REG) { result_reg = Z_REG(res_addr); + tmp_reg = ZREG_XMM0; } else if (Z_MODE(op1_addr) == IS_REG && zend_ssa_is_last_use(op_array, ssa, ssa->ops[opline-op_array->opcodes].op1_use, opline-op_array->opcodes)) { result_reg = Z_REG(op1_addr); + tmp_reg = ZREG_XMM0; } else { result_reg = ZREG_XMM0; + tmp_reg = ZREG_XMM1; } if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { zend_reg op1_reg; @@ -7618,9 +7621,15 @@ static int zend_jit_return(dasm_State **Dst, const zend_op *opline, zend_op_arra } // if (!EX(return_value)) - | mov r1, EX->return_value - | test r1, r1 - ret_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R1, 0); + if (Z_MODE(op1_addr) == IS_REG && Z_REG(op1_addr) == ZREG_R1) { + | mov r2, EX->return_value + | test r2, r2 + ret_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R2, 0); + } else { + | mov r1, EX->return_value + | test r1, r1 + ret_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R1, 0); + } if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { | jz >1 @@ -8708,6 +8717,19 @@ static zend_bool zend_jit_may_be_in_reg(zend_op_array *op_array, zend_ssa *ssa, return 1; } +static zend_bool zend_needs_extra_reg_for_const(zend_op_array *op_array, zend_uchar op_type, znode_op op) +{ +|.if X64 +|| if (op_type == IS_CONST) { +|| zval *zv = RT_CONSTANT(op_array, op); +|| if (Z_TYPE_P(zv) == IS_DOUBLE && !Z_DVAL_Z(zv) == 0 && !IS_32BIT(zv)) { +|| return 1; +|| } +|| } +|.endif + return 0; +} + static zend_regset zend_jit_get_scratch_regset(zend_op_array *op_array, zend_ssa *ssa, uint32_t line, int current_var) { const zend_op *opline = op_array->opcodes + line; @@ -8718,19 +8740,25 @@ static zend_regset zend_jit_get_scratch_regset(zend_op_array *op_array, zend_ssa case ZEND_NOP: case ZEND_OP_DATA: case ZEND_JMP: - regset = ZEND_REGSET_EMPTY; - break; case ZEND_RETURN: - regset = ZEND_REGSET(ZREG_R1); + regset = ZEND_REGSET_EMPTY; break; case ZEND_QM_ASSIGN: + if (ssa->ops[line].op1_def == current_var || + ssa->ops[line].result_def == current_var) { + regset = ZEND_REGSET_EMPTY; + break; + } + /* break missing intentionally */ case ZEND_SEND_VAL: case ZEND_SEND_VAL_EX: + if (ssa->ops[line].op1_use == current_var) { + regset = ZEND_REGSET_EMPTY; + break; + } op1_info = OP1_INFO(); if (!(op1_info & MAY_BE_UNDEF)) { - if (ssa->ops[line].op1_use == current_var) { - regset = ZEND_REGSET_EMPTY; - } else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) { + if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) { regset = ZEND_REGSET(ZREG_XMM0); } else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) { regset = ZEND_REGSET(ZREG_R0); @@ -8740,11 +8768,14 @@ static zend_regset zend_jit_get_scratch_regset(zend_op_array *op_array, zend_ssa } break; case ZEND_SEND_VAR: + if (ssa->ops[line].op1_use == current_var || + ssa->ops[line].op1_def == current_var) { + regset = ZEND_REGSET_EMPTY; + break; + } op1_info = OP1_INFO(); if (!(op1_info & MAY_BE_UNDEF)) { - if (ssa->ops[line].op1_use == current_var) { - regset = ZEND_REGSET_EMPTY; - } else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) { + if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) { regset = ZEND_REGSET(ZREG_XMM0); } else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) { regset = ZEND_REGSET(ZREG_R0); @@ -8757,14 +8788,19 @@ static zend_regset zend_jit_get_scratch_regset(zend_op_array *op_array, zend_ssa } break; case ZEND_ASSIGN: + if (ssa->ops[line].op2_use == current_var || + ssa->ops[line].op2_def == current_var || + ssa->ops[line].op1_def == current_var || + ssa->ops[line].result_def == current_var) { + regset = ZEND_REGSET_EMPTY; + break; + } op1_info = OP1_INFO(); op2_info = OP2_INFO(); if (opline->op1_type == IS_CV && !(op2_info & MAY_BE_UNDEF) && !(op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_RESOURCE|MAY_BE_REF))) { - if (ssa->ops[line].op2_use == current_var) { - regset = ZEND_REGSET_EMPTY; - } else if ((op2_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) { + if ((op2_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) { regset = ZEND_REGSET(ZREG_XMM0); } else if ((op2_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) { regset = ZEND_REGSET(ZREG_R0); @@ -8777,14 +8813,16 @@ static zend_regset zend_jit_get_scratch_regset(zend_op_array *op_array, zend_ssa case ZEND_PRE_DEC: case ZEND_POST_INC: case ZEND_POST_DEC: + if (ssa->ops[line].op1_use == current_var || + ssa->ops[line].op1_def == current_var || + ssa->ops[line].result_def == current_var) { + regset = ZEND_REGSET_EMPTY; + break; + } op1_info = OP1_INFO(); if (opline->op1_type == IS_CV && !(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG))) { regset = ZEND_REGSET_EMPTY; - if (opline->result_type != IS_UNUSED) { - ZEND_REGSET_INCL(regset, ZREG_R0); - ZEND_REGSET_INCL(regset, ZREG_R1); - } } break; case ZEND_ADD: @@ -8794,23 +8832,46 @@ static zend_regset zend_jit_get_scratch_regset(zend_op_array *op_array, zend_ssa op2_info = OP1_INFO(); if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) && !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { - res_info = OP1_INFO(); - if (res_info & MAY_BE_LONG) { - regset = ZEND_REGSET(ZREG_R0); + + regset = ZEND_REGSET_EMPTY; + if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)) { + if (ssa->ops[line].result_def != current_var && + (ssa->ops[line].op1_use != current_var || !zend_ssa_is_last_use(op_array, ssa, current_var, opline-op_array->opcodes))) { + ZEND_REGSET_INCL(regset, ZREG_R0); + } + res_info = OP1_INFO(); if (res_info & MAY_BE_DOUBLE) { ZEND_REGSET_INCL(regset, ZREG_XMM0); ZEND_REGSET_INCL(regset, ZREG_XMM1); } - } else { - regset = ZEND_REGSET(ZREG_XMM0); - if (opline->opcode == ZEND_SUB && (op2_info & MAY_BE_LONG)) { - ZEND_REGSET_INCL(regset, ZREG_XMM1); + } + if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_DOUBLE)) { + if (ssa->ops[line].result_def != current_var) { + ZEND_REGSET_INCL(regset, ZREG_XMM0); + } + } + if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_LONG)) { + if (zend_is_commutative(opline->opcode)) { + if (ssa->ops[line].result_def != current_var) { + ZEND_REGSET_INCL(regset, ZREG_XMM0); + } + } else { + ZEND_REGSET_INCL(regset, ZREG_XMM0); + if (ssa->ops[line].result_def != current_var && + (ssa->ops[line].op1_use != current_var || !zend_ssa_is_last_use(op_array, ssa, current_var, opline-op_array->opcodes))) { + ZEND_REGSET_INCL(regset, ZREG_XMM1); + } } - |.if X64 - || if (opline->op1_type == IS_CONST || opline->op2_type == IS_CONST) { - || ZEND_REGSET_INCL(regset, ZREG_R0); - || } - |.endif + } + if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_DOUBLE)) { + if (ssa->ops[line].result_def != current_var && + (ssa->ops[line].op1_use != current_var || !zend_ssa_is_last_use(op_array, ssa, current_var, opline-op_array->opcodes))) { + ZEND_REGSET_INCL(regset, ZREG_XMM0); + } + } + if (zend_needs_extra_reg_for_const(op_array, opline->op1_type, opline->op1) || + zend_needs_extra_reg_for_const(op_array, opline->op1_type, opline->op2)) { + ZEND_REGSET_INCL(regset, ZREG_R0); } } break; @@ -8826,19 +8887,28 @@ static zend_regset zend_jit_get_scratch_regset(zend_op_array *op_array, zend_ssa if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) && !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { regset = ZEND_REGSET_EMPTY; - if ((op1_info & MAY_BE_LONG) && - (op2_info & MAY_BE_LONG) && - opline->op1_type != IS_CONST && - opline->op2_type != IS_CONST) { - ZEND_REGSET_INCL(regset, ZREG_R0); + if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && + opline->op1_type != IS_CONST && opline->op2_type != IS_CONST) { + if (ssa->ops[line].op1_use != current_var && + ssa->ops[line].op2_use != current_var) { + ZEND_REGSET_INCL(regset, ZREG_R0); + } } - if ((op1_info & MAY_BE_DOUBLE) || (op2_info & MAY_BE_DOUBLE)) { + if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_DOUBLE)) { ZEND_REGSET_INCL(regset, ZREG_XMM0); - |.if X64 - || if (opline->op1_type == IS_CONST || opline->op2_type == IS_CONST) { - || ZEND_REGSET_INCL(regset, ZREG_R0); - || } - |.endif + } + if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_LONG)) { + ZEND_REGSET_INCL(regset, ZREG_XMM0); + } + if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_DOUBLE)) { + if (ssa->ops[line].op1_use != current_var && + ssa->ops[line].op2_use != current_var) { + ZEND_REGSET_INCL(regset, ZREG_XMM0); + } + } + if (zend_needs_extra_reg_for_const(op_array, opline->op1_type, opline->op1) || + zend_needs_extra_reg_for_const(op_array, opline->op1_type, opline->op2)) { + ZEND_REGSET_INCL(regset, ZREG_R0); } } break; From fc358c98c4aa8f43f9a78d34c96f21211ef6622e Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 14 Apr 2017 16:13:11 +0300 Subject: [PATCH 434/569] Fixed live-interval construction --- ext/opcache/jit/zend_jit.c | 78 ++++++++++++++++++++++++-------------- 1 file changed, 49 insertions(+), 29 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index a51f25301309f..0d247eec30fc9 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -1064,14 +1064,26 @@ static int zend_jit_add_range(zend_lifetime_interval **intervals, int var, uint3 return SUCCESS; } -static int zend_jit_begin_range(zend_lifetime_interval **intervals, int var, uint32_t from) +static int zend_jit_begin_range(zend_lifetime_interval **intervals, int var, uint32_t block_start, uint32_t from) { - if (intervals[var]) { + if (block_start != from && intervals[var]) { zend_life_range *range = &intervals[var]->range; do { if (from >= range->start && from <= range->end) { - range->start = from; + if (range->start == block_start) { + range->start = from; + } else { + zend_life_range *r = zend_arena_alloc(&CG(arena), sizeof(zend_life_range)); + if (!r) { + return FAILURE; + } + r->start = from; + r->end = range->end; + r->next = range->next; + range->end = block_start - 1; + range->next = r; + } return SUCCESS; } range = range->next; @@ -1410,19 +1422,19 @@ static int zend_jit_compute_liveness(zend_op_array *op_array, zend_ssa *ssa, zen /* setFrom(opd, op) */ /* live.remove(opd) */ if (op->op1_def >= 0 && zend_bitset_in(candidates, op->op1_def)) { - if (zend_jit_begin_range(intervals, op->op1_def, num) != SUCCESS) { + if (zend_jit_begin_range(intervals, op->op1_def, b->start, num) != SUCCESS) { goto failure; } zend_bitset_excl(live, op->op1_def); } if (op->op2_def >= 0 && zend_bitset_in(candidates, op->op2_def)) { - if (zend_jit_begin_range(intervals, op->op2_def, num) != SUCCESS) { + if (zend_jit_begin_range(intervals, op->op2_def, b->start, num) != SUCCESS) { goto failure; } zend_bitset_excl(live, op->op2_def); } if (op->result_def >= 0 && zend_bitset_in(candidates, op->result_def)) { - if (zend_jit_begin_range(intervals, op->result_def, num) != SUCCESS) { + if (zend_jit_begin_range(intervals, op->result_def, b->start, num) != SUCCESS) { goto failure; } zend_bitset_excl(live, op->result_def); @@ -1462,33 +1474,41 @@ static int zend_jit_compute_liveness(zend_op_array *op_array, zend_ssa *ssa, zen zend_bitset_excl(live, phi->ssa_var); } + /* b.liveIn = live */ + zend_bitset_copy(live_in + set_size * i, live, set_size); + } + + for (i = ssa->cfg.blocks_count - 1; i >= 0; i--) { + zend_basic_block *b = ssa->cfg.blocks + i; + /* if b is loop header */ - if ((b->flags & ZEND_BB_LOOP_HEADER) && - !zend_bitset_empty(live, set_size)) { - uint32_t set_size2 = zend_bitset_len(op_array->last); - - zend_bitset_clear(loop_body, set_size2); - zend_jit_compute_loop_body(ssa, i, i, loop_body); - while (!zend_bitset_empty(loop_body, set_size2)) { - uint32_t from = zend_bitset_first(loop_body, set_size2); - uint32_t to = from; - - do { - zend_bitset_excl(loop_body, to); - to++; - } while (zend_bitset_in(loop_body, to)); - to--; - - ZEND_BITSET_FOREACH(live, set_size, j) { - if (zend_jit_add_range(intervals, j, from, to) != SUCCESS) { - goto failure; - } - } ZEND_BITSET_FOREACH_END(); + if ((b->flags & ZEND_BB_LOOP_HEADER)) { + live = live_in + set_size * i; + + if (!zend_bitset_empty(live, set_size)) { + uint32_t set_size2 = zend_bitset_len(op_array->last); + + zend_bitset_clear(loop_body, set_size2); + zend_jit_compute_loop_body(ssa, i, i, loop_body); + while (!zend_bitset_empty(loop_body, set_size2)) { + uint32_t from = zend_bitset_first(loop_body, set_size2); + uint32_t to = from; + + do { + zend_bitset_excl(loop_body, to); + to++; + } while (zend_bitset_in(loop_body, to)); + to--; + + ZEND_BITSET_FOREACH(live, set_size, j) { + if (zend_jit_add_range(intervals, j, from, to) != SUCCESS) { + goto failure; + } + } ZEND_BITSET_FOREACH_END(); + } } } - /* b.liveIn = live */ - zend_bitset_copy(live_in + set_size * i, live, set_size); } if (zend_jit_reg_alloc >= ZEND_JIT_REG_ALLOC_GLOBAL) { From 89a867d85ee005e4ef56991b3b5ac31b56a91e8d Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 14 Apr 2017 16:23:43 +0300 Subject: [PATCH 435/569] typo --- ext/opcache/jit/zend_jit_x86.dasc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 0dd8b558a9281..4b32377a6eb3a 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -8722,7 +8722,7 @@ static zend_bool zend_needs_extra_reg_for_const(zend_op_array *op_array, zend_uc |.if X64 || if (op_type == IS_CONST) { || zval *zv = RT_CONSTANT(op_array, op); -|| if (Z_TYPE_P(zv) == IS_DOUBLE && !Z_DVAL_Z(zv) == 0 && !IS_32BIT(zv)) { +|| if (Z_TYPE_P(zv) == IS_DOUBLE && Z_DVAL_P(zv) != 0 && !IS_32BIT(zv)) { || return 1; || } || } From 8944f82703e663eb2f2cd6013b7035ed258879ea Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 14 Apr 2017 17:16:08 +0300 Subject: [PATCH 436/569] Don't allocate registers for spilled intervals used once --- ext/opcache/jit/zend_jit.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 0d247eec30fc9..e94fc4fdd144d 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -2150,6 +2150,16 @@ static zend_lifetime_interval** zend_jit_allocate_registers(zend_op_array *op_ar } } } + /* Remove intervals used once */ + for (i = 0; i < ssa->vars_count; i++) { + if (intervals[i] && + intervals[i]->load && + intervals[i]->store && + (ssa->vars[i].use_chain < 0 || + zend_ssa_next_use(ssa->ops, i, ssa->vars[i].use_chain) < 0)) { + intervals[i] = NULL; + } + } } if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_REG_ALLOC) { From 25f97813f37bf6e682a911eb50ee8383c91085d2 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 17 Apr 2017 10:49:35 +0300 Subject: [PATCH 437/569] Don't split CFG at live range boundaries --- ext/opcache/jit/zend_jit.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index e94fc4fdd144d..eebab256e9b0f 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -917,7 +917,7 @@ static int zend_may_overflow(const zend_op *opline, zend_op_array *op_array, zen static int zend_jit_build_cfg(zend_op_array *op_array, zend_cfg *cfg, uint32_t *flags) { - if (zend_build_cfg(&CG(arena), op_array, ZEND_CFG_STACKLESS | ZEND_CFG_RECV_ENTRY | ZEND_RT_CONSTANTS | ZEND_CFG_SPLIT_AT_LIVE_RANGES | ZEND_CFG_NO_ENTRY_PREDECESSORS | ZEND_SSA_RC_INFERENCE_FLAG | ZEND_SSA_USE_CV_RESULTS, cfg, flags) != SUCCESS) { + if (zend_build_cfg(&CG(arena), op_array, ZEND_CFG_STACKLESS | ZEND_CFG_RECV_ENTRY | ZEND_RT_CONSTANTS | ZEND_CFG_NO_ENTRY_PREDECESSORS | ZEND_SSA_RC_INFERENCE_FLAG | ZEND_SSA_USE_CV_RESULTS, cfg, flags) != SUCCESS) { return FAILURE; } From a5224b564c2fa177802776d848ecd7ffb0fde445 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 17 Apr 2017 12:51:26 +0300 Subject: [PATCH 438/569] Add extra case for JIT INC/DEC IS_DOUBLE --- ext/opcache/jit/zend_jit_x86.dasc | 61 ++++++++++++++++++++++++++++--- 1 file changed, 55 insertions(+), 6 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 4b32377a6eb3a..299b40f1e1f40 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -1848,6 +1848,13 @@ static int zend_jit_not_obj_stub(dasm_State **Dst) return 1; } +static int zend_jit_double_one_stub(dasm_State **Dst) +{ + |->one: + |.dword 0, 0x3ff00000 + return 1; +} + static const zend_jit_stub zend_jit_stubs[] = { JIT_STUB(interrupt_handler), JIT_STUB(exception_handler), @@ -1863,6 +1870,7 @@ static const zend_jit_stub zend_jit_stubs[] = { JIT_STUB(undefined_index_ex), JIT_STUB(cannot_add_element_ex), JIT_STUB(not_obj), + JIT_STUB(double_one), }; static int zend_jit_setup(void) @@ -2404,10 +2412,47 @@ static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, zend_op_arr | SEPARATE_ZVAL_NOREF var_addr, op1_info, 0, op_array->filename, opline->lineno } - if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { - | EXT_CALL increment_function, r0 + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) { + if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { + | EXT_CALL increment_function, r0 + } else { + | EXT_CALL decrement_function, r0 + } } else { - | EXT_CALL decrement_function, r0 + zend_reg tmp_reg; + if (Z_MODE(op1_def_addr) == IS_REG) { + tmp_reg = Z_REG(op1_def_addr); + } else if (Z_MODE(op1_addr) == IS_REG && zend_ssa_is_last_use(op_array, ssa, ssa->ops[opline-op_array->opcodes].op1_use, opline-op_array->opcodes)) { + tmp_reg = Z_REG(op1_addr); + } else { + tmp_reg = ZREG_XMM0; + } + |.if SSE + | SSE_GET_ZVAL_DVAL tmp_reg, op1_addr + if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { + if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { + | vaddsd xmm(tmp_reg-ZREG_XMM0), xmm(tmp_reg-ZREG_XMM0), qword [->one] + } else { + | addsd xmm(tmp_reg-ZREG_XMM0), qword [->one] + } + } else { + if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { + | vsubsd xmm(tmp_reg-ZREG_XMM0), xmm(tmp_reg-ZREG_XMM0), qword [->one] + } else { + | subsd xmm(tmp_reg-ZREG_XMM0), qword [->one] + } + } + | SSE_SET_ZVAL_DVAL op1_def_addr, tmp_reg + |.else + | FPU_GET_ZVAL_DVAL op1_addr + | fld1 + if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { + | fadd + } else { + | fsub + } + | FPU_SET_ZVAL_DVAL op1_def_addr + |.endif } if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && opline->result_type != IS_UNUSED) { @@ -8651,7 +8696,8 @@ static zend_bool zend_jit_opline_supports_reg(zend_op_array *op_array, zend_ssa op1_info = OP1_INFO(); return opline->op1_type == IS_CV && - !(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG))); + (op1_info & MAY_BE_LONG) && + !(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))); case ZEND_BOOL: case ZEND_BOOL_NOT: case ZEND_JMPZ: @@ -8778,7 +8824,6 @@ static zend_regset zend_jit_get_scratch_regset(zend_op_array *op_array, zend_ssa if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) { regset = ZEND_REGSET(ZREG_XMM0); } else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) { - regset = ZEND_REGSET(ZREG_R0); } else { regset = ZEND_REGSET_UNION(ZEND_REGSET(ZREG_R0), ZEND_REGSET(ZREG_R2)); if (op1_info & MAY_BE_REF) { @@ -8821,8 +8866,12 @@ static zend_regset zend_jit_get_scratch_regset(zend_op_array *op_array, zend_ssa } op1_info = OP1_INFO(); if (opline->op1_type == IS_CV - && !(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG))) { + && (op1_info & MAY_BE_LONG) + && !(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { regset = ZEND_REGSET_EMPTY; + if (op1_info & MAY_BE_DOUBLE) { + regset = ZEND_REGSET(ZREG_XMM0); + } } break; case ZEND_ADD: From ac2ff1ccb55e3ed4f4d16d057b3417c84886694e Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 18 Apr 2017 11:25:57 +0300 Subject: [PATCH 439/569] JIT for BW_OR, BW_AND, BW_XOR, SL, SR, MOD and corresponding ASSIGN_OP (incomplete for some edge cases) --- ext/opcache/jit/zend_jit.c | 27 +- ext/opcache/jit/zend_jit_disasm_x86.c | 6 + ext/opcache/jit/zend_jit_helpers.c | 336 +++++++++++++++++++ ext/opcache/jit/zend_jit_x86.dasc | 454 +++++++++++++++++++++----- 4 files changed, 743 insertions(+), 80 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index eebab256e9b0f..d5574c582a14c 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -103,8 +103,19 @@ static zend_bool zend_is_commutative(zend_uchar opcode) return opcode == ZEND_ADD || opcode == ZEND_MUL || + opcode == ZEND_BW_OR || + opcode == ZEND_BW_AND || + opcode == ZEND_BW_XOR || opcode == ZEND_ASSIGN_ADD || - opcode == ZEND_ASSIGN_MUL; + opcode == ZEND_ASSIGN_MUL|| + opcode == ZEND_ASSIGN_BW_OR || + opcode == ZEND_ASSIGN_BW_AND || + opcode == ZEND_ASSIGN_BW_XOR; +} + +static zend_bool zend_long_is_power_of_two(zend_long x) +{ + return (x > 0) && !(x & (x - 1)); } #include "dynasm/dasm_x86.h" @@ -2392,9 +2403,13 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa, const zend_op *rt_op goto jit_failure; } goto done; - case ZEND_SR: + case ZEND_BW_OR: + case ZEND_BW_AND: + case ZEND_BW_XOR: case ZEND_SL: - if (!zend_jit_shift(&dasm_state, opline, op_array, ssa)) { + case ZEND_SR: + case ZEND_MOD: + if (!zend_jit_long_math(&dasm_state, opline, &i, op_array, ssa, ra)) { goto jit_failure; } goto done; @@ -2417,6 +2432,12 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa, const zend_op *rt_op case ZEND_ASSIGN_MUL: // case ZEND_ASSIGN_DIV: // TODO: check for division by zero ??? case ZEND_ASSIGN_CONCAT: + case ZEND_ASSIGN_BW_OR: + case ZEND_ASSIGN_BW_AND: + case ZEND_ASSIGN_BW_XOR: + case ZEND_ASSIGN_SL: + case ZEND_ASSIGN_SR: + case ZEND_ASSIGN_MOD: if (!zend_jit_assign_op(&dasm_state, opline, op_array, ssa)) { goto jit_failure; } diff --git a/ext/opcache/jit/zend_jit_disasm_x86.c b/ext/opcache/jit/zend_jit_disasm_x86.c index 20497a55c535b..54ec42ec50d84 100644 --- a/ext/opcache/jit/zend_jit_disasm_x86.c +++ b/ext/opcache/jit/zend_jit_disasm_x86.c @@ -408,6 +408,12 @@ static int zend_jit_disasm_init(void) REGISTER_HELPER(zend_jit_assign_dim_sub_helper); REGISTER_HELPER(zend_jit_assign_dim_mul_helper); REGISTER_HELPER(zend_jit_assign_dim_div_helper); + REGISTER_HELPER(zend_jit_assign_dim_bw_or_helper); + REGISTER_HELPER(zend_jit_assign_dim_bw_and_helper); + REGISTER_HELPER(zend_jit_assign_dim_bw_xor_helper); + REGISTER_HELPER(zend_jit_assign_dim_sl_helper); + REGISTER_HELPER(zend_jit_assign_dim_sr_helper); + REGISTER_HELPER(zend_jit_assign_dim_mod_helper); REGISTER_HELPER(zend_jit_assign_dim_concat_helper); REGISTER_HELPER(zend_jit_fast_assign_concat_helper); REGISTER_HELPER(zend_jit_fast_concat_helper); diff --git a/ext/opcache/jit/zend_jit_helpers.c b/ext/opcache/jit/zend_jit_helpers.c index ef20240aabd79..3433b4b20d2f3 100644 --- a/ext/opcache/jit/zend_jit_helpers.c +++ b/ext/opcache/jit/zend_jit_helpers.c @@ -1179,6 +1179,342 @@ static void ZEND_FASTCALL zend_jit_assign_dim_div_helper(zval *container, zval * } } +static void ZEND_FASTCALL zend_jit_assign_dim_bw_or_helper(zval *container, zval *dim, zval *value) +{ + if (EXPECTED(Z_TYPE_P(container) == IS_OBJECT)) { + zval *object = container; + zval *property = dim; + zval *z; + zval rv, res; + + if (Z_OBJ_HT_P(object)->read_dimension && + (z = Z_OBJ_HT_P(object)->read_dimension(object, property, BP_VAR_R, &rv)) != NULL) { + + if (Z_TYPE_P(z) == IS_OBJECT && Z_OBJ_HT_P(z)->get) { + zval rv2; + zval *value = Z_OBJ_HT_P(z)->get(z, &rv2); + + if (z == &rv) { + zval_ptr_dtor(&rv); + } + ZVAL_COPY_VALUE(z, value); + } + bitwise_or_function(&res, Z_ISREF_P(z) ? Z_REFVAL_P(z) : z, value); + Z_OBJ_HT_P(object)->write_dimension(object, property, &res); + if (z == &rv) { + zval_ptr_dtor(&rv); + } +//??? if (retval) { +//??? ZVAL_COPY(retval, &res); +//??? } + zval_ptr_dtor(&res); + } else { + zend_error(E_WARNING, "Attempt to assign property of non-object"); +//??? if (retval) { +//??? ZVAL_NULL(retval); +//??? } + } + } else { + if (UNEXPECTED(Z_TYPE_P(container) == IS_STRING)) { + if (!dim) { + zend_throw_error(NULL, "[] operator not supported for strings"); + } else { + zend_check_string_offset(dim, BP_VAR_RW); + zend_wrong_string_offset(); + } +//??? } else if (EXPECTED(Z_TYPE_P(container) <= IS_FALSE)) { +//??? ZEND_VM_C_GOTO(assign_dim_op_convert_to_array); + } else { +//??? if (UNEXPECTED(OP1_TYPE != IS_VAR || EXPECTED(!Z_ISERROR_P(container)))) { + zend_error(E_WARNING, "Cannot use a scalar value as an array"); +//??? } +//??? if (retval) { +//??? ZVAL_NULL(retval); +//??? } + } + } +} + +static void ZEND_FASTCALL zend_jit_assign_dim_bw_and_helper(zval *container, zval *dim, zval *value) +{ + if (EXPECTED(Z_TYPE_P(container) == IS_OBJECT)) { + zval *object = container; + zval *property = dim; + zval *z; + zval rv, res; + + if (Z_OBJ_HT_P(object)->read_dimension && + (z = Z_OBJ_HT_P(object)->read_dimension(object, property, BP_VAR_R, &rv)) != NULL) { + + if (Z_TYPE_P(z) == IS_OBJECT && Z_OBJ_HT_P(z)->get) { + zval rv2; + zval *value = Z_OBJ_HT_P(z)->get(z, &rv2); + + if (z == &rv) { + zval_ptr_dtor(&rv); + } + ZVAL_COPY_VALUE(z, value); + } + bitwise_and_function(&res, Z_ISREF_P(z) ? Z_REFVAL_P(z) : z, value); + Z_OBJ_HT_P(object)->write_dimension(object, property, &res); + if (z == &rv) { + zval_ptr_dtor(&rv); + } +//??? if (retval) { +//??? ZVAL_COPY(retval, &res); +//??? } + zval_ptr_dtor(&res); + } else { + zend_error(E_WARNING, "Attempt to assign property of non-object"); +//??? if (retval) { +//??? ZVAL_NULL(retval); +//??? } + } + } else { + if (UNEXPECTED(Z_TYPE_P(container) == IS_STRING)) { + if (!dim) { + zend_throw_error(NULL, "[] operator not supported for strings"); + } else { + zend_check_string_offset(dim, BP_VAR_RW); + zend_wrong_string_offset(); + } +//??? } else if (EXPECTED(Z_TYPE_P(container) <= IS_FALSE)) { +//??? ZEND_VM_C_GOTO(assign_dim_op_convert_to_array); + } else { +//??? if (UNEXPECTED(OP1_TYPE != IS_VAR || EXPECTED(!Z_ISERROR_P(container)))) { + zend_error(E_WARNING, "Cannot use a scalar value as an array"); +//??? } +//??? if (retval) { +//??? ZVAL_NULL(retval); +//??? } + } + } +} + +static void ZEND_FASTCALL zend_jit_assign_dim_bw_xor_helper(zval *container, zval *dim, zval *value) +{ + if (EXPECTED(Z_TYPE_P(container) == IS_OBJECT)) { + zval *object = container; + zval *property = dim; + zval *z; + zval rv, res; + + if (Z_OBJ_HT_P(object)->read_dimension && + (z = Z_OBJ_HT_P(object)->read_dimension(object, property, BP_VAR_R, &rv)) != NULL) { + + if (Z_TYPE_P(z) == IS_OBJECT && Z_OBJ_HT_P(z)->get) { + zval rv2; + zval *value = Z_OBJ_HT_P(z)->get(z, &rv2); + + if (z == &rv) { + zval_ptr_dtor(&rv); + } + ZVAL_COPY_VALUE(z, value); + } + bitwise_xor_function(&res, Z_ISREF_P(z) ? Z_REFVAL_P(z) : z, value); + Z_OBJ_HT_P(object)->write_dimension(object, property, &res); + if (z == &rv) { + zval_ptr_dtor(&rv); + } +//??? if (retval) { +//??? ZVAL_COPY(retval, &res); +//??? } + zval_ptr_dtor(&res); + } else { + zend_error(E_WARNING, "Attempt to assign property of non-object"); +//??? if (retval) { +//??? ZVAL_NULL(retval); +//??? } + } + } else { + if (UNEXPECTED(Z_TYPE_P(container) == IS_STRING)) { + if (!dim) { + zend_throw_error(NULL, "[] operator not supported for strings"); + } else { + zend_check_string_offset(dim, BP_VAR_RW); + zend_wrong_string_offset(); + } +//??? } else if (EXPECTED(Z_TYPE_P(container) <= IS_FALSE)) { +//??? ZEND_VM_C_GOTO(assign_dim_op_convert_to_array); + } else { +//??? if (UNEXPECTED(OP1_TYPE != IS_VAR || EXPECTED(!Z_ISERROR_P(container)))) { + zend_error(E_WARNING, "Cannot use a scalar value as an array"); +//??? } +//??? if (retval) { +//??? ZVAL_NULL(retval); +//??? } + } + } +} + +static void ZEND_FASTCALL zend_jit_assign_dim_sl_helper(zval *container, zval *dim, zval *value) +{ + if (EXPECTED(Z_TYPE_P(container) == IS_OBJECT)) { + zval *object = container; + zval *property = dim; + zval *z; + zval rv, res; + + if (Z_OBJ_HT_P(object)->read_dimension && + (z = Z_OBJ_HT_P(object)->read_dimension(object, property, BP_VAR_R, &rv)) != NULL) { + + if (Z_TYPE_P(z) == IS_OBJECT && Z_OBJ_HT_P(z)->get) { + zval rv2; + zval *value = Z_OBJ_HT_P(z)->get(z, &rv2); + + if (z == &rv) { + zval_ptr_dtor(&rv); + } + ZVAL_COPY_VALUE(z, value); + } + shift_left_function(&res, Z_ISREF_P(z) ? Z_REFVAL_P(z) : z, value); + Z_OBJ_HT_P(object)->write_dimension(object, property, &res); + if (z == &rv) { + zval_ptr_dtor(&rv); + } +//??? if (retval) { +//??? ZVAL_COPY(retval, &res); +//??? } + zval_ptr_dtor(&res); + } else { + zend_error(E_WARNING, "Attempt to assign property of non-object"); +//??? if (retval) { +//??? ZVAL_NULL(retval); +//??? } + } + } else { + if (UNEXPECTED(Z_TYPE_P(container) == IS_STRING)) { + if (!dim) { + zend_throw_error(NULL, "[] operator not supported for strings"); + } else { + zend_check_string_offset(dim, BP_VAR_RW); + zend_wrong_string_offset(); + } +//??? } else if (EXPECTED(Z_TYPE_P(container) <= IS_FALSE)) { +//??? ZEND_VM_C_GOTO(assign_dim_op_convert_to_array); + } else { +//??? if (UNEXPECTED(OP1_TYPE != IS_VAR || EXPECTED(!Z_ISERROR_P(container)))) { + zend_error(E_WARNING, "Cannot use a scalar value as an array"); +//??? } +//??? if (retval) { +//??? ZVAL_NULL(retval); +//??? } + } + } +} + +static void ZEND_FASTCALL zend_jit_assign_dim_sr_helper(zval *container, zval *dim, zval *value) +{ + if (EXPECTED(Z_TYPE_P(container) == IS_OBJECT)) { + zval *object = container; + zval *property = dim; + zval *z; + zval rv, res; + + if (Z_OBJ_HT_P(object)->read_dimension && + (z = Z_OBJ_HT_P(object)->read_dimension(object, property, BP_VAR_R, &rv)) != NULL) { + + if (Z_TYPE_P(z) == IS_OBJECT && Z_OBJ_HT_P(z)->get) { + zval rv2; + zval *value = Z_OBJ_HT_P(z)->get(z, &rv2); + + if (z == &rv) { + zval_ptr_dtor(&rv); + } + ZVAL_COPY_VALUE(z, value); + } + shift_right_function(&res, Z_ISREF_P(z) ? Z_REFVAL_P(z) : z, value); + Z_OBJ_HT_P(object)->write_dimension(object, property, &res); + if (z == &rv) { + zval_ptr_dtor(&rv); + } +//??? if (retval) { +//??? ZVAL_COPY(retval, &res); +//??? } + zval_ptr_dtor(&res); + } else { + zend_error(E_WARNING, "Attempt to assign property of non-object"); +//??? if (retval) { +//??? ZVAL_NULL(retval); +//??? } + } + } else { + if (UNEXPECTED(Z_TYPE_P(container) == IS_STRING)) { + if (!dim) { + zend_throw_error(NULL, "[] operator not supported for strings"); + } else { + zend_check_string_offset(dim, BP_VAR_RW); + zend_wrong_string_offset(); + } +//??? } else if (EXPECTED(Z_TYPE_P(container) <= IS_FALSE)) { +//??? ZEND_VM_C_GOTO(assign_dim_op_convert_to_array); + } else { +//??? if (UNEXPECTED(OP1_TYPE != IS_VAR || EXPECTED(!Z_ISERROR_P(container)))) { + zend_error(E_WARNING, "Cannot use a scalar value as an array"); +//??? } +//??? if (retval) { +//??? ZVAL_NULL(retval); +//??? } + } + } +} + +static void ZEND_FASTCALL zend_jit_assign_dim_mod_helper(zval *container, zval *dim, zval *value) +{ + if (EXPECTED(Z_TYPE_P(container) == IS_OBJECT)) { + zval *object = container; + zval *property = dim; + zval *z; + zval rv, res; + + if (Z_OBJ_HT_P(object)->read_dimension && + (z = Z_OBJ_HT_P(object)->read_dimension(object, property, BP_VAR_R, &rv)) != NULL) { + + if (Z_TYPE_P(z) == IS_OBJECT && Z_OBJ_HT_P(z)->get) { + zval rv2; + zval *value = Z_OBJ_HT_P(z)->get(z, &rv2); + + if (z == &rv) { + zval_ptr_dtor(&rv); + } + ZVAL_COPY_VALUE(z, value); + } + mod_function(&res, Z_ISREF_P(z) ? Z_REFVAL_P(z) : z, value); + Z_OBJ_HT_P(object)->write_dimension(object, property, &res); + if (z == &rv) { + zval_ptr_dtor(&rv); + } +//??? if (retval) { +//??? ZVAL_COPY(retval, &res); +//??? } + zval_ptr_dtor(&res); + } else { + zend_error(E_WARNING, "Attempt to assign property of non-object"); +//??? if (retval) { +//??? ZVAL_NULL(retval); +//??? } + } + } else { + if (UNEXPECTED(Z_TYPE_P(container) == IS_STRING)) { + if (!dim) { + zend_throw_error(NULL, "[] operator not supported for strings"); + } else { + zend_check_string_offset(dim, BP_VAR_RW); + zend_wrong_string_offset(); + } +//??? } else if (EXPECTED(Z_TYPE_P(container) <= IS_FALSE)) { +//??? ZEND_VM_C_GOTO(assign_dim_op_convert_to_array); + } else { +//??? if (UNEXPECTED(OP1_TYPE != IS_VAR || EXPECTED(!Z_ISERROR_P(container)))) { + zend_error(E_WARNING, "Cannot use a scalar value as an array"); +//??? } +//??? if (retval) { +//??? ZVAL_NULL(retval); +//??? } + } + } +} + static void ZEND_FASTCALL zend_jit_assign_dim_concat_helper(zval *container, zval *dim, zval *value) { if (EXPECTED(Z_TYPE_P(container) == IS_OBJECT)) { diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 299b40f1e1f40..add1cf570256e 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -595,10 +595,20 @@ static void* dasm_labels[zend_lb_MAX]; || case ZEND_ASSIGN_MUL: | LONG_OP imul, reg, addr || break; -|| case ZEND_DIV: -|| case ZEND_ASSIGN_DIV: -| idiv aword [Ra(Z_REG(addr))+Z_OFFSET(addr)] // (reg == r0) +|| case ZEND_BW_OR: +|| case ZEND_ASSIGN_BW_OR: +| LONG_OP or, reg, addr +|| break; +|| case ZEND_BW_AND: +|| case ZEND_ASSIGN_BW_AND: +| LONG_OP and, reg, addr || break; +|| case ZEND_BW_XOR: +|| case ZEND_ASSIGN_BW_XOR: +| LONG_OP xor, reg, addr +|| break; +|| default: +|| ZEND_ASSERT(0); || } |.endmacro @@ -616,10 +626,20 @@ static void* dasm_labels[zend_lb_MAX]; || case ZEND_ASSIGN_MUL: | imul dst_reg, src_reg || break; -|| case ZEND_DIV: -|| case ZEND_ASSIGN_DIV: -| idiv src_reg // (reg1 == r0) -|| break; +|| case ZEND_BW_OR: +|| case ZEND_ASSIGN_BW_OR: +| or dst_reg, src_reg +|| break; +|| case ZEND_BW_AND: +|| case ZEND_ASSIGN_BW_AND: +| and dst_reg, src_reg +|| break; +|| case ZEND_BW_XOR: +|| case ZEND_ASSIGN_BW_XOR: +| xor dst_reg, src_reg +|| break; +|| default: +|| ZEND_ASSERT(0); || } |.endmacro @@ -2664,7 +2684,7 @@ static int zend_jit_math_double_long(dasm_State **Dst, | SSE_SET_ZVAL_DVAL res_addr, result_reg |.else | FPU_GET_ZVAL_LVAL op2_addr - if (opline->opcode == ZEND_ADD || opline->opcode == ZEND_MUL || opline->opcode == ZEND_ASSIGN_ADD || opline->opcode == ZEND_ASSIGN_MUL) { + if (zend_is_commutative(opline->opcode)) { | FPU_MATH opline->opcode, op1_addr } else { | FPU_GET_ZVAL_DVAL op1_addr @@ -2764,72 +2784,6 @@ static int zend_jit_math_double_double(dasm_State **Dst, return 1; } -static int zend_jit_shift(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) -{ - uint32_t op1_info; - zend_jit_addr op1_addr, op2_addr, res_addr; - zend_bool has_slow; - - if (!ssa->ops || !ssa->var_info) { - goto fallback; - } - - op1_info = OP1_INFO(); - - op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, NULL, -1); - op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2, opline, NULL, -1); - res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1); - - if ((op1_info & MAY_BE_UNDEF) || - Z_MODE(op2_addr) != IS_CONST_ZVAL || - Z_TYPE_P(Z_ZV(op2_addr)) != IS_LONG || - Z_LVAL_P(Z_ZV(op2_addr)) >= SIZEOF_ZEND_LONG * 8) { - goto fallback; - } - - has_slow = (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_LONG)) != 0; - - if (op1_info & (MAY_BE_ANY-MAY_BE_LONG)) { - | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >9 - } - | GET_ZVAL_LVAL ZREG_R0, op1_addr - if (opline->opcode == ZEND_SR) { - | shr, r0, Z_LVAL_P(Z_ZV(op2_addr)) - } else { - | shl, r0, Z_LVAL_P(Z_ZV(op2_addr)) - } - | SET_ZVAL_LVAL res_addr, r0 - | SET_ZVAL_TYPE_INFO res_addr, IS_LONG - - if (has_slow) { - |.cold_code - |9: - | LOAD_ZVAL_ADDR FCARG1a, res_addr - | LOAD_ZVAL_ADDR FCARG2a, op1_addr - |.if X64 - | LOAD_ZVAL_ADDR CARG3, op2_addr - |.else - | PUSH_ZVAL_ADDR op2_addr, r0 - |.endif - if (opline->opcode == ZEND_SR) { - | EXT_CALL shift_right_function, r0 - } else { - | EXT_CALL shift_left_function, r0 - } - | jmp >1 - |.code - |1: - } - - | FREE_OP opline->op1_type, opline->op1, op1_info, 1, op_array, opline - - return 1; - -fallback: - /* fallback to subroutine threading */ - return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); -} - static int zend_jit_math_helper(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, @@ -3106,6 +3060,238 @@ fallback: return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); } +static int zend_jit_long_math_helper(dasm_State **Dst, + const zend_op *opline, + zend_op_array *op_array, + zend_ssa *ssa, + zend_uchar op1_type, + znode_op op1, + zend_jit_addr op1_addr, + uint32_t op1_info, + zend_uchar op2_type, + znode_op op2, + zend_jit_addr op2_addr, + uint32_t op2_info, + uint32_t res_var, + zend_jit_addr res_addr, + uint32_t res_info, + uint32_t res_use_info, + zend_bool separate_op1) +/* Labels: 6 */ +{ + zend_bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); + zend_reg result_reg; + zend_uchar opcode = opline->opcode; + zval tmp; + + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6 + } + if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) { + | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6 + } + + if (opcode == ZEND_MOD && Z_MODE(op2_addr) == IS_CONST_ZVAL) { + zend_long l = Z_LVAL_P(Z_ZV(op2_addr)); + + if (zend_long_is_power_of_two(l)) { + /* Optimisation for mod of power of 2 */ + opcode = ZEND_BW_AND; + ZVAL_LONG(&tmp, l - 1); + op2_addr = ZEND_ADDR_CONST_ZVAL(&tmp); + } + } + + if (opcode == ZEND_MOD) { + result_reg = ZREG_RAX; + } else if (Z_MODE(res_addr) == IS_REG) { + result_reg = Z_REG(res_addr); + } else if (Z_MODE(op1_addr) == IS_REG && zend_ssa_is_last_use(op_array, ssa, ssa->ops[opline-op_array->opcodes].op1_use, opline-op_array->opcodes)) { + result_reg = Z_REG(op1_addr); + } else { + result_reg = ZREG_R0; + } + + | GET_ZVAL_LVAL result_reg, op1_addr + if (opcode == ZEND_SL || opcode == ZEND_ASSIGN_SL) { + //TODO: add checks ??? + if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { + | shl Ra(result_reg), Z_LVAL_P(Z_ZV(op2_addr)) + } else { + if (Z_MODE(op2_addr) != IS_REG || Z_REG(op2_addr) != ZREG_RCX) { + | GET_ZVAL_LVAL ZREG_RCX, op2_addr + } + | shl Ra(result_reg), cl + } + } else if (opcode == ZEND_SR || opcode == ZEND_ASSIGN_SR) { + //TODO: add checks ??? + if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { + | shr Ra(result_reg), Z_LVAL_P(Z_ZV(op2_addr)) + } else { + if (Z_MODE(op2_addr) != IS_REG || Z_REG(op2_addr) != ZREG_RCX) { + | GET_ZVAL_LVAL ZREG_RCX, op2_addr + } + | shr Ra(result_reg), cl + } + } else if (opcode == ZEND_MOD || opcode == ZEND_ASSIGN_MOD) { + //TODO: add checks ??? + zend_jit_addr val_addr = op2_addr; + + if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { + | GET_ZVAL_LVAL ZREG_RCX, op2_addr + val_addr = ZEND_ADDR_REG(ZREG_RCX); + } + | xor Ra(ZREG_RDX), Ra(ZREG_RDX) + if (Z_MODE(val_addr) == IS_MEM_ZVAL) { + | idiv aword [Ra(Z_REG(val_addr))+Z_OFFSET(val_addr)] + } else if (Z_MODE(val_addr) == IS_REG) { + | idiv Ra(Z_REG(val_addr)) + } + result_reg = ZREG_RDX; + } else if (same_ops) { + | LONG_MATH_REG opcode, Ra(result_reg), Ra(result_reg) + } else { + | LONG_MATH opcode, Ra(result_reg), op2_addr + } + + if (Z_MODE(res_addr) == IS_MEM_ZVAL) { + | SET_ZVAL_LVAL res_addr, Ra(result_reg) + if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { + if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_LONG) { + | SET_ZVAL_TYPE_INFO res_addr, IS_LONG + } + } + } + + if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) || + (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) { + if ((op1_info & MAY_BE_LONG) && + (op2_info & MAY_BE_LONG)) { + |5: + |.cold_code + } + |6: + | SAVE_VALID_OPLINE opline + if (separate_op1) { + | SEPARATE_ZVAL_NOREF op1_addr, op1_info, 0, op_array->filename, opline->lineno + } + if (Z_MODE(res_addr) == IS_REG) { + zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); + | LOAD_ZVAL_ADDR FCARG1a, real_addr + } else if (Z_REG(res_addr) != ZREG_FCARG1a || Z_OFFSET(res_addr) != 0) { + | LOAD_ZVAL_ADDR FCARG1a, res_addr + } + if (Z_MODE(op1_addr) == IS_REG) { + zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var); + if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { + return 0; + } + op1_addr = real_addr; + } + | LOAD_ZVAL_ADDR FCARG2a, op1_addr + if (Z_MODE(op2_addr) == IS_REG) { + zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op2.var); + if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) { + return 0; + } + op2_addr = real_addr; + } + |.if X64 + | LOAD_ZVAL_ADDR CARG3, op2_addr + |.else + | PUSH_ZVAL_ADDR op2_addr, r0 + |.endif + if (opline->opcode == ZEND_BW_OR || opline->opcode == ZEND_ASSIGN_BW_OR) { + | EXT_CALL bitwise_or_function, r0 + } else if (opline->opcode == ZEND_BW_AND || opline->opcode == ZEND_ASSIGN_BW_AND) { + | EXT_CALL bitwise_and_function, r0 + } else if (opline->opcode == ZEND_BW_XOR || opline->opcode == ZEND_ASSIGN_BW_XOR) { + | EXT_CALL bitwise_xor_function, r0 + } else if (opline->opcode == ZEND_SL || opline->opcode == ZEND_ASSIGN_SL) { + | EXT_CALL shift_left_function, r0 + } else if (opline->opcode == ZEND_SR || opline->opcode == ZEND_ASSIGN_SR) { + | EXT_CALL shift_right_function, r0 + } else if (opline->opcode == ZEND_MOD || opline->opcode == ZEND_ASSIGN_MOD) { + | EXT_CALL mod_function, r0 + } else { + ZEND_ASSERT(0); + } + | FREE_OP op1_type, op1, op1_info, 0, op_array, opline + | FREE_OP op2_type, op2, op2_info, 0, op_array, opline + if (zend_may_throw(opline, op_array, ssa)) { + zend_jit_check_exception(Dst); + } + if (Z_MODE(res_addr) == IS_REG) { + zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); + if (!zend_jit_load_reg(Dst, real_addr, res_addr, res_info)) { + return 0; + } + } + if ((op1_info & MAY_BE_LONG) && + (op2_info & MAY_BE_LONG)) { + | jmp <5 + |.code + } + } + + return 1; +} + +static int zend_jit_long_math(dasm_State **Dst, const zend_op *opline, int *opnum, zend_op_array *op_array, zend_ssa *ssa, zend_lifetime_interval **ra) +{ + uint32_t op1_info, op2_info, res_use_info; + zend_jit_addr op1_addr, op2_addr, res_addr; + + if (!ssa->ops || !ssa->var_info) { + goto fallback; + } + + op1_info = OP1_INFO(); + op2_info = OP2_INFO(); + res_use_info = RES_USE_INFO(); + + if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) { + goto fallback; + } + + if (!(op1_info & MAY_BE_LONG) || + !(op2_info & MAY_BE_LONG)) { + goto fallback; + } + + op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].op1_use : -1); + op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].op2_use : -1); + + if (opline->result_type == IS_TMP_VAR && + (opline+1)->opcode == ZEND_SEND_VAL && + (opline+1)->op1_type == IS_TMP_VAR && + (opline+1)->op1.var == opline->result.var) { + /* Eliminate the following SEND_VAL */ + (*opnum)++; + if (!reuse_ip) { + zend_jit_start_reuse_ip(); + | // call = EX(call); + | mov RX, EX->call + } + res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var); + res_use_info = -1; + } else { + res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, ra, ssa->ops[opline - op_array->opcodes].result_def); + } + + if (!zend_jit_long_math_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, opline->result.var, res_addr, RES_INFO(), res_use_info, 0)) { + return 0; + } + if (ra && !zend_jit_store_ssa_var_if_necessary(Dst, ssa, ra, res_addr, ssa->ops[opline - op_array->opcodes].result_def, ssa->ops[opline - op_array->opcodes].result_use)) { + return 0; + } + return 1; + +fallback: + /* fallback to subroutine threading */ + return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); +} + static int zend_jit_concat_helper(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, @@ -4168,6 +4354,16 @@ static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, zend_ return 0; } break; + case ZEND_ASSIGN_BW_OR: + case ZEND_ASSIGN_BW_AND: + case ZEND_ASSIGN_BW_XOR: + case ZEND_ASSIGN_SL: + case ZEND_ASSIGN_SR: + case ZEND_ASSIGN_MOD: + if (!zend_jit_long_math_helper(Dst, opline, op_array, ssa, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, op3_addr, OP1_DATA_INFO(), 0, var_addr, OP1_DEF_INFO(), var_info, 1)) { + return 0; + } + break; case ZEND_ASSIGN_CONCAT: if (!zend_jit_concat_helper(Dst, opline, op_array, ssa, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, op3_addr, OP1_DATA_INFO(), var_addr, OP1_DEF_INFO())) { return 0; @@ -4270,6 +4466,24 @@ static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, zend_ case ZEND_ASSIGN_DIV: | EXT_CALL zend_jit_assign_dim_div_helper, r0 break; + case ZEND_ASSIGN_BW_OR: + | EXT_CALL zend_jit_assign_dim_bw_or_helper, r0 + break; + case ZEND_ASSIGN_BW_AND: + | EXT_CALL zend_jit_assign_dim_bw_and_helper, r0 + break; + case ZEND_ASSIGN_BW_XOR: + | EXT_CALL zend_jit_assign_dim_bw_xor_helper, r0 + break; + case ZEND_ASSIGN_SL: + | EXT_CALL zend_jit_assign_dim_sl_helper, r0 + break; + case ZEND_ASSIGN_SR: + | EXT_CALL zend_jit_assign_dim_sr_helper, r0 + break; + case ZEND_ASSIGN_MOD: + | EXT_CALL zend_jit_assign_dim_mod_helper, r0 + break; case ZEND_ASSIGN_CONCAT: | EXT_CALL zend_jit_assign_dim_concat_helper, r0 break; @@ -4333,6 +4547,17 @@ static int zend_jit_assign_op(dasm_State **Dst, const zend_op *opline, zend_op_a goto fallback; } break; + case ZEND_ASSIGN_BW_OR: + case ZEND_ASSIGN_BW_AND: + case ZEND_ASSIGN_BW_XOR: + case ZEND_ASSIGN_SL: + case ZEND_ASSIGN_SR: + case ZEND_ASSIGN_MOD: + if (!(op1_info & MAY_BE_LONG) || + !(op2_info & MAY_BE_LONG)) { + goto fallback; + } + break; case ZEND_ASSIGN_CONCAT: if (!(op1_info & MAY_BE_STRING) || !(op2_info & MAY_BE_STRING)) { @@ -4355,6 +4580,13 @@ static int zend_jit_assign_op(dasm_State **Dst, const zend_op *opline, zend_op_a case ZEND_ASSIGN_MUL: case ZEND_ASSIGN_DIV: return zend_jit_math_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, opline->op1.var, op1_addr, OP1_DEF_INFO(), op1_info, 1); + case ZEND_ASSIGN_BW_OR: + case ZEND_ASSIGN_BW_AND: + case ZEND_ASSIGN_BW_XOR: + case ZEND_ASSIGN_SL: + case ZEND_ASSIGN_SR: + case ZEND_ASSIGN_MOD: + return zend_jit_long_math_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, opline->op1.var, op1_addr, OP1_DEF_INFO(), op1_info, 1); case ZEND_ASSIGN_CONCAT: return zend_jit_concat_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, op1_addr, OP1_DEF_INFO()); default: @@ -8650,6 +8882,9 @@ static zend_bool zend_jit_may_reuse_reg(zend_op_array *op_array, zend_ssa *ssa, case ZEND_ADD: case ZEND_SUB: case ZEND_MUL: + case ZEND_BW_OR: + case ZEND_BW_AND: + case ZEND_BW_XOR: if (def_var == ssa->ops[position].result_def && use_var == ssa->ops[position].op1_use) { return 1; @@ -8670,9 +8905,6 @@ static zend_bool zend_jit_opline_supports_reg(zend_op_array *op_array, zend_ssa case ZEND_SEND_VAR: case ZEND_SEND_VAL: case ZEND_SEND_VAL_EX: - case ZEND_ADD: - case ZEND_SUB: - case ZEND_MUL: case ZEND_IS_SMALLER: case ZEND_IS_SMALLER_OR_EQUAL: case ZEND_IS_EQUAL: @@ -8689,6 +8921,25 @@ static zend_bool zend_jit_opline_supports_reg(zend_op_array *op_array, zend_ssa opline->op1_type == IS_CV && !(op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_RESOURCE|MAY_BE_REF)) && !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))); + case ZEND_ADD: + case ZEND_SUB: + case ZEND_MUL: + op1_info = OP1_INFO(); + op2_info = OP1_INFO(); + return + !(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) && + !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))); + case ZEND_BW_OR: + case ZEND_BW_AND: + case ZEND_BW_XOR: + case ZEND_SL: + case ZEND_SR: + case ZEND_MOD: + op1_info = OP1_INFO(); + op2_info = OP1_INFO(); + return + !(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG)) && + !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG)); case ZEND_PRE_INC: case ZEND_PRE_DEC: case ZEND_POST_INC: @@ -8924,6 +9175,55 @@ static zend_regset zend_jit_get_scratch_regset(zend_op_array *op_array, zend_ssa } } break; + case ZEND_BW_OR: + case ZEND_BW_AND: + case ZEND_BW_XOR: + op1_info = OP1_INFO(); + op2_info = OP1_INFO(); + if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG)) && + !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG))) { + regset = ZEND_REGSET_EMPTY; + if (ssa->ops[line].result_def != current_var && + (ssa->ops[line].op1_use != current_var || !zend_ssa_is_last_use(op_array, ssa, current_var, opline-op_array->opcodes))) { + ZEND_REGSET_INCL(regset, ZREG_R0); + } + } + break; + case ZEND_SL: + case ZEND_SR: + op1_info = OP1_INFO(); + op2_info = OP1_INFO(); + if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG)) && + !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG))) { + regset = ZEND_REGSET_EMPTY; + if (ssa->ops[line].result_def != current_var && + (ssa->ops[line].op1_use != current_var || !zend_ssa_is_last_use(op_array, ssa, current_var, opline-op_array->opcodes))) { + ZEND_REGSET_INCL(regset, ZREG_R0); + } + if (opline->op2_type != IS_CONST && ssa->ops[line].op2_use != current_var) { + ZEND_REGSET_INCL(regset, ZREG_R1); + } + } + break; + case ZEND_MOD: + op1_info = OP1_INFO(); + op2_info = OP1_INFO(); + if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG)) && + !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG))) { + regset = ZEND_REGSET_EMPTY; + if (opline->op2_type == IS_CONST && + Z_TYPE_P(RT_CONSTANT(op_array, opline->op2)) == IS_LONG && + zend_long_is_power_of_two(Z_LVAL_P(RT_CONSTANT(op_array, opline->op2)))) { + if (ssa->ops[line].result_def != current_var && + (ssa->ops[line].op1_use != current_var || !zend_ssa_is_last_use(op_array, ssa, current_var, opline-op_array->opcodes))) { + ZEND_REGSET_INCL(regset, ZREG_R0); + } + } else { + ZEND_REGSET_INCL(regset, ZREG_R0); + ZEND_REGSET_INCL(regset, ZREG_R1); + } + } + break; case ZEND_IS_SMALLER: case ZEND_IS_SMALLER_OR_EQUAL: case ZEND_IS_EQUAL: From dec08fffe66e6e4c065265c86f19802bdf737482 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 18 Apr 2017 11:44:43 +0300 Subject: [PATCH 440/569] Cleanup code generated for INC/DEC IS_DOUBLE --- ext/opcache/jit/zend_jit_x86.dasc | 132 ++++++++++++++++-------------- 1 file changed, 69 insertions(+), 63 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index add1cf570256e..fbc36610b9008 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -2392,47 +2392,47 @@ static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, zend_op_arr if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { |.cold_code |2: - | SAVE_VALID_OPLINE opline - if (op1_info & MAY_BE_UNDEF) { - | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >2 - | // zend_error(E_NOTICE, "Undefined variable: %s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); - | mov FCARG1d, opline->op1.var - | EXT_CALL zend_jit_undefined_op_helper, r0 - | SET_ZVAL_TYPE_INFO op1_addr, IS_NULL - op1_info |= MAY_BE_NULL; - } - |2: - | LOAD_ZVAL_ADDR FCARG1a, op1_addr - | // ZVAL_DEREF(var_ptr); - | ZVAL_DEREF FCARG1a, op1_info - if ((opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC)) { - if (opline->result_type != IS_UNUSED) { - zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); - - | ZVAL_COPY_VALUE res_addr, res_use_info, val_addr, op1_info, ZREG_R0, ZREG_R2 - | //ZVAL_COPY_CTOR op1_info, ah, r2, op_array->filename, opline->lineno - if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { - if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY)) { - | IF_NOT_FLAGS ah, IS_TYPE_REFCOUNTED + IS_TYPE_COPYABLE, >2 - | IF_FLAGS ah, IS_TYPE_COPYABLE, >1 - | GC_ADDREF r2 - | jmp >2 - |1: - | mov aword [r4], FCARG1a // save - | ZVAL_COPY_CTOR_FUNC op_array->filename, opline->lineno - | mov FCARG1a, aword [r4] // restore - |2: - } else { - | TRY_ADDREF op1_info, ah, r2 + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) { + | SAVE_VALID_OPLINE opline + if (op1_info & MAY_BE_UNDEF) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >2 + | // zend_error(E_NOTICE, "Undefined variable: %s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); + | mov FCARG1d, opline->op1.var + | EXT_CALL zend_jit_undefined_op_helper, r0 + | SET_ZVAL_TYPE_INFO op1_addr, IS_NULL + op1_info |= MAY_BE_NULL; + } + |2: + | LOAD_ZVAL_ADDR FCARG1a, op1_addr + | // ZVAL_DEREF(var_ptr); + | ZVAL_DEREF FCARG1a, op1_info + if ((opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC)) { + if (opline->result_type != IS_UNUSED) { + zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); + + | ZVAL_COPY_VALUE res_addr, res_use_info, val_addr, op1_info, ZREG_R0, ZREG_R2 + | //ZVAL_COPY_CTOR op1_info, ah, r2, op_array->filename, opline->lineno + if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { + if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY)) { + | IF_NOT_FLAGS ah, IS_TYPE_REFCOUNTED + IS_TYPE_COPYABLE, >2 + | IF_FLAGS ah, IS_TYPE_COPYABLE, >1 + | GC_ADDREF r2 + | jmp >2 + |1: + | mov aword [r4], FCARG1a // save + | ZVAL_COPY_CTOR_FUNC op_array->filename, opline->lineno + | mov FCARG1a, aword [r4] // restore + |2: + } else { + | TRY_ADDREF op1_info, ah, r2 + } } } - } - } else { - zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); + } else { + zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); - | SEPARATE_ZVAL_NOREF var_addr, op1_info, 0, op_array->filename, opline->lineno - } - if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) { + | SEPARATE_ZVAL_NOREF var_addr, op1_info, 0, op_array->filename, opline->lineno + } if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { | EXT_CALL increment_function, r0 } else { @@ -2440,38 +2440,44 @@ static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, zend_op_arr } } else { zend_reg tmp_reg; - if (Z_MODE(op1_def_addr) == IS_REG) { - tmp_reg = Z_REG(op1_def_addr); - } else if (Z_MODE(op1_addr) == IS_REG && zend_ssa_is_last_use(op_array, ssa, ssa->ops[opline-op_array->opcodes].op1_use, opline-op_array->opcodes)) { - tmp_reg = Z_REG(op1_addr); - } else { - tmp_reg = ZREG_XMM0; + + if ((opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC)) { + if (opline->result_type != IS_UNUSED) { + | ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, MAY_BE_DOUBLE, ZREG_R0, ZREG_R2 + } } |.if SSE - | SSE_GET_ZVAL_DVAL tmp_reg, op1_addr - if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { - if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { - | vaddsd xmm(tmp_reg-ZREG_XMM0), xmm(tmp_reg-ZREG_XMM0), qword [->one] + if (Z_MODE(op1_def_addr) == IS_REG) { + tmp_reg = Z_REG(op1_def_addr); + } else if (Z_MODE(op1_addr) == IS_REG && zend_ssa_is_last_use(op_array, ssa, ssa->ops[opline-op_array->opcodes].op1_use, opline-op_array->opcodes)) { + tmp_reg = Z_REG(op1_addr); } else { - | addsd xmm(tmp_reg-ZREG_XMM0), qword [->one] + tmp_reg = ZREG_XMM0; } - } else { - if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { - | vsubsd xmm(tmp_reg-ZREG_XMM0), xmm(tmp_reg-ZREG_XMM0), qword [->one] + | SSE_GET_ZVAL_DVAL tmp_reg, op1_addr + if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { + if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { + | vaddsd xmm(tmp_reg-ZREG_XMM0), xmm(tmp_reg-ZREG_XMM0), qword [->one] + } else { + | addsd xmm(tmp_reg-ZREG_XMM0), qword [->one] + } } else { - | subsd xmm(tmp_reg-ZREG_XMM0), qword [->one] + if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { + | vsubsd xmm(tmp_reg-ZREG_XMM0), xmm(tmp_reg-ZREG_XMM0), qword [->one] + } else { + | subsd xmm(tmp_reg-ZREG_XMM0), qword [->one] + } } - } - | SSE_SET_ZVAL_DVAL op1_def_addr, tmp_reg + | SSE_SET_ZVAL_DVAL op1_def_addr, tmp_reg |.else - | FPU_GET_ZVAL_DVAL op1_addr - | fld1 - if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { - | fadd - } else { - | fsub - } - | FPU_SET_ZVAL_DVAL op1_def_addr + | FPU_GET_ZVAL_DVAL op1_addr + | fld1 + if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { + | fadd + } else { + | fsub + } + | FPU_SET_ZVAL_DVAL op1_def_addr |.endif } if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && From 027bbdf05971f44603f7f7e5f7f2479f0c4f6b77 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 18 Apr 2017 13:36:07 +0300 Subject: [PATCH 441/569] Fixed use-after-free --- ext/opcache/jit/zend_jit_x86.dasc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index fbc36610b9008..6847500fcded1 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -3925,6 +3925,10 @@ static int zend_jit_assign_to_variable(dasm_State **Dst, return 0; } | mov FCARG1a, aword [r4] // restore + if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { + | cmp dword [FCARG1a], 0 + | jnz >3 + } | ZVAL_DTOR_FUNC var_info, op_array->filename, opline | jmp >3 |4: From f0fad467e91ab407b7f43cd3eaddd67164339263 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 18 Apr 2017 17:01:05 +0300 Subject: [PATCH 442/569] Fixed JIT for 64-bit systems, where PHP code is loaded in high addresses (above 4GB) --- ext/opcache/jit/zend_jit_x86.dasc | 258 ++++++++++++++---------------- 1 file changed, 121 insertions(+), 137 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 6847500fcded1..7acbb44224a58 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -107,19 +107,88 @@ static void* dasm_labels[zend_lb_MAX]; | .endif |.endmacro +|.macro ADDR_OP1, addr_ins, addr, tmp_reg +| .if X64 +|| if (IS_32BIT(addr)) { +| addr_ins ((ptrdiff_t)addr) +|| } else { +| mov64 tmp_reg, ((ptrdiff_t)addr) +| addr_ins tmp_reg +|| } +| .else +| addr_ins ((ptrdiff_t)addr) +| .endif +|.endmacro + +|.macro ADDR_OP2_2, addr_ins, op1, addr, tmp_reg +| .if X64 +|| if (IS_32BIT(addr)) { +| addr_ins op1, ((ptrdiff_t)addr) +|| } else { +| mov64 tmp_reg, ((ptrdiff_t)addr) +| addr_ins op1, tmp_reg +|| } +| .else +| addr_ins op1, ((ptrdiff_t)addr) +| .endif +|.endmacro + |.macro PUSH_ADDR, addr, tmp_reg +| ADDR_OP1 push, addr, tmp_reg +|.endmacro + +|.macro MEM_OP1, mem_ins, prefix, addr, tmp_reg | .if X64 || if (IS_32BIT(addr)) { -| push ((ptrdiff_t)addr) // 0x48 0xc7 0xc0 +| mem_ins prefix [addr] || } else { -| mov64 tmp_reg, ((ptrdiff_t)addr) // 0x48 0xb8 -| push tmp_reg +| mov64 tmp_reg, ((ptrdiff_t)addr) +| mem_ins prefix [tmp_reg] || } | .else -| push ((ptrdiff_t)addr) +| mem_ins prefix [addr] | .endif |.endmacro +|.macro MEM_OP2_1, mem_ins, prefix, addr, op2, tmp_reg +| .if X64 +|| if (IS_32BIT(addr)) { +| mem_ins prefix [addr], op2 +|| } else { +| mov64 tmp_reg, ((ptrdiff_t)addr) +| mem_ins prefix [tmp_reg], op2 +|| } +| .else +| mem_ins prefix [addr], op2 +| .endif +|.endmacro + +|.macro MEM_OP2_2, mem_ins, op1, prefix, addr, tmp_reg +| .if X64 +|| if (IS_32BIT(addr)) { +| mem_ins op1, prefix [addr] +|| } else { +| mov64 tmp_reg, ((ptrdiff_t)addr) +| mem_ins op1, prefix [tmp_reg] +|| } +| .else +| mem_ins op1, prefix [addr] +| .endif +|.endmacro + +|.macro MEM_OP3_3, mem_ins, op1, op2, prefix, addr, tmp_reg +| .if X64 +|| if (IS_32BIT(addr)) { +| mem_ins op1, op2, prefix [addr] +|| } else { +| mov64 tmp_reg, ((ptrdiff_t)addr) +| mem_ins op1, op2, prefix [tmp_reg] +|| } +| .else +| mem_ins op1, op2, prefix [addr] +| .endif +|.endmacro + |.macro LOAD_BASE_ADDR, reg, base, offset || if (offset) { | lea reg, qword [Ra(base)+offset] @@ -231,14 +300,7 @@ static void* dasm_labels[zend_lb_MAX]; |.macro FPU_OP, fp_ins, addr || if (Z_MODE(addr) == IS_CONST_ZVAL) { -| .if X64 -|| if (IS_32BIT(Z_ZV(addr))) { -| fp_ins qword [Z_ZV(addr)] -|| } else { -| LOAD_ADDR r0, Z_ZV(addr) -| fp_ins qword [r0] -|| } -| .endif +| MEM_OP1 fp_ins, qword, Z_ZV(addr), r0 || } else if (Z_MODE(addr) == IS_MEM_ZVAL) { | fp_ins qword [Ra(Z_REG(addr))+Z_OFFSET(addr)] || } else { @@ -286,16 +348,7 @@ static void* dasm_labels[zend_lb_MAX]; |.macro SSE_OP, sse_ins, reg, addr || if (Z_MODE(addr) == IS_CONST_ZVAL) { -| .if X64 -|| if (IS_32BIT(Z_ZV(addr))) { -| sse_ins xmm(reg-ZREG_XMM0), qword [Z_ZV(addr)] -|| } else { -| LOAD_ADDR r0, Z_ZV(addr) -| sse_ins, xmm(reg-ZREG_XMM0), qword [r0] -|| } -| .else -| sse_ins xmm(reg-ZREG_XMM0), qword [Z_ZV(addr)] -| .endif +| MEM_OP2_2 sse_ins, xmm(reg-ZREG_XMM0), qword, Z_ZV(addr), r0 || } else if (Z_MODE(addr) == IS_MEM_ZVAL) { | sse_ins xmm(reg-ZREG_XMM0), qword [Ra(Z_REG(addr))+Z_OFFSET(addr)] || } else if (Z_MODE(addr) == IS_REG) { @@ -328,28 +381,11 @@ static void* dasm_labels[zend_lb_MAX]; |.macro SSE_GET_ZVAL_LVAL, reg, addr || if (Z_MODE(addr) == IS_CONST_ZVAL) { -| .if X64 -|| if (IS_32BIT(Z_ZV(addr))) { -|| if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { -| vcvtsi2sd xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), aword [Z_ZV(addr)] -|| } else { -| cvtsi2sd xmm(reg-ZREG_XMM0), aword [Z_ZV(addr)] -|| } -|| } else { -| LOAD_ADDR r0, Z_ZV(addr) -|| if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { -| vcvtsi2sd xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), aword [r0] -|| } else { -| cvtsi2sd xmm(reg-ZREG_XMM0), aword [r0] -|| } -|| } -| .else -|| if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { -| vcvtsi2sd xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), aword [Z_ZV(addr)] -|| } else { -| cvtsi2sd xmm(reg-ZREG_XMM0), aword [Z_ZV(addr)] -|| } -| .endif +|| if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { +| MEM_OP3_3 vcvtsi2sd, xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), aword, Z_ZV(addr), r0 +|| } else { +| MEM_OP2_2 cvtsi2sd, xmm(reg-ZREG_XMM0), aword, Z_ZV(addr), r0 +|| } || } else if (Z_MODE(addr) == IS_MEM_ZVAL) { || if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { | vcvtsi2sd xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), aword [Ra(Z_REG(addr))+Z_OFFSET(addr)] @@ -445,16 +481,7 @@ static void* dasm_labels[zend_lb_MAX]; |.macro AVX_OP, avx_ins, reg, op1_reg, addr || if (Z_MODE(addr) == IS_CONST_ZVAL) { -| .if X64 -|| if (IS_32BIT(Z_ZV(addr))) { -| avx_ins xmm(reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), qword [Z_ZV(addr)] -|| } else { -| LOAD_ADDR r0, Z_ZV(addr) -| avx_ins xmm(reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), qword [r0] -|| } -| .else -| avx_ins xmm(reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), qword [Z_ZV(addr)] -| .endif +| MEM_OP3_3 avx_ins, xmm(reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), qword, Z_ZV(addr), r0 || } else if (Z_MODE(addr) == IS_MEM_ZVAL) { | avx_ins xmm(reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), qword [Ra(Z_REG(addr))+Z_OFFSET(addr)] || } else if (Z_MODE(addr) == IS_REG) { @@ -654,16 +681,7 @@ static void* dasm_labels[zend_lb_MAX]; |.macro FPU_LONG_OP, fp_ins, addr || if (Z_MODE(addr) == IS_CONST_ZVAL) { -| .if X64 -|| if (IS_32BIT(Z_ZV(addr))) { -| fp_ins dword [Z_ZV(addr)] -|| } else { -| LOAD_ADDR r0, Z_ZV(addr) -| fp_ins qword [r0] -|| } -| .else -| fp_ins dword [Z_ZV(addr)] -| .endif +| MEM_OP1 fp_ins, aword, Z_ZV(addr), r0 || } else if (Z_MODE(addr) == IS_MEM_ZVAL) { | fp_ins aword [Ra(Z_REG(addr))+Z_OFFSET(addr)] || } else if (Z_MODE(addr) == IS_REG) { @@ -1182,16 +1200,7 @@ static void* dasm_labels[zend_lb_MAX]; || if (op == last_valid_opline) { | mov aword EX->opline, IP || } else { -| .if X64 -|| if (IS_32BIT(op)) { -| mov aword EX->opline, ((ptrdiff_t)op) -|| } else { -| mov64 r0, ((ptrdiff_t)op) -| mov aword EX->opline, r0 -|| } -| .else -| mov aword EX->opline, op -| .endif +| ADDR_OP2_2 mov, aword EX->opline, op, r0 || } |.endmacro @@ -1535,9 +1544,9 @@ static int zend_jit_interrupt_handler_stub(dasm_State **Dst) { |->interrupt_handler: | //EG(vm_interrupt) = 0; - | mov byte [&EG(vm_interrupt)], 0 + | MEM_OP2_1 mov, byte, &EG(vm_interrupt), 0, r0 | //if (EG(timed_out)) { - | cmp byte [&EG(timed_out)], 0 + | MEM_OP2_1 cmp, byte, &EG(timed_out), 0, r0 | je >1 | //zend_timeout(0); |.if X64 @@ -1563,7 +1572,7 @@ static int zend_jit_interrupt_handler_stub(dasm_State **Dst) |.endif | //ZEND_VM_ENTER(); | //execute_data = EG(current_execute_data); - | mov FP, aword [&EG(current_execute_data)] + | MEM_OP2_2 mov, FP, aword, &EG(current_execute_data), r0 | // LOAD_OPLINE(); | mov IP, EX->opline | //} @@ -1613,7 +1622,7 @@ static int zend_jit_leave_throw_stub(dasm_State **Dst) | cmp byte OP:IP->opcode, ZEND_HANDLE_EXCEPTION | je >5 | // EG(opline_before_exception) = opline; - | mov aword [&EG(opline_before_exception)], IP + | MEM_OP2_1 mov, aword, &EG(opline_before_exception), IP, r0 |5: | // opline = EG(exception_op); | LOAD_ADDR IP, &EG(exception_op) @@ -1982,7 +1991,7 @@ static int zend_jit_check_timeout(dasm_State **Dst, const zend_op *opline) return 0; } #else - | cmp byte [&EG(vm_interrupt)], 0 + | MEM_OP2_1 cmp, byte, &EG(vm_interrupt), 0, r0 | jne >1 |.cold_code |1: @@ -1992,14 +2001,14 @@ static int zend_jit_check_timeout(dasm_State **Dst, const zend_op *opline) return 1; #endif } - | cmp byte [&EG(vm_interrupt)], 0 + | MEM_OP2_1 cmp, byte, &EG(vm_interrupt), 0, r0 | jne ->interrupt_handler return 1; } static int zend_jit_check_exception(dasm_State **Dst) { - | cmp aword [&EG(exception)], 0 + | MEM_OP2_1 cmp, aword, &EG(exception), 0, r0 | jne ->exception_handler return 1; } @@ -2007,7 +2016,7 @@ static int zend_jit_check_exception(dasm_State **Dst) static int zend_jit_check_exception_undef_result(dasm_State **Dst, const zend_op *opline) { if (opline->result_type & (IS_TMP_VAR|IS_VAR)) { - | cmp aword [&EG(exception)], 0 + | MEM_OP2_1 cmp, aword, &EG(exception), 0, r0 | mov r0, opline->result.var | jne ->exception_handler_undef return 1; @@ -6181,9 +6190,8 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, int b, |3: } if (zend_may_throw(opline, op_array, ssa)) { - if (!zend_jit_check_exception_undef_result(Dst, opline)) { - return 0; - } + | MEM_OP2_1 cmp, aword, &EG(exception), 0, r1 + | jne ->exception_handler_undef } | test r0, r0 @@ -6320,9 +6328,9 @@ static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, zen zend_jit_start_reuse_ip(); | // if (UNEXPECTED(used_stack > (size_t)(((char*)EG(vm_stack_end)) - (char*)call))) { - | mov RX, aword [&EG(vm_stack_top)] + | MEM_OP2_2 mov, RX, aword, &EG(vm_stack_top), r1 | // Check Stack Overflow - | mov r2, aword [&EG(vm_stack_end)] + | MEM_OP2_2 mov, r2, aword, &EG(vm_stack_end), r1 | sub r2, RX if (func) { | cmp r2, used_stack @@ -6347,9 +6355,9 @@ static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, zen |.code if (func) { - | add aword [&EG(vm_stack_top)], used_stack + | MEM_OP2_1 add, aword, &EG(vm_stack_top), used_stack, r1 } else { - | add aword [&EG(vm_stack_top)], FCARG1a + | MEM_OP2_1 add, aword, &EG(vm_stack_top), FCARG1a, r1 } | // zend_vm_init_call_frame(call, call_info, func, num_args, called_scope, object); | // ZEND_SET_CALL_INFO(call, 0, call_info); @@ -6357,16 +6365,7 @@ static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, zen | // call->func = func; if (func && func->type == ZEND_INTERNAL_FUNCTION) { |1: - |.if X64 - if (!IS_32BIT(func)) { - | mov aword EX:RX->func, func - } else { - | LOAD_ADDR r0, func - | mov aword EX:RX->func, r0 - } - |.else - | mov aword EX:RX->func, func - |.endif + | ADDR_OP2_2 mov, aword EX:RX->func, func, r1 } else { | mov aword EX:RX->func, r0 |1: @@ -6647,7 +6646,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar |.cold_code |1: | EXT_CALL zend_jit_deprecated_or_abstract_helper, r0 - | cmp aword [&EG(exception)], 0 + | MEM_OP2_1 cmp, aword, &EG(exception), 0, r0 | jne ->exception_handler | mov r0, EX:RX->func // reload | jmp >1 @@ -6658,7 +6657,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar | jmp ->exception_handler } else if (func->common.fn_flags & ZEND_ACC_DEPRECATED) { | EXT_CALL zend_jit_deprecated_or_abstract_helper, r0 - | cmp aword [&EG(exception)], 0 + | MEM_OP2_1 cmp, aword, &EG(exception), 0, r0 | jne ->exception_handler | mov r0, EX:RX->func // reload } @@ -6723,7 +6722,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar |.endif | // EG(current_execute_data) = execute_data; - | mov aword [&EG(current_execute_data)], RX + | MEM_OP2_1 mov, aword, &EG(current_execute_data), RX, r1 | mov FP, RX | // opline = op_array->opcodes; @@ -6790,7 +6789,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar /* recursive call */ #ifdef CONTEXT_THREADED_JIT | call =>(call_info->num_args+ssa->cfg.blocks_count) - | cmp aword [&EG(exception)], 0 + | MEM_OP2_1 cmp, aword, &EG(exception), 0, r0 | jne ->exception_handler if (!func) { | jmp >9 @@ -6801,7 +6800,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar } else { #ifdef CONTEXT_THREADED_JIT | call aword [IP] - | cmp aword [&EG(exception)], 0 + | MEM_OP2_1 cmp, aword, &EG(exception), 0, r0 | jne ->exception_handler if (!func) { | jmp >9 @@ -6844,7 +6843,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar |.cold_code |1: | EXT_CALL zend_jit_deprecated_or_abstract_helper, r0 - | cmp aword [&EG(exception)], 0 + | MEM_OP2_1 cmp, aword, &EG(exception), 0, r0 | jne ->exception_handler | mov r0, EX:RX->func // reload | jmp >1 @@ -6855,7 +6854,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar | jmp ->exception_handler } else if (func->common.fn_flags & ZEND_ACC_DEPRECATED) { | EXT_CALL zend_jit_deprecated_or_abstract_helper, r0 - | cmp aword [&EG(exception)], 0 + | MEM_OP2_1 cmp, aword, &EG(exception), 0, r0 | jne ->exception_handler | mov r0, EX:RX->func // reload } @@ -6870,7 +6869,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar | SET_Z_TYPE_INFO FCARG2a, IS_NULL | // EG(current_execute_data) = execute_data; - | mov aword [&EG(current_execute_data)], RX + | MEM_OP2_1 mov, aword, &EG(current_execute_data), RX, r1 if (!func || (func->common.fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) { if (!func) { @@ -6915,7 +6914,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar |.endif | // EG(current_execute_data) = execute_data; - | mov aword [&EG(current_execute_data)], FP + | MEM_OP2_1 mov, aword, &EG(current_execute_data), FP, r0 | // zend_vm_stack_free_args(call); if (func) { @@ -6939,7 +6938,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar | GET_Z_PTR r0, RX + offsetof(zend_execute_data, This) if (opline->op1.num & ZEND_CALL_CTOR) { | // if (UNEXPECTED(EG(exception) - | cmp aword [&EG(exception)], 0 + | MEM_OP2_1 cmp, aword, &EG(exception), 0, r1 | je >1 | // GC_REFCOUNT(object)--; | GC_DELREF r0 @@ -6964,7 +6963,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar | EXT_CALL zend_jit_free_call_frame, r0 | jmp >1 |.code - | mov aword [&EG(vm_stack_top)], RX + | MEM_OP2_1 mov, aword, &EG(vm_stack_top), RX, r0 |1: if (!RETURN_VALUE_USED(opline)) { @@ -6979,7 +6978,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar } | // if (UNEXPECTED(EG(exception) != NULL)) { - | cmp aword [&EG(exception)], 0 + | MEM_OP2_1 cmp, aword, &EG(exception), 0, r0 | jne >1 |.cold_code |1: @@ -7807,7 +7806,7 @@ static int zend_jit_leave_func(dasm_State **Dst, const zend_op *opline, zend_op_ if (op_array->scope || (op_array->fn_flags & ZEND_ACC_CLOSURE)) { | // EG(current_execute_data) = EX(prev_execute_data); | mov r0, EX->prev_execute_data - | mov aword [&EG(current_execute_data)], r0 + | MEM_OP2_1 mov, aword, &EG(current_execute_data), r0, r1 if (op_array->scope) { | // if (call_info & ZEND_CALL_RELEASE_THIS) | test FCARG1d, ZEND_CALL_RELEASE_THIS @@ -7822,7 +7821,7 @@ static int zend_jit_leave_func(dasm_State **Dst, const zend_op *opline, zend_op_ | mov r0, EX->This.value.obj if (op_array->scope && op_array->scope->constructor == (zend_function*)op_array) { | // if (UNEXPECTED(EG(exception) != NULL) - | cmp aword [&EG(exception)], 0 + | MEM_OP2_1 cmp, aword, &EG(exception), 0, r1 | jne >6 |.cold_code |6: @@ -7863,19 +7862,19 @@ static int zend_jit_leave_func(dasm_State **Dst, const zend_op *opline, zend_op_ } |4: | // EG(vm_stack_top) = (zval*)execute_data; - | mov aword [&EG(vm_stack_top)], FP + | MEM_OP2_1 mov, aword, &EG(vm_stack_top), FP, r0 | // execute_data = EX(prev_execute_data); | mov FP, EX->prev_execute_data } else { | // EG(vm_stack_top) = (zval*)execute_data; - | mov aword [&EG(vm_stack_top)], FP + | MEM_OP2_1 mov, aword, &EG(vm_stack_top), FP, r0 | // execute_data = EX(prev_execute_data); | mov FP, EX->prev_execute_data | // EG(current_execute_data) = execute_data - | mov aword [&EG(current_execute_data)], FP + | MEM_OP2_1 mov, aword, &EG(current_execute_data), FP, r0 } | // if (EG(exception)) - | cmp aword [&EG(exception)], 0 + | MEM_OP2_1 cmp, aword, &EG(exception), 0, r0 | mov IP, EX->opline | jne ->leave_throw_handler | // opline = EX(opline) + 1 @@ -8342,7 +8341,7 @@ static int zend_jit_bind_global(dasm_State **Dst, const zend_op *opline, zend_op | mov r0, aword [r0 + Z_CACHE_SLOT_P(varname)] | sub r0, 1 //if (EXPECTED(idx < EG(symbol_table).nNumUsed)) - | cmp r0, [&EG(symbol_table).nNumUsed] + | MEM_OP2_2 cmp, eax, dword, &EG(symbol_table).nNumUsed, r1 | jae >9 //Bucket *p = EG(symbol_table).arData + idx; |.if X64 @@ -8350,37 +8349,22 @@ static int zend_jit_bind_global(dasm_State **Dst, const zend_op *opline, zend_op |.else | imul r0, sizeof(Bucket) |.endif - |.if X64 - | LOAD_ADDR r1, &EG(symbol_table.arData) - | add r0, [r1] - |.else - | add r0, [&EG(symbol_table).arData] - |.endif + | MEM_OP2_2 add, r0, aword, &EG(symbol_table).arData, r1 | IF_Z_TYPE r0, IS_UNDEF, >9 // (EXPECTED(p->key == Z_STR_P(varname)) - |.if X64 - | LOAD_ADDR r1, Z_PTR_P(varname) - | cmp [r0 + offsetof(Bucket, key)], r1 - |.else - | cmp aword [r0 + offsetof(Bucket, key)], Z_PTR_P(varname) - |.endif + | ADDR_OP2_2 cmp, aword [r0 + offsetof(Bucket, key)], Z_PTR_P(varname), r1 | jne >1 |.cold_code |1: //(EXPECTED(p->h == ZSTR_H(Z_STR_P(varname))) - |.if X64 - | mov64 r1, ZSTR_H(Z_STR_P(varname)) - | cmp qword [r0 + offsetof(Bucket, h)], r1 - |.else - | cmp dword [r0 + offsetof(Bucket, h)], ZSTR_H(Z_STR_P(varname)) - |.endif + | ADDR_OP2_2 cmp, aword [r0 + offsetof(Bucket, h)], ZSTR_H(Z_STR_P(varname)), r1 | jne >9 //EXPECTED(p->key != NULL) | mov r1, [r0 + offsetof(Bucket, key)] | test r1, r1 | jz >9 //EXPECTED(ZSTR_LEN(p->key) == Z_STRLEN_P(varname)) - | cmp dword [r1 + offsetof(zend_string, len)], Z_STRLEN_P(varname) + | ADDR_OP2_2 cmp, aword [r1 + offsetof(zend_string, len)], Z_STRLEN_P(varname), r2 | jne >9 //EXPECTED(memcmp(ZSTR_VAL(p->key), Z_STRVAL_P(varname), Z_STRLEN_P(varname)) == 0) | add r1, offsetof(zend_string, val) From 506b895d3050854a5165075e6fbde977bca27361 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 19 Apr 2017 13:01:54 +0300 Subject: [PATCH 443/569] Fixed bug in register-allocator --- ext/opcache/jit/zend_jit.c | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index d5574c582a14c..f46fc9c6d60be 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -2168,7 +2168,20 @@ static zend_lifetime_interval** zend_jit_allocate_registers(zend_op_array *op_ar intervals[i]->store && (ssa->vars[i].use_chain < 0 || zend_ssa_next_use(ssa->ops, i, ssa->vars[i].use_chain) < 0)) { - intervals[i] = NULL; + zend_bool may_remove = 1; + zend_ssa_phi *phi = ssa->vars[i].phi_use_chain; + + while (phi) { + if (intervals[phi->ssa_var] && + !intervals[phi->ssa_var]->load) { + may_remove = 0; + break; + } + phi = zend_ssa_next_use_phi(ssa, i, phi); + } + if (may_remove) { + intervals[i] = NULL; + } } } } From 70a945c99baa309e024cdbb51f539db61b644b25 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 19 Apr 2017 14:56:56 +0300 Subject: [PATCH 444/569] Support for edge cases in shift operators --- ext/opcache/jit/zend_jit_x86.dasc | 96 +++++++++++++++++++--- ext/opcache/tests/jit/shift_left_001.phpt | 37 +++++++++ ext/opcache/tests/jit/shift_left_002.phpt | 49 +++++++++++ ext/opcache/tests/jit/shift_right_001.phpt | 43 ++++++++++ ext/opcache/tests/jit/shift_right_002.phpt | 55 +++++++++++++ 5 files changed, 270 insertions(+), 10 deletions(-) create mode 100644 ext/opcache/tests/jit/shift_left_001.phpt create mode 100644 ext/opcache/tests/jit/shift_left_002.phpt create mode 100644 ext/opcache/tests/jit/shift_right_001.phpt create mode 100644 ext/opcache/tests/jit/shift_right_002.phpt diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 7acbb44224a58..381f6ce45ae10 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -1861,7 +1861,6 @@ static int zend_jit_cannot_add_element_stub(dasm_State **Dst) static int zend_jit_not_obj_stub(dasm_State **Dst) { |->not_obj: - |8: |.if X64 | xor CARG1, CARG1 | LOAD_ADDR CARG2, "Using $this when not in object context" @@ -1877,6 +1876,24 @@ static int zend_jit_not_obj_stub(dasm_State **Dst) return 1; } +static int zend_jit_negative_shift_stub(dasm_State **Dst) +{ + |->negative_shift: + |.if X64 + | LOAD_ADDR CARG1, zend_ce_arithmetic_error + | LOAD_ADDR CARG2, "Bit shift by negative number" + | EXT_CALL zend_throw_error, r0 + |.else + | sub r4, 8 + | push "Bit shift by negative number" + | PUSH_ADDR zend_ce_arithmetic_error, r0 + | EXT_CALL zend_throw_error, r0 + | add r4, 16 + |.endif + | jmp ->exception_handler + return 1; +} + static int zend_jit_double_one_stub(dasm_State **Dst) { |->one: @@ -1899,6 +1916,7 @@ static const zend_jit_stub zend_jit_stubs[] = { JIT_STUB(undefined_index_ex), JIT_STUB(cannot_add_element_ex), JIT_STUB(not_obj), + JIT_STUB(negative_shift), JIT_STUB(double_one), }; @@ -3087,6 +3105,7 @@ static int zend_jit_long_math_helper(dasm_State **Dst, znode_op op2, zend_jit_addr op2_addr, uint32_t op2_info, + int op2_ssa_var, uint32_t res_var, zend_jit_addr res_addr, uint32_t res_info, @@ -3127,31 +3146,80 @@ static int zend_jit_long_math_helper(dasm_State **Dst, result_reg = ZREG_R0; } - | GET_ZVAL_LVAL result_reg, op1_addr if (opcode == ZEND_SL || opcode == ZEND_ASSIGN_SL) { - //TODO: add checks ??? if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { - | shl Ra(result_reg), Z_LVAL_P(Z_ZV(op2_addr)) + zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr)); + + if (UNEXPECTED((zend_ulong)op2_lval >= SIZEOF_ZEND_LONG * 8)) { + if (EXPECTED(op2_lval > 0)) { + | xor Ra(result_reg), Ra(result_reg) + } else { + | jmp ->negative_shift + } + } else { + | GET_ZVAL_LVAL result_reg, op1_addr + | shl Ra(result_reg), op2_lval + } } else { if (Z_MODE(op2_addr) != IS_REG || Z_REG(op2_addr) != ZREG_RCX) { | GET_ZVAL_LVAL ZREG_RCX, op2_addr } + if (op2_ssa_var < 0 || + !ssa->var_info[op2_ssa_var].has_range || + ssa->var_info[op2_ssa_var].range.min < 0 || + ssa->var_info[op2_ssa_var].range.max >= SIZEOF_ZEND_LONG * 8) { + | cmp r1, (SIZEOF_ZEND_LONG*8) + | jae >1 + |.cold_code + |1: + | jl ->negative_shift + | xor Ra(result_reg), Ra(result_reg) + | jmp >1 + |.code + } + | GET_ZVAL_LVAL result_reg, op1_addr | shl Ra(result_reg), cl + |1: } } else if (opcode == ZEND_SR || opcode == ZEND_ASSIGN_SR) { - //TODO: add checks ??? + | GET_ZVAL_LVAL result_reg, op1_addr if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { - | shr Ra(result_reg), Z_LVAL_P(Z_ZV(op2_addr)) + zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr)); + + if (UNEXPECTED((zend_ulong)op2_lval >= SIZEOF_ZEND_LONG * 8)) { + if (EXPECTED(op2_lval > 0)) { + | sar Ra(result_reg), (SIZEOF_ZEND_LONG * 8) - 1 + } else { + | jmp ->negative_shift + } + } else { + | sar Ra(result_reg), op2_lval + } } else { if (Z_MODE(op2_addr) != IS_REG || Z_REG(op2_addr) != ZREG_RCX) { | GET_ZVAL_LVAL ZREG_RCX, op2_addr } - | shr Ra(result_reg), cl + if (op2_ssa_var < 0 || + !ssa->var_info[op2_ssa_var].has_range || + ssa->var_info[op2_ssa_var].range.min < 0 || + ssa->var_info[op2_ssa_var].range.max >= SIZEOF_ZEND_LONG * 8) { + | cmp r1, (SIZEOF_ZEND_LONG*8) + | jae >1 + |.cold_code + |1: + | jl ->negative_shift + | sar Ra(result_reg), (SIZEOF_ZEND_LONG * 8) - 1 + | jmp >1 + |.code + } + | sar Ra(result_reg), cl + |1: } } else if (opcode == ZEND_MOD || opcode == ZEND_ASSIGN_MOD) { //TODO: add checks ??? zend_jit_addr val_addr = op2_addr; + | GET_ZVAL_LVAL result_reg, op1_addr if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { | GET_ZVAL_LVAL ZREG_RCX, op2_addr val_addr = ZEND_ADDR_REG(ZREG_RCX); @@ -3164,8 +3232,10 @@ static int zend_jit_long_math_helper(dasm_State **Dst, } result_reg = ZREG_RDX; } else if (same_ops) { + | GET_ZVAL_LVAL result_reg, op1_addr | LONG_MATH_REG opcode, Ra(result_reg), Ra(result_reg) } else { + | GET_ZVAL_LVAL result_reg, op1_addr | LONG_MATH opcode, Ra(result_reg), op2_addr } @@ -3256,11 +3326,13 @@ static int zend_jit_long_math(dasm_State **Dst, const zend_op *opline, int *opnu { uint32_t op1_info, op2_info, res_use_info; zend_jit_addr op1_addr, op2_addr, res_addr; + int op2_ssa_var; if (!ssa->ops || !ssa->var_info) { goto fallback; } + op2_ssa_var = ssa->ops[opline - op_array->opcodes].op2_use; op1_info = OP1_INFO(); op2_info = OP2_INFO(); res_use_info = RES_USE_INFO(); @@ -3294,7 +3366,7 @@ static int zend_jit_long_math(dasm_State **Dst, const zend_op *opline, int *opnu res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, ra, ssa->ops[opline - op_array->opcodes].result_def); } - if (!zend_jit_long_math_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, opline->result.var, res_addr, RES_INFO(), res_use_info, 0)) { + if (!zend_jit_long_math_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, op2_ssa_var, opline->result.var, res_addr, RES_INFO(), res_use_info, 0)) { return 0; } if (ra && !zend_jit_store_ssa_var_if_necessary(Dst, ssa, ra, res_addr, ssa->ops[opline - op_array->opcodes].result_def, ssa->ops[opline - op_array->opcodes].result_use)) { @@ -4243,6 +4315,7 @@ static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, zend_ { uint32_t op1_info, op2_info; zend_jit_addr op1_addr, op2_addr, op3_addr, var_addr; + int op3_ssa_var; if (opline->op1_type != IS_CV || opline->result_type != IS_UNUSED) { goto fallback; @@ -4252,6 +4325,7 @@ static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, zend_ goto fallback; } + op3_ssa_var = ssa->ops[opline - op_array->opcodes + 1].op1_use; op1_info = OP1_INFO(); op2_info = OP2_INFO(); @@ -4379,7 +4453,7 @@ static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, zend_ case ZEND_ASSIGN_SL: case ZEND_ASSIGN_SR: case ZEND_ASSIGN_MOD: - if (!zend_jit_long_math_helper(Dst, opline, op_array, ssa, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, op3_addr, OP1_DATA_INFO(), 0, var_addr, OP1_DEF_INFO(), var_info, 1)) { + if (!zend_jit_long_math_helper(Dst, opline, op_array, ssa, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, op3_addr, OP1_DATA_INFO(), op3_ssa_var, 0, var_addr, OP1_DEF_INFO(), var_info, 1)) { return 0; } break; @@ -4531,6 +4605,7 @@ static int zend_jit_assign_op(dasm_State **Dst, const zend_op *opline, zend_op_a { uint32_t op1_info, op2_info; zend_jit_addr op1_addr, op2_addr; + int op2_ssa_var; if (opline->extended_value == ZEND_ASSIGN_DIM) { return zend_jit_assign_dim_op(Dst, opline, op_array, ssa); @@ -4546,6 +4621,7 @@ static int zend_jit_assign_op(dasm_State **Dst, const zend_op *opline, zend_op_a goto fallback; } + op2_ssa_var = ssa->ops[opline - op_array->opcodes].op2_use; op1_info = OP1_INFO(); op2_info = OP2_INFO(); @@ -4605,7 +4681,7 @@ static int zend_jit_assign_op(dasm_State **Dst, const zend_op *opline, zend_op_a case ZEND_ASSIGN_SL: case ZEND_ASSIGN_SR: case ZEND_ASSIGN_MOD: - return zend_jit_long_math_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, opline->op1.var, op1_addr, OP1_DEF_INFO(), op1_info, 1); + return zend_jit_long_math_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, op2_ssa_var, opline->op1.var, op1_addr, OP1_DEF_INFO(), op1_info, 1); case ZEND_ASSIGN_CONCAT: return zend_jit_concat_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, op1_addr, OP1_DEF_INFO()); default: diff --git a/ext/opcache/tests/jit/shift_left_001.phpt b/ext/opcache/tests/jit/shift_left_001.phpt new file mode 100644 index 0000000000000..81868869335c2 --- /dev/null +++ b/ext/opcache/tests/jit/shift_left_001.phpt @@ -0,0 +1,37 @@ +--TEST-- +JIT Shift Left: 001 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +opcache.protect_memory=1 +--SKIPIF-- + +--FILE-- +getMessage() . "\n"; +} +try { + var_dump(shl(1, -1)); +} catch (Throwable $e) { + echo "Exception (" . get_class($e) . "): " . $e->getMessage() . "\n"; +} +?> +--EXPECT-- +int(1) +int(2) +int(4) +int(-4) +int(0) +Exception (ArithmeticError): Bit shift by negative number diff --git a/ext/opcache/tests/jit/shift_left_002.phpt b/ext/opcache/tests/jit/shift_left_002.phpt new file mode 100644 index 0000000000000..65c5cfd79ce55 --- /dev/null +++ b/ext/opcache/tests/jit/shift_left_002.phpt @@ -0,0 +1,49 @@ +--TEST-- +JIT Shift Left: 002 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +opcache.protect_memory=1 +--SKIPIF-- + +--FILE-- +getMessage() . "\n"; +} +try { + var_dump(shlNEG(1)); +} catch (Throwable $e) { + echo "Exception (" . get_class($e) . "): " . $e->getMessage() . "\n"; +} +?> +--EXPECT-- +int(1) +int(2) +int(4) +int(-4) +int(0) +Exception (ArithmeticError): Bit shift by negative number diff --git a/ext/opcache/tests/jit/shift_right_001.phpt b/ext/opcache/tests/jit/shift_right_001.phpt new file mode 100644 index 0000000000000..8f7f7ffc83c6e --- /dev/null +++ b/ext/opcache/tests/jit/shift_right_001.phpt @@ -0,0 +1,43 @@ +--TEST-- +JIT Shift Right: 001 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +opcache.protect_memory=1 +--SKIPIF-- + +--FILE-- +> $b; +} +var_dump(shr(256, 0)); +var_dump(shr(256, 1)); +var_dump(shr(256, 2)); +var_dump(shr(-8, 2)); +try { + var_dump(shr(1, 64)); +} catch (Throwable $e) { + echo "Exception " . $e->getMessage() . "\n"; +} +try { + var_dump(shr(-1, 64)); +} catch (Throwable $e) { + echo "Exception " . $e->getMessage() . "\n"; +} +try { + var_dump(shr(1, -1)); +} catch (Throwable $e) { + echo "Exception (" . get_class($e) . "): " . $e->getMessage() . "\n"; +} +?> +--EXPECT-- +int(256) +int(128) +int(64) +int(-2) +int(0) +int(-1) +Exception (ArithmeticError): Bit shift by negative number diff --git a/ext/opcache/tests/jit/shift_right_002.phpt b/ext/opcache/tests/jit/shift_right_002.phpt new file mode 100644 index 0000000000000..5ead9cbeb370d --- /dev/null +++ b/ext/opcache/tests/jit/shift_right_002.phpt @@ -0,0 +1,55 @@ +--TEST-- +JIT Shift Right: 002 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +opcache.protect_memory=1 +--SKIPIF-- + +--FILE-- +> 0; +} +function shr1(int $a) { + return $a >> 1; +} +function shr2(int $a) { + return $a >> 2; +} +function shr64(int $a) { + return $a >> 64; +} +function shrNEG(int $a) { + return $a >> -1; +} +var_dump(shr0(256)); +var_dump(shr1(256)); +var_dump(shr2(256)); +var_dump(shr2(-8)); +try { + var_dump(shr64(1)); +} catch (Throwable $e) { + echo "Exception " . $e->getMessage() . "\n"; +} +try { + var_dump(shr64(-1)); +} catch (Throwable $e) { + echo "Exception " . $e->getMessage() . "\n"; +} +try { + var_dump(shrNEG(1)); +} catch (Throwable $e) { + echo "Exception (" . get_class($e) . "): " . $e->getMessage() . "\n"; +} +?> +--EXPECT-- +int(256) +int(128) +int(64) +int(-2) +int(0) +int(-1) +Exception (ArithmeticError): Bit shift by negative number From 91965cf90592c34cb5bdebac7c89f21599593137 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 19 Apr 2017 16:03:05 +0300 Subject: [PATCH 445/569] Support for modulo by zero --- ext/opcache/jit/zend_jit_x86.dasc | 88 +++++++++++++++++++++++------- ext/opcache/tests/jit/mod_001.phpt | 41 ++++++++++++++ ext/opcache/tests/jit/mod_002.phpt | 53 ++++++++++++++++++ 3 files changed, 162 insertions(+), 20 deletions(-) create mode 100644 ext/opcache/tests/jit/mod_001.phpt create mode 100644 ext/opcache/tests/jit/mod_002.phpt diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 381f6ce45ae10..aca61b3ea8f57 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -1894,6 +1894,24 @@ static int zend_jit_negative_shift_stub(dasm_State **Dst) return 1; } +static int zend_jit_mod_by_zero_stub(dasm_State **Dst) +{ + |->mod_by_zero: + |.if X64 + | LOAD_ADDR CARG1, zend_ce_division_by_zero_error + | LOAD_ADDR CARG2, "Modulo by zero" + | EXT_CALL zend_throw_error, r0 + |.else + | sub r4, 8 + | push "Modulo by zero" + | PUSH_ADDR zend_ce_division_by_zero_error, r0 + | EXT_CALL zend_throw_error, r0 + | add r4, 16 + |.endif + | jmp ->exception_handler + return 1; +} + static int zend_jit_double_one_stub(dasm_State **Dst) { |->one: @@ -1917,6 +1935,7 @@ static const zend_jit_stub zend_jit_stubs[] = { JIT_STUB(cannot_add_element_ex), JIT_STUB(not_obj), JIT_STUB(negative_shift), + JIT_STUB(mod_by_zero), JIT_STUB(double_one), }; @@ -3101,6 +3120,7 @@ static int zend_jit_long_math_helper(dasm_State **Dst, znode_op op1, zend_jit_addr op1_addr, uint32_t op1_info, + int op1_ssa_var, zend_uchar op2_type, znode_op op2, zend_jit_addr op2_addr, @@ -3125,7 +3145,10 @@ static int zend_jit_long_math_helper(dasm_State **Dst, | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6 } - if (opcode == ZEND_MOD && Z_MODE(op2_addr) == IS_CONST_ZVAL) { + if (opcode == ZEND_MOD && Z_MODE(op2_addr) == IS_CONST_ZVAL && + op1_ssa_var >= 0 && + ssa->var_info[op1_ssa_var].has_range && + ssa->var_info[op1_ssa_var].range.min >= 0) { zend_long l = Z_LVAL_P(Z_ZV(op2_addr)); if (zend_long_is_power_of_two(l)) { @@ -3216,21 +3239,44 @@ static int zend_jit_long_math_helper(dasm_State **Dst, |1: } } else if (opcode == ZEND_MOD || opcode == ZEND_ASSIGN_MOD) { - //TODO: add checks ??? - zend_jit_addr val_addr = op2_addr; - - | GET_ZVAL_LVAL result_reg, op1_addr if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { - | GET_ZVAL_LVAL ZREG_RCX, op2_addr - val_addr = ZEND_ADDR_REG(ZREG_RCX); - } - | xor Ra(ZREG_RDX), Ra(ZREG_RDX) - if (Z_MODE(val_addr) == IS_MEM_ZVAL) { - | idiv aword [Ra(Z_REG(val_addr))+Z_OFFSET(val_addr)] - } else if (Z_MODE(val_addr) == IS_REG) { - | idiv Ra(Z_REG(val_addr)) + zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr)); + + if (op2_lval == 0) { + | jmp ->mod_by_zero + } else if (op2_lval == -1) { + | xor Ra(result_reg), Ra(result_reg) + } else { + result_reg = ZREG_RDX; + | GET_ZVAL_LVAL ZREG_RAX, op1_addr + | GET_ZVAL_LVAL ZREG_RCX, op2_addr + | cdq + | idiv Ra(ZREG_RCX) + } + } else { + if (op2_ssa_var < 0 || + !ssa->var_info[op2_ssa_var].has_range || + (ssa->var_info[op2_ssa_var].range.min <= 0 && + ssa->var_info[op2_ssa_var].range.max >= 0)) { + if (Z_MODE(op2_addr) == IS_MEM_ZVAL) { + | cmp aword [Ra(Z_REG(op2_addr))+Z_OFFSET(op2_addr)], 0 + } else if (Z_MODE(op2_addr) == IS_REG) { + | test Ra(Z_REG(op2_addr)), Ra(Z_REG(op2_addr)) + } + | jz ->mod_by_zero + } + + //TODO: add check for -1 ??? + + result_reg = ZREG_RDX; + | GET_ZVAL_LVAL ZREG_RAX, op1_addr + | cdq + if (Z_MODE(op2_addr) == IS_MEM_ZVAL) { + | idiv aword [Ra(Z_REG(op2_addr))+Z_OFFSET(op2_addr)] + } else if (Z_MODE(op2_addr) == IS_REG) { + | idiv Ra(Z_REG(op2_addr)) + } } - result_reg = ZREG_RDX; } else if (same_ops) { | GET_ZVAL_LVAL result_reg, op1_addr | LONG_MATH_REG opcode, Ra(result_reg), Ra(result_reg) @@ -3239,8 +3285,8 @@ static int zend_jit_long_math_helper(dasm_State **Dst, | LONG_MATH opcode, Ra(result_reg), op2_addr } + | SET_ZVAL_LVAL res_addr, Ra(result_reg) if (Z_MODE(res_addr) == IS_MEM_ZVAL) { - | SET_ZVAL_LVAL res_addr, Ra(result_reg) if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_LONG) { | SET_ZVAL_TYPE_INFO res_addr, IS_LONG @@ -3326,12 +3372,13 @@ static int zend_jit_long_math(dasm_State **Dst, const zend_op *opline, int *opnu { uint32_t op1_info, op2_info, res_use_info; zend_jit_addr op1_addr, op2_addr, res_addr; - int op2_ssa_var; + int op1_ssa_var, op2_ssa_var; if (!ssa->ops || !ssa->var_info) { goto fallback; } + op1_ssa_var = ssa->ops[opline - op_array->opcodes].op1_use; op2_ssa_var = ssa->ops[opline - op_array->opcodes].op2_use; op1_info = OP1_INFO(); op2_info = OP2_INFO(); @@ -3366,7 +3413,7 @@ static int zend_jit_long_math(dasm_State **Dst, const zend_op *opline, int *opnu res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, ra, ssa->ops[opline - op_array->opcodes].result_def); } - if (!zend_jit_long_math_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, op2_ssa_var, opline->result.var, res_addr, RES_INFO(), res_use_info, 0)) { + if (!zend_jit_long_math_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, op1_addr, op1_info, op1_ssa_var, opline->op2_type, opline->op2, op2_addr, op2_info, op2_ssa_var, opline->result.var, res_addr, RES_INFO(), res_use_info, 0)) { return 0; } if (ra && !zend_jit_store_ssa_var_if_necessary(Dst, ssa, ra, res_addr, ssa->ops[opline - op_array->opcodes].result_def, ssa->ops[opline - op_array->opcodes].result_use)) { @@ -4453,7 +4500,7 @@ static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, zend_ case ZEND_ASSIGN_SL: case ZEND_ASSIGN_SR: case ZEND_ASSIGN_MOD: - if (!zend_jit_long_math_helper(Dst, opline, op_array, ssa, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, op3_addr, OP1_DATA_INFO(), op3_ssa_var, 0, var_addr, OP1_DEF_INFO(), var_info, 1)) { + if (!zend_jit_long_math_helper(Dst, opline, op_array, ssa, IS_CV, opline->op1, var_addr, var_info, -1, (opline+1)->op1_type, (opline+1)->op1, op3_addr, OP1_DATA_INFO(), op3_ssa_var, 0, var_addr, OP1_DEF_INFO(), var_info, 1)) { return 0; } break; @@ -4605,7 +4652,7 @@ static int zend_jit_assign_op(dasm_State **Dst, const zend_op *opline, zend_op_a { uint32_t op1_info, op2_info; zend_jit_addr op1_addr, op2_addr; - int op2_ssa_var; + int op1_ssa_var, op2_ssa_var; if (opline->extended_value == ZEND_ASSIGN_DIM) { return zend_jit_assign_dim_op(Dst, opline, op_array, ssa); @@ -4621,6 +4668,7 @@ static int zend_jit_assign_op(dasm_State **Dst, const zend_op *opline, zend_op_a goto fallback; } + op1_ssa_var = ssa->ops[opline - op_array->opcodes].op1_use; op2_ssa_var = ssa->ops[opline - op_array->opcodes].op2_use; op1_info = OP1_INFO(); op2_info = OP2_INFO(); @@ -4681,7 +4729,7 @@ static int zend_jit_assign_op(dasm_State **Dst, const zend_op *opline, zend_op_a case ZEND_ASSIGN_SL: case ZEND_ASSIGN_SR: case ZEND_ASSIGN_MOD: - return zend_jit_long_math_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, op2_ssa_var, opline->op1.var, op1_addr, OP1_DEF_INFO(), op1_info, 1); + return zend_jit_long_math_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, op1_addr, op1_info, op1_ssa_var, opline->op2_type, opline->op2, op2_addr, op2_info, op2_ssa_var, opline->op1.var, op1_addr, OP1_DEF_INFO(), op1_info, 1); case ZEND_ASSIGN_CONCAT: return zend_jit_concat_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, op1_addr, OP1_DEF_INFO()); default: diff --git a/ext/opcache/tests/jit/mod_001.phpt b/ext/opcache/tests/jit/mod_001.phpt new file mode 100644 index 0000000000000..ea61b41b9488d --- /dev/null +++ b/ext/opcache/tests/jit/mod_001.phpt @@ -0,0 +1,41 @@ +--TEST-- +JIT MOD: 001 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +opcache.protect_memory=1 +--SKIPIF-- + +--FILE-- +getMessage() . "\n"; +} +try { + var_dump(mod(125, 0)); +} catch (Throwable $e) { + echo "Exception (" . get_class($e) . "): " . $e->getMessage() . "\n"; +} +?> +--EXPECT-- +int(26) +int(29) +int(-26) +int(-29) +int(26) +int(-26) +int(0) +Exception (DivisionByZeroError): Modulo by zero diff --git a/ext/opcache/tests/jit/mod_002.phpt b/ext/opcache/tests/jit/mod_002.phpt new file mode 100644 index 0000000000000..5674ca9884d09 --- /dev/null +++ b/ext/opcache/tests/jit/mod_002.phpt @@ -0,0 +1,53 @@ +--TEST-- +JIT MOD: 002 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +opcache.protect_memory=1 +--SKIPIF-- + +--FILE-- +getMessage() . "\n"; +} +try { + var_dump(mod0(125)); +} catch (Throwable $e) { + echo "Exception (" . get_class($e) . "): " . $e->getMessage() . "\n"; +} +?> +--EXPECT-- +int(26) +int(29) +int(-26) +int(-29) +int(26) +int(-26) +int(0) +Exception (DivisionByZeroError): Modulo by zero From aa6a7365afa003da5e3d16975aed6e7fbfa31d21 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 19 Apr 2017 16:13:09 +0300 Subject: [PATCH 446/569] Added old tests --- ext/opcache/tests/jit/assign_001.phpt | 27 +++++ ext/opcache/tests/jit/assign_002.phpt | 29 +++++ ext/opcache/tests/jit/assign_003.phpt | 21 ++++ ext/opcache/tests/jit/assign_004.phpt | 24 ++++ ext/opcache/tests/jit/assign_005.phpt | 24 ++++ ext/opcache/tests/jit/assign_006.phpt | 22 ++++ ext/opcache/tests/jit/assign_007.phpt | 21 ++++ ext/opcache/tests/jit/assign_008.phpt | 21 ++++ ext/opcache/tests/jit/assign_009.phpt | 21 ++++ ext/opcache/tests/jit/assign_010.phpt | 21 ++++ ext/opcache/tests/jit/assign_011.phpt | 28 +++++ ext/opcache/tests/jit/assign_012.phpt | 28 +++++ ext/opcache/tests/jit/assign_013.phpt | 21 ++++ ext/opcache/tests/jit/assign_014.phpt | 22 ++++ ext/opcache/tests/jit/assign_015.phpt | 21 ++++ ext/opcache/tests/jit/assign_016.phpt | 21 ++++ ext/opcache/tests/jit/assign_017.phpt | 21 ++++ ext/opcache/tests/jit/assign_018.phpt | 26 ++++ ext/opcache/tests/jit/assign_019.phpt | 24 ++++ ext/opcache/tests/jit/assign_020.phpt | 22 ++++ ext/opcache/tests/jit/assign_021.phpt | 24 ++++ ext/opcache/tests/jit/assign_022.phpt | 24 ++++ ext/opcache/tests/jit/assign_023.phpt | 24 ++++ ext/opcache/tests/jit/assign_024.phpt | 23 ++++ ext/opcache/tests/jit/assign_025.phpt | 24 ++++ ext/opcache/tests/jit/assign_026.phpt | 25 ++++ ext/opcache/tests/jit/assign_027.phpt | 22 ++++ ext/opcache/tests/jit/assign_028.phpt | 21 ++++ ext/opcache/tests/jit/assign_029.phpt | 22 ++++ ext/opcache/tests/jit/assign_030.phpt | 23 ++++ ext/opcache/tests/jit/assign_031.phpt | 19 +++ ext/opcache/tests/jit/assign_032.phpt | 27 +++++ ext/opcache/tests/jit/assign_033.phpt | 24 ++++ ext/opcache/tests/jit/assign_034.phpt | 21 ++++ ext/opcache/tests/jit/assign_dim_002.phpt | 128 ++++++++++++++++++++ ext/opcache/tests/jit/fetch_dim_r_001.phpt | 45 +++++++ ext/opcache/tests/jit/fetch_dim_r_002.phpt | 45 +++++++ ext/opcache/tests/jit/fetch_dim_r_003.phpt | 61 ++++++++++ ext/opcache/tests/jit/fetch_dim_r_004.phpt | 61 ++++++++++ ext/opcache/tests/jit/fetch_dim_rw_001.phpt | 27 +++++ ext/opcache/tests/jit/fetch_obj_001.phpt | 128 ++++++++++++++++++++ ext/opcache/tests/jit/fetch_obj_002.phpt | 41 +++++++ ext/opcache/tests/jit/fetch_obj_003.phpt | 46 +++++++ ext/opcache/tests/jit/fetch_obj_004.phpt | 43 +++++++ ext/opcache/tests/jit/inc_001.phpt | 22 ++++ ext/opcache/tests/jit/inc_002.phpt | 21 ++++ ext/opcache/tests/jit/inc_003.phpt | 22 ++++ ext/opcache/tests/jit/inc_004.phpt | 21 ++++ ext/opcache/tests/jit/inc_005.phpt | 23 ++++ ext/opcache/tests/jit/inc_006.phpt | 22 ++++ ext/opcache/tests/jit/inc_007.phpt | 21 ++++ ext/opcache/tests/jit/inc_008.phpt | 20 +++ ext/opcache/tests/jit/inc_009.phpt | 22 ++++ ext/opcache/tests/jit/inc_010.phpt | 21 ++++ ext/opcache/tests/jit/inc_011.phpt | 22 ++++ ext/opcache/tests/jit/inc_012.phpt | 21 ++++ ext/opcache/tests/jit/inc_013.phpt | 23 ++++ ext/opcache/tests/jit/inc_014.phpt | 22 ++++ ext/opcache/tests/jit/inc_015.phpt | 21 ++++ ext/opcache/tests/jit/inc_016.phpt | 20 +++ ext/opcache/tests/jit/inc_017.phpt | 20 +++ ext/opcache/tests/jit/inc_018.phpt | 20 +++ ext/opcache/tests/jit/inc_019.phpt | 27 +++++ ext/opcache/tests/jit/noval_001.phpt | 34 ++++++ 64 files changed, 1858 insertions(+) create mode 100644 ext/opcache/tests/jit/assign_001.phpt create mode 100644 ext/opcache/tests/jit/assign_002.phpt create mode 100644 ext/opcache/tests/jit/assign_003.phpt create mode 100644 ext/opcache/tests/jit/assign_004.phpt create mode 100644 ext/opcache/tests/jit/assign_005.phpt create mode 100644 ext/opcache/tests/jit/assign_006.phpt create mode 100644 ext/opcache/tests/jit/assign_007.phpt create mode 100644 ext/opcache/tests/jit/assign_008.phpt create mode 100644 ext/opcache/tests/jit/assign_009.phpt create mode 100644 ext/opcache/tests/jit/assign_010.phpt create mode 100644 ext/opcache/tests/jit/assign_011.phpt create mode 100644 ext/opcache/tests/jit/assign_012.phpt create mode 100644 ext/opcache/tests/jit/assign_013.phpt create mode 100644 ext/opcache/tests/jit/assign_014.phpt create mode 100644 ext/opcache/tests/jit/assign_015.phpt create mode 100644 ext/opcache/tests/jit/assign_016.phpt create mode 100644 ext/opcache/tests/jit/assign_017.phpt create mode 100644 ext/opcache/tests/jit/assign_018.phpt create mode 100644 ext/opcache/tests/jit/assign_019.phpt create mode 100644 ext/opcache/tests/jit/assign_020.phpt create mode 100644 ext/opcache/tests/jit/assign_021.phpt create mode 100644 ext/opcache/tests/jit/assign_022.phpt create mode 100644 ext/opcache/tests/jit/assign_023.phpt create mode 100644 ext/opcache/tests/jit/assign_024.phpt create mode 100644 ext/opcache/tests/jit/assign_025.phpt create mode 100644 ext/opcache/tests/jit/assign_026.phpt create mode 100644 ext/opcache/tests/jit/assign_027.phpt create mode 100644 ext/opcache/tests/jit/assign_028.phpt create mode 100644 ext/opcache/tests/jit/assign_029.phpt create mode 100644 ext/opcache/tests/jit/assign_030.phpt create mode 100644 ext/opcache/tests/jit/assign_031.phpt create mode 100644 ext/opcache/tests/jit/assign_032.phpt create mode 100644 ext/opcache/tests/jit/assign_033.phpt create mode 100644 ext/opcache/tests/jit/assign_034.phpt create mode 100644 ext/opcache/tests/jit/assign_dim_002.phpt create mode 100644 ext/opcache/tests/jit/fetch_dim_r_001.phpt create mode 100644 ext/opcache/tests/jit/fetch_dim_r_002.phpt create mode 100644 ext/opcache/tests/jit/fetch_dim_r_003.phpt create mode 100644 ext/opcache/tests/jit/fetch_dim_r_004.phpt create mode 100644 ext/opcache/tests/jit/fetch_dim_rw_001.phpt create mode 100644 ext/opcache/tests/jit/fetch_obj_001.phpt create mode 100644 ext/opcache/tests/jit/fetch_obj_002.phpt create mode 100644 ext/opcache/tests/jit/fetch_obj_003.phpt create mode 100644 ext/opcache/tests/jit/fetch_obj_004.phpt create mode 100644 ext/opcache/tests/jit/inc_001.phpt create mode 100644 ext/opcache/tests/jit/inc_002.phpt create mode 100644 ext/opcache/tests/jit/inc_003.phpt create mode 100644 ext/opcache/tests/jit/inc_004.phpt create mode 100644 ext/opcache/tests/jit/inc_005.phpt create mode 100644 ext/opcache/tests/jit/inc_006.phpt create mode 100644 ext/opcache/tests/jit/inc_007.phpt create mode 100644 ext/opcache/tests/jit/inc_008.phpt create mode 100644 ext/opcache/tests/jit/inc_009.phpt create mode 100644 ext/opcache/tests/jit/inc_010.phpt create mode 100644 ext/opcache/tests/jit/inc_011.phpt create mode 100644 ext/opcache/tests/jit/inc_012.phpt create mode 100644 ext/opcache/tests/jit/inc_013.phpt create mode 100644 ext/opcache/tests/jit/inc_014.phpt create mode 100644 ext/opcache/tests/jit/inc_015.phpt create mode 100644 ext/opcache/tests/jit/inc_016.phpt create mode 100644 ext/opcache/tests/jit/inc_017.phpt create mode 100644 ext/opcache/tests/jit/inc_018.phpt create mode 100644 ext/opcache/tests/jit/inc_019.phpt create mode 100644 ext/opcache/tests/jit/noval_001.phpt diff --git a/ext/opcache/tests/jit/assign_001.phpt b/ext/opcache/tests/jit/assign_001.phpt new file mode 100644 index 0000000000000..8a17a4920f1fe --- /dev/null +++ b/ext/opcache/tests/jit/assign_001.phpt @@ -0,0 +1,27 @@ +--TEST-- +JIT ASSIGN: 001 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +;opcache.jit_debug=257 +--SKIPIF-- + +--FILE-- + +--FILE-- + +--FILE-- + +--FILE-- + +--FILE-- + +--FILE-- + +--FILE-- + +--FILE-- + +--FILE-- + +--FILE-- + +--FILE-- + +--FILE-- + +--FILE-- + +--FILE-- + +--FILE-- + +--FILE-- + +--FILE-- + +--FILE-- + +--FILE-- + +--FILE-- + +--FILE-- + +--FILE-- + +--FILE-- + +--FILE-- + +--FILE-- + +--FILE-- +a=1; + $a->b=2; + $b=&$a; +} +foo(); +echo "ok\n"; +--EXPECT-- +ok diff --git a/ext/opcache/tests/jit/assign_027.phpt b/ext/opcache/tests/jit/assign_027.phpt new file mode 100644 index 0000000000000..d9b313cde68f1 --- /dev/null +++ b/ext/opcache/tests/jit/assign_027.phpt @@ -0,0 +1,22 @@ +--TEST-- +JIT ASSIGN: 027 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +;opcache.jit_debug=257 +--SKIPIF-- + +--FILE-- + +--FILE-- + +--FILE-- + +--FILE-- + +--FILE-- + +--EXPECT-- +ok diff --git a/ext/opcache/tests/jit/assign_032.phpt b/ext/opcache/tests/jit/assign_032.phpt new file mode 100644 index 0000000000000..e7a8710efd351 --- /dev/null +++ b/ext/opcache/tests/jit/assign_032.phpt @@ -0,0 +1,27 @@ +--TEST-- +JIT ASSIGN: 032 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +opcache.protect_memory=1 +;opcache.jit_debug=257 +--SKIPIF-- + +--FILE-- + +--EXPECT-- +string(5) "space" +string(5) "space" +string(5) "space" +Done diff --git a/ext/opcache/tests/jit/assign_033.phpt b/ext/opcache/tests/jit/assign_033.phpt new file mode 100644 index 0000000000000..5dbee390154f6 --- /dev/null +++ b/ext/opcache/tests/jit/assign_033.phpt @@ -0,0 +1,24 @@ +--TEST-- +JIT ASSIGN: 033 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +opcache.protect_memory=1 +;opcache.jit_debug=257 +--SKIPIF-- + +--FILE-- + +--FILE-- + +--FILE-- + "dummy"); + $array[] = array(); + + $array = new ArrayObject(); + $array[index()] = 1; + $array[offset()] = 2; + + var_dump($array); +} +foo3(); + +function index() { + return 2; +} + +function offset() { + return "a"; +} + +function foo4() { + $array = array(); + $array[] = array(); + $array[0][] = 1; + $array[0][1] = 1; + var_dump($array); + + $array[function() {}] = 2; + var_dump($array); + + $array2[][] = 3; + var_dump($array); +} +foo4(); + +function foo5() { + $a = 1; + $a[2] = 1; + return $a; +} +var_dump(foo5()); + +--EXPECTF-- +array(1) { + [0]=> + int(1) +} +array(1) { + [0]=> + bool(true) +} +array(1) { + [0]=> + array(0) { + } +} + +Warning: Cannot add element to the array as the next element is already occupied in %sassign_dim_002.php on line 22 +object(ArrayObject)#1 (1) { + ["storage":"ArrayObject":private]=> + array(2) { + [2]=> + int(1) + ["a"]=> + int(2) + } +} +array(1) { + [0]=> + array(2) { + [0]=> + int(1) + [1]=> + int(1) + } +} + +Warning: Illegal offset type in %sassign_dim_002.php on line 47 +array(1) { + [0]=> + array(2) { + [0]=> + int(1) + [1]=> + int(1) + } +} +array(1) { + [0]=> + array(2) { + [0]=> + int(1) + [1]=> + int(1) + } +} + +Warning: Cannot use a scalar value as an array in %sassign_dim_002.php on line 57 +int(1) diff --git a/ext/opcache/tests/jit/fetch_dim_r_001.phpt b/ext/opcache/tests/jit/fetch_dim_r_001.phpt new file mode 100644 index 0000000000000..0aaef02d9d5a2 --- /dev/null +++ b/ext/opcache/tests/jit/fetch_dim_r_001.phpt @@ -0,0 +1,45 @@ +--TEST-- +JIT FETCH_DIM_R: 001 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +;opcache.jit_debug=257 +--SKIPIF-- + +--FILE-- +4,"ab"=>5,"2x"=>6); + var_dump($a[0]); + var_dump($a[2]); + var_dump($a[1.0]); + var_dump($a["0"]); + var_dump($a["2"]); + var_dump($a[false]); + var_dump($a[true]); + var_dump($a[null]); + var_dump($a["ab"]); + $x = "a"; + $y = "b"; + var_dump($a[$x . $y]); + var_dump($a["2x"]); + $x = "2"; + $y = "x"; + var_dump($a[$x . $y]); +} +foo(); +--EXPECT-- +int(1) +int(3) +int(2) +int(1) +int(3) +int(1) +int(2) +int(4) +int(5) +int(5) +int(6) +int(6) diff --git a/ext/opcache/tests/jit/fetch_dim_r_002.phpt b/ext/opcache/tests/jit/fetch_dim_r_002.phpt new file mode 100644 index 0000000000000..be7f3b0ae604b --- /dev/null +++ b/ext/opcache/tests/jit/fetch_dim_r_002.phpt @@ -0,0 +1,45 @@ +--TEST-- +JIT FETCH_DIM_R: 002 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +;opcache.jit_debug=257 +--SKIPIF-- + +--FILE-- +4,"ab"=>5,"2x"=>6); + var_dump($a[$n]); +} +foo(0); +foo(2); +foo(1.0); +foo("0"); +foo("2"); +foo(false); +foo(true); +foo(null); +foo("ab"); +$x="a"; +$y="b"; +foo($x.$y); +foo("2x"); +$x=2; +$y="x"; +foo($x.$y); +--EXPECT-- +int(1) +int(3) +int(2) +int(1) +int(3) +int(1) +int(2) +int(4) +int(5) +int(5) +int(6) +int(6) diff --git a/ext/opcache/tests/jit/fetch_dim_r_003.phpt b/ext/opcache/tests/jit/fetch_dim_r_003.phpt new file mode 100644 index 0000000000000..ec26f196dc8c0 --- /dev/null +++ b/ext/opcache/tests/jit/fetch_dim_r_003.phpt @@ -0,0 +1,61 @@ +--TEST-- +JIT FETCH_DIM_R: 003 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +;opcache.jit_debug=257 +--SKIPIF-- + +--FILE-- + +--FILE-- + +--FILE-- + + int(2) +} diff --git a/ext/opcache/tests/jit/fetch_obj_001.phpt b/ext/opcache/tests/jit/fetch_obj_001.phpt new file mode 100644 index 0000000000000..c02ee07fedbcf --- /dev/null +++ b/ext/opcache/tests/jit/fetch_obj_001.phpt @@ -0,0 +1,128 @@ +--TEST-- +JIT: FETCH_OBJ +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +--SKIPIF-- + +--FILE-- +a); +var_dump($obj); +foo2($obj->b); +var_dump($obj); +foo3($obj->a, "2" . "3"); +foo3($obj->a, $obj->b); +var_dump($obj); + +$a = &$obj->a; +$a = fopen(__FILE__, "r"); +var_dump($obj); + +function bar() { + foo($obj->a); + var_dump($obj); + foo2($obj->b); + var_dump($obj); + foo3($obj->a, "2" . "3"); + foo3($obj->a, $obj->b); + var_dump($obj); + + $a = &$obj->a; + $a = fopen(__FILE__, "r"); + var_dump($obj); + + $d = array(); + foo($d->{"ab" ."c"}); + var_dump($d); + + $e = NULL; + foo($e->{"ab" ."c"}); + var_dump($e); + + $f = ""; + foo($f->{"ab" ."c"}); + var_dump($f); +} + +bar(); +?> +--EXPECTF-- +object(stdClass)#%d (1) { + ["a"]=> + int(2) +} +object(stdClass)#%d (2) { + ["a"]=> + int(2) + ["b"]=> + array(0) { + } +} +object(stdClass)#%d (2) { + ["a"]=> + array(0) { + } + ["b"]=> + array(0) { + } +} +object(stdClass)#%d (2) { + ["a"]=> + &resource(5) of type (stream) + ["b"]=> + array(0) { + } +} +object(stdClass)#%d (1) { + ["a"]=> + int(2) +} +object(stdClass)#%d (2) { + ["a"]=> + int(2) + ["b"]=> + array(0) { + } +} +object(stdClass)#%d (2) { + ["a"]=> + array(0) { + } + ["b"]=> + array(0) { + } +} +object(stdClass)#%d (2) { + ["a"]=> + &resource(6) of type (stream) + ["b"]=> + array(0) { + } +} + +Warning: Attempt to modify property of non-object in %sfetch_obj_001.php on line 40 +array(0) { +} +object(stdClass)#%d (1) { + ["abc"]=> + int(2) +} +object(stdClass)#%d (1) { + ["abc"]=> + int(2) +} diff --git a/ext/opcache/tests/jit/fetch_obj_002.phpt b/ext/opcache/tests/jit/fetch_obj_002.phpt new file mode 100644 index 0000000000000..b98ef373c8d8c --- /dev/null +++ b/ext/opcache/tests/jit/fetch_obj_002.phpt @@ -0,0 +1,41 @@ +--TEST-- +JIT: FETCH_OBJ 002 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +--SKIPIF-- + +--FILE-- +x); + var_dump($a->y); + $b = new B(); + var_dump($b->x); + unset($b->x); + $b->x; +} + +bar(); +?> +--EXPECTF-- +int(2) + +Notice: Undefined property: A::$y in %sfetch_obj_002.php on line 16 +NULL +int(3) +string(5) "__get" diff --git a/ext/opcache/tests/jit/fetch_obj_003.phpt b/ext/opcache/tests/jit/fetch_obj_003.phpt new file mode 100644 index 0000000000000..e545e671bb7be --- /dev/null +++ b/ext/opcache/tests/jit/fetch_obj_003.phpt @@ -0,0 +1,46 @@ +--TEST-- +JIT: FETCH_OBJ 003 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +--SKIPIF-- + +--FILE-- +a = 1; + unset($x->a); + $x->a += 2; + var_dump($x); +} +function bar() { + $x = new C; + $x->a = 1; + $x->b = 2; + unset($x->a); + $x->a += 2; + var_dump($x); +} +foo(); +bar(); +?> +--EXPECTF-- +Notice: Undefined property: C::$a in %sfetch_obj_003.php on line 9 +object(C)#1 (1) { + ["a"]=> + int(2) +} + +Notice: Undefined property: C::$a in %sfetch_obj_003.php on line 17 +object(C)#1 (2) { + ["a"]=> + int(2) + ["b"]=> + int(2) +} diff --git a/ext/opcache/tests/jit/fetch_obj_004.phpt b/ext/opcache/tests/jit/fetch_obj_004.phpt new file mode 100644 index 0000000000000..a7bee66c01a8a --- /dev/null +++ b/ext/opcache/tests/jit/fetch_obj_004.phpt @@ -0,0 +1,43 @@ +--TEST-- +JIT: FETCH_OBJ 004 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +--SKIPIF-- + +--FILE-- +a = 1; + unset($x->a); + $x->a = 3; + var_dump($x); +} +function bar() { + $x = new C; + $x->a = 1; + $x->b = 2; + unset($x->a); + $x->a = 3; + var_dump($x); +} +foo(); +bar(); +?> +--EXPECT-- +object(C)#1 (1) { + ["a"]=> + int(3) +} +object(C)#1 (2) { + ["a"]=> + int(3) + ["b"]=> + int(2) +} diff --git a/ext/opcache/tests/jit/inc_001.phpt b/ext/opcache/tests/jit/inc_001.phpt new file mode 100644 index 0000000000000..7c66a677625fa --- /dev/null +++ b/ext/opcache/tests/jit/inc_001.phpt @@ -0,0 +1,22 @@ +--TEST-- +JIT INC: 001 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +opcache.protect_memory=1 +;opcache.jit_debug=257 +--SKIPIF-- + +--FILE-- + mem + var_dump($x); +} +foo(); +--EXPECT-- +int(2) diff --git a/ext/opcache/tests/jit/inc_002.phpt b/ext/opcache/tests/jit/inc_002.phpt new file mode 100644 index 0000000000000..37e715528b404 --- /dev/null +++ b/ext/opcache/tests/jit/inc_002.phpt @@ -0,0 +1,21 @@ +--TEST-- +JIT INC: 002 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +opcache.protect_memory=1 +;opcache.jit_debug=257 +--SKIPIF-- + +--FILE-- + mem + var_dump($x); +} +foo(); +--EXPECT-- +int(2) diff --git a/ext/opcache/tests/jit/inc_003.phpt b/ext/opcache/tests/jit/inc_003.phpt new file mode 100644 index 0000000000000..8d45cba4f6393 --- /dev/null +++ b/ext/opcache/tests/jit/inc_003.phpt @@ -0,0 +1,22 @@ +--TEST-- +JIT INC: 003 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +opcache.protect_memory=1 +;opcache.jit_debug=257 +--SKIPIF-- + +--FILE-- + reg + return $x; +} +var_dump(foo()); +--EXPECT-- +int(2) diff --git a/ext/opcache/tests/jit/inc_004.phpt b/ext/opcache/tests/jit/inc_004.phpt new file mode 100644 index 0000000000000..1d7deabde971d --- /dev/null +++ b/ext/opcache/tests/jit/inc_004.phpt @@ -0,0 +1,21 @@ +--TEST-- +JIT INC: 004 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +opcache.protect_memory=1 +;opcache.jit_debug=257 +--SKIPIF-- + +--FILE-- + reg + return $x; +} +var_dump(foo()); +--EXPECT-- +int(2) diff --git a/ext/opcache/tests/jit/inc_005.phpt b/ext/opcache/tests/jit/inc_005.phpt new file mode 100644 index 0000000000000..a1fc59b82b5bf --- /dev/null +++ b/ext/opcache/tests/jit/inc_005.phpt @@ -0,0 +1,23 @@ +--TEST-- +JIT INC: 005 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +opcache.protect_memory=1 +;opcache.jit_debug=257 +--SKIPIF-- + +--FILE-- + mem, mem + var_dump($x); +} +foo(); +--EXPECT-- +int(2) +int(2) diff --git a/ext/opcache/tests/jit/inc_006.phpt b/ext/opcache/tests/jit/inc_006.phpt new file mode 100644 index 0000000000000..cecb58809b7c3 --- /dev/null +++ b/ext/opcache/tests/jit/inc_006.phpt @@ -0,0 +1,22 @@ +--TEST-- +JIT INC: 006 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +opcache.protect_memory=1 +;opcache.jit_debug=257 +--SKIPIF-- + +--FILE-- + mem, mem + var_dump($x); +} +foo(); +--EXPECT-- +int(2) +int(2) diff --git a/ext/opcache/tests/jit/inc_007.phpt b/ext/opcache/tests/jit/inc_007.phpt new file mode 100644 index 0000000000000..d4a666ebb0da7 --- /dev/null +++ b/ext/opcache/tests/jit/inc_007.phpt @@ -0,0 +1,21 @@ +--TEST-- +JIT INC: 007 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +opcache.protect_memory=1 +;opcache.jit_debug=257 +--SKIPIF-- + +--FILE-- + reg, reg +} +var_dump(foo()); +--EXPECT-- +int(2) diff --git a/ext/opcache/tests/jit/inc_008.phpt b/ext/opcache/tests/jit/inc_008.phpt new file mode 100644 index 0000000000000..b9cb6a173ae8c --- /dev/null +++ b/ext/opcache/tests/jit/inc_008.phpt @@ -0,0 +1,20 @@ +--TEST-- +JIT INC: 008 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +opcache.protect_memory=1 +;opcache.jit_debug=257 +--SKIPIF-- + +--FILE-- + reg, reg +} +var_dump(foo()); +--EXPECT-- +int(2) diff --git a/ext/opcache/tests/jit/inc_009.phpt b/ext/opcache/tests/jit/inc_009.phpt new file mode 100644 index 0000000000000..b587be1f5001f --- /dev/null +++ b/ext/opcache/tests/jit/inc_009.phpt @@ -0,0 +1,22 @@ +--TEST-- +JIT INC: 009 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +opcache.protect_memory=1 +;opcache.jit_debug=257 +--SKIPIF-- + +--FILE-- + mem + var_dump($x); +} +foo(); +--EXPECT-- +float(2) diff --git a/ext/opcache/tests/jit/inc_010.phpt b/ext/opcache/tests/jit/inc_010.phpt new file mode 100644 index 0000000000000..d1aa19069b494 --- /dev/null +++ b/ext/opcache/tests/jit/inc_010.phpt @@ -0,0 +1,21 @@ +--TEST-- +JIT INC: 010 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +opcache.protect_memory=1 +;opcache.jit_debug=257 +--SKIPIF-- + +--FILE-- + mem + var_dump($x); +} +foo(); +--EXPECT-- +float(2) diff --git a/ext/opcache/tests/jit/inc_011.phpt b/ext/opcache/tests/jit/inc_011.phpt new file mode 100644 index 0000000000000..62d3b3c7c4b15 --- /dev/null +++ b/ext/opcache/tests/jit/inc_011.phpt @@ -0,0 +1,22 @@ +--TEST-- +JIT INC: 011 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +opcache.protect_memory=1 +;opcache.jit_debug=257 +--SKIPIF-- + +--FILE-- + reg + return $x; +} +var_dump(foo()); +--EXPECT-- +float(2) diff --git a/ext/opcache/tests/jit/inc_012.phpt b/ext/opcache/tests/jit/inc_012.phpt new file mode 100644 index 0000000000000..c6f49553d2149 --- /dev/null +++ b/ext/opcache/tests/jit/inc_012.phpt @@ -0,0 +1,21 @@ +--TEST-- +JIT INC: 012 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +opcache.protect_memory=1 +;opcache.jit_debug=257 +--SKIPIF-- + +--FILE-- + reg + return $x; +} +var_dump(foo()); +--EXPECT-- +float(2) diff --git a/ext/opcache/tests/jit/inc_013.phpt b/ext/opcache/tests/jit/inc_013.phpt new file mode 100644 index 0000000000000..f7eb69ee39808 --- /dev/null +++ b/ext/opcache/tests/jit/inc_013.phpt @@ -0,0 +1,23 @@ +--TEST-- +JIT INC: 013 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +opcache.protect_memory=1 +;opcache.jit_debug=257 +--SKIPIF-- + +--FILE-- + mem, mem + var_dump($x); +} +foo(); +--EXPECT-- +float(2) +float(2) diff --git a/ext/opcache/tests/jit/inc_014.phpt b/ext/opcache/tests/jit/inc_014.phpt new file mode 100644 index 0000000000000..434ab3c1516ce --- /dev/null +++ b/ext/opcache/tests/jit/inc_014.phpt @@ -0,0 +1,22 @@ +--TEST-- +JIT INC: 014 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +opcache.protect_memory=1 +;opcache.jit_debug=257 +--SKIPIF-- + +--FILE-- + mem, mem + var_dump($x); +} +foo(); +--EXPECT-- +float(2) +float(2) diff --git a/ext/opcache/tests/jit/inc_015.phpt b/ext/opcache/tests/jit/inc_015.phpt new file mode 100644 index 0000000000000..9e53dfe2a35ac --- /dev/null +++ b/ext/opcache/tests/jit/inc_015.phpt @@ -0,0 +1,21 @@ +--TEST-- +JIT INC: 015 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +opcache.protect_memory=1 +;opcache.jit_debug=257 +--SKIPIF-- + +--FILE-- + reg, reg +} +var_dump(foo()); +--EXPECT-- +float(2) diff --git a/ext/opcache/tests/jit/inc_016.phpt b/ext/opcache/tests/jit/inc_016.phpt new file mode 100644 index 0000000000000..62c2c484783f0 --- /dev/null +++ b/ext/opcache/tests/jit/inc_016.phpt @@ -0,0 +1,20 @@ +--TEST-- +JIT INC: 016 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +opcache.protect_memory=1 +;opcache.jit_debug=257 +--SKIPIF-- + +--FILE-- + reg, reg +} +var_dump(foo()); +--EXPECT-- +float(2) diff --git a/ext/opcache/tests/jit/inc_017.phpt b/ext/opcache/tests/jit/inc_017.phpt new file mode 100644 index 0000000000000..5dc7d1ebe5c37 --- /dev/null +++ b/ext/opcache/tests/jit/inc_017.phpt @@ -0,0 +1,20 @@ +--TEST-- +JIT INC: 017 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +opcache.protect_memory=1 +;opcache.jit_debug=257 +--SKIPIF-- + +--FILE-- + reg, reg +} +var_dump(foo()); +--EXPECT-- +bool(true) diff --git a/ext/opcache/tests/jit/inc_018.phpt b/ext/opcache/tests/jit/inc_018.phpt new file mode 100644 index 0000000000000..217f1308b1e52 --- /dev/null +++ b/ext/opcache/tests/jit/inc_018.phpt @@ -0,0 +1,20 @@ +--TEST-- +JIT INC: 018 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +opcache.protect_memory=1 +;opcache.jit_debug=257 +--SKIPIF-- + +--FILE-- + reg, reg +} +var_dump(foo()); +--EXPECT-- +bool(false) diff --git a/ext/opcache/tests/jit/inc_019.phpt b/ext/opcache/tests/jit/inc_019.phpt new file mode 100644 index 0000000000000..806e86af20a35 --- /dev/null +++ b/ext/opcache/tests/jit/inc_019.phpt @@ -0,0 +1,27 @@ +--TEST-- +JIT INC: 018 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +opcache.protect_memory=1 +;opcache.jit_debug=257 +--SKIPIF-- + +--FILE-- + +--FILE-- + Date: Wed, 19 Apr 2017 16:28:30 +0300 Subject: [PATCH 447/569] Fixed MOD for 64-bit --- ext/opcache/jit/zend_jit_x86.dasc | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index aca61b3ea8f57..81d1d53e84416 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -3250,7 +3250,11 @@ static int zend_jit_long_math_helper(dasm_State **Dst, result_reg = ZREG_RDX; | GET_ZVAL_LVAL ZREG_RAX, op1_addr | GET_ZVAL_LVAL ZREG_RCX, op2_addr + |.if X64 + | cqo + |.else | cdq + |.endif | idiv Ra(ZREG_RCX) } } else { @@ -3270,7 +3274,11 @@ static int zend_jit_long_math_helper(dasm_State **Dst, result_reg = ZREG_RDX; | GET_ZVAL_LVAL ZREG_RAX, op1_addr + |.if X64 + | cqo + |.else | cdq + |.endif if (Z_MODE(op2_addr) == IS_MEM_ZVAL) { | idiv aword [Ra(Z_REG(op2_addr))+Z_OFFSET(op2_addr)] } else if (Z_MODE(op2_addr) == IS_REG) { From d4cf1afdb479b33ce3811f6ad1a8b71b193c5682 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 20 Apr 2017 11:57:27 +0300 Subject: [PATCH 448/569] JIT for ECHO CONST --- ext/opcache/jit/zend_jit.c | 5 ++++ ext/opcache/jit/zend_jit_x86.dasc | 42 +++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index f46fc9c6d60be..339596ffb74da 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -2604,6 +2604,11 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa, const zend_op *rt_op goto jit_failure; } goto done; + case ZEND_ECHO: + if (!zend_jit_echo(&dasm_state, opline, op_array, ssa)) { + goto jit_failure; + } + goto done; default: break; } diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 81d1d53e84416..beca13f21f767 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -8990,6 +8990,48 @@ static int zend_jit_free(dasm_State **Dst, const zend_op *opline, zend_op_array return 1; } +static int zend_jit_echo(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) +{ + if (opline->op1_type == IS_CONST) { + zval *zv = RT_CONSTANT(op_array, opline->op1); + + if (Z_TYPE_P(zv) == IS_STRING) { + size_t len = Z_STRLEN_P(zv); + + if (len > 0) { + const char *str = Z_STRVAL_P(zv); + + if (!zend_jit_set_valid_ip(Dst, opline)) { + return 0; + } + | //SAVE_OPLINE(); + | mov EX->opline, IP + |.if X64 + | LOAD_ADDR CARG1, str + | LOAD_ADDR CARG2, len + | EXT_CALL zend_write, r0 + |.else + | sub r4, 8 + | push len + | push str + | EXT_CALL zend_write, r0 + | add r4, 16 + |.endif + if (!zend_jit_check_exception(Dst)) { + return 0; + } + } + return 1; + } + } + + if (!zend_jit_handler(Dst, opline, 1)) { + return 0; + } + + return 1; +} + static zend_bool zend_jit_may_reuse_reg(zend_op_array *op_array, zend_ssa *ssa, uint32_t position, int def_var, int use_var) { if (ssa->var_info[def_var].type != ssa->var_info[use_var].type) { From 82646ea8113c9f15ed0e189c49c64af5ad9d9c92 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 20 Apr 2017 14:24:05 +0300 Subject: [PATCH 449/569] Made global register-allocation compatible with run-time JIT triggered by hot counters (load necessary CPU registers on trnasition for VM to JIT). --- ext/opcache/jit/zend_jit.c | 67 +++++++++++++++++++++++++------------- 1 file changed, 45 insertions(+), 22 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 339596ffb74da..6748cb9cb0054 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -186,11 +186,12 @@ static zend_string *zend_jit_func_name(zend_op_array *op_array) } } -static void *dasm_link_and_encode(dasm_State **dasm_state, - zend_op_array *op_array, - zend_cfg *cfg, - const zend_op *rt_opline, - const char *name) +static void *dasm_link_and_encode(dasm_State **dasm_state, + zend_op_array *op_array, + zend_ssa *ssa, + const zend_op *rt_opline, + zend_lifetime_interval **ra, + const char *name) { size_t size; int ret; @@ -199,19 +200,41 @@ static void *dasm_link_and_encode(dasm_State **dasm_state, zend_string *str = NULL; #endif - if (rt_opline && cfg && cfg->map) { + if (rt_opline && ssa && ssa->cfg.map) { /* Create additional entry point, to switch from interpreter to JIT-ed * code at run-time. */ - int b = cfg->map[rt_opline - op_array->opcodes]; + int b = ssa->cfg.map[rt_opline - op_array->opcodes]; #ifdef CONTEXT_THREADED_JIT - if (!(cfg->blocks[b].flags & (ZEND_BB_START|ZEND_BB_RECV_ENTRY))) { + if (!(ssa->cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_RECV_ENTRY))) { #else - if (!(cfg->blocks[b].flags & (ZEND_BB_START|ZEND_BB_ENTRY|ZEND_BB_RECV_ENTRY))) { + if (!(ssa->cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_ENTRY|ZEND_BB_RECV_ENTRY))) { #endif - zend_jit_label(dasm_state, cfg->blocks_count + b); + zend_jit_label(dasm_state, ssa->cfg.blocks_count + b); zend_jit_prologue(dasm_state); + if (ra) { + int i; + zend_lifetime_interval *ival; + zend_life_range *range; + uint32_t pos = rt_opline - op_array->opcodes; + + for (i = 0; i < ssa->vars_count; i++) { + ival = ra[i]; + + if (ival && ival->reg != ZREG_NONE) { + range = &ival->range; + + if (pos >= range->start && pos <= range->end) { + if (!zend_jit_load_ssa_var(dasm_state, ssa, i, ival->reg)) { + return NULL; + } + break; + } + range = range->next; + } + } + } zend_jit_jmp(dasm_state, b); } } @@ -237,26 +260,26 @@ static void *dasm_link_and_encode(dasm_State **dasm_state, entry = *dasm_ptr; *dasm_ptr = (void*)((char*)*dasm_ptr + ZEND_MM_ALIGNED_SIZE_EX(size, DASM_ALIGNMENT)); - if (op_array && cfg) { + if (op_array && ssa) { int b; - for (b = 0; b < cfg->blocks_count; b++) { + for (b = 0; b < ssa->cfg.blocks_count; b++) { #ifdef CONTEXT_THREADED_JIT - if (cfg->blocks[b].flags & (ZEND_BB_START|ZEND_BB_RECV_ENTRY)) { + if (ssa->cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_RECV_ENTRY)) { #else - if (cfg->blocks[b].flags & (ZEND_BB_START|ZEND_BB_ENTRY|ZEND_BB_RECV_ENTRY)) { + if (ssa->cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_ENTRY|ZEND_BB_RECV_ENTRY)) { #endif - zend_op *opline = op_array->opcodes + cfg->blocks[b].start; + zend_op *opline = op_array->opcodes + ssa->cfg.blocks[b].start; opline->handler = (void*)(((char*)entry) + - dasm_getpclabel(dasm_state, cfg->blocks_count + b)); + dasm_getpclabel(dasm_state, ssa->cfg.blocks_count + b)); } } - if (rt_opline && cfg && cfg->map) { - int b = cfg->map[rt_opline - op_array->opcodes]; + if (rt_opline && ssa && ssa->cfg.map) { + int b = ssa->cfg.map[rt_opline - op_array->opcodes]; zend_op *opline = (zend_op*)rt_opline; opline->handler = (void*)(((char*)entry) + - dasm_getpclabel(dasm_state, cfg->blocks_count + b)); + dasm_getpclabel(dasm_state, ssa->cfg.blocks_count + b)); } } @@ -278,7 +301,7 @@ static void *dasm_link_and_encode(dasm_State **dasm_state, name, (op_array && op_array->filename) ? ZSTR_VAL(op_array->filename) : NULL, op_array, - cfg, + &ssa->cfg, entry, size); } @@ -2759,7 +2782,7 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa, const zend_op *rt_op } } - handler = dasm_link_and_encode(&dasm_state, op_array, &ssa->cfg, rt_opline, NULL); + handler = dasm_link_and_encode(&dasm_state, op_array, ssa, rt_opline, ra, NULL); if (!handler) { goto jit_failure; } @@ -3196,7 +3219,7 @@ static int zend_jit_make_stubs(void) if (!zend_jit_stubs[i].stub(&dasm_state)) { return 0; } - if (!dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, zend_jit_stubs[i].name)) { + if (!dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, zend_jit_stubs[i].name)) { return 0; } } From d2c6ab409eb4f145693908726d9dbf34e3f2b131 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 21 Apr 2017 12:00:04 +0300 Subject: [PATCH 450/569] Fixed overflow check --- ext/opcache/jit/zend_jit_x86.dasc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index beca13f21f767..c0a83834f0fcf 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -4453,6 +4453,7 @@ static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, zend_ if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { uint32_t var_info = zend_array_element_type(op1_info, 0, 0); + uint32_t var_def_info = zend_array_element_type(OP1_DEF_INFO(), 1, 0); |6: if (opline->op2_type == IS_UNUSED) { @@ -4498,7 +4499,7 @@ static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, zend_ case ZEND_ASSIGN_SUB: case ZEND_ASSIGN_MUL: case ZEND_ASSIGN_DIV: - if (!zend_jit_math_helper(Dst, opline, op_array, ssa, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, op3_addr, OP1_DATA_INFO(), 0, var_addr, OP1_DEF_INFO(), var_info, 1)) { + if (!zend_jit_math_helper(Dst, opline, op_array, ssa, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, op3_addr, OP1_DATA_INFO(), 0, var_addr, var_def_info, var_info, 1)) { return 0; } break; From 55153eab4a5c5a43a3d140b365cecf50077b20ef Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 24 Apr 2017 13:08:23 +0300 Subject: [PATCH 451/569] Moved cold code out of the fast path --- ext/opcache/jit/zend_jit_x86.dasc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index c0a83834f0fcf..26f775d56a79b 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -8288,11 +8288,14 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, const zend_op *opline, zend | GET_ZVAL_PTR r1, val_addr | IF_NOT_ZVAL_TYPE val_addr, IS_REFERENCE, >1 | cmp dword [r1], 1 - | jne >1 + | je >3 + |.cold_code + |3: | LOAD_ZVAL_ADDR FCARG1a, res_addr | LOAD_ZVAL_ADDR FCARG2a, val_addr | EXT_CALL zend_jit_zval_copy_unref_helper, r0 | jmp >9 + |.code |1: | GC_ADDREF r1 |2: From d570de2cb6a88d9772f510d112a04ce4022110cd Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 9 May 2017 10:15:20 +0300 Subject: [PATCH 452/569] JIT support for HYBRID VM --- ext/opcache/jit/zend_jit_vm_helpers.c | 5 +++ ext/opcache/jit/zend_jit_x86.dasc | 62 +++++++++++++++++++++++++-- 2 files changed, 64 insertions(+), 3 deletions(-) diff --git a/ext/opcache/jit/zend_jit_vm_helpers.c b/ext/opcache/jit/zend_jit_vm_helpers.c index 649f300ca76e5..e0b338f972912 100644 --- a/ext/opcache/jit/zend_jit_vm_helpers.c +++ b/ext/opcache/jit/zend_jit_vm_helpers.c @@ -19,6 +19,7 @@ #include "Zend/zend_execute.h" #include "Zend/zend_exceptions.h" +#include "Zend/zend_vm.h" #include #include "Optimizer/zend_func_info.h" @@ -84,7 +85,11 @@ void ZEND_FASTCALL zend_jit_leave_top_func_helper(uint32_t call_info) OBJ_RELEASE((zend_object*)EX(func)->op_array.prototype); } execute_data = EG(current_execute_data); +#if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID) + opline = zend_get_real_exit_op(); +#else opline = NULL; +#endif } void ZEND_FASTCALL zend_jit_copy_extra_args_helper(void) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 26f775d56a79b..a9952aca6cc1a 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -61,6 +61,8 @@ |.define SSE, 1 |.endif +|.define HYBRID_SPAD, 16 + #include "jit/zend_jit_x86.h" #include "jit/zend_cpuid.h" @@ -1578,7 +1580,11 @@ static int zend_jit_interrupt_handler_stub(dasm_State **Dst) | //} } | //ZEND_VM_CONTINUE() +#if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID) + | add r4, HYBRID_SPAD +#else | add r4, SPAD // stack alignment +#endif | jmp aword [IP] return 1; @@ -1586,9 +1592,19 @@ static int zend_jit_interrupt_handler_stub(dasm_State **Dst) static int zend_jit_exception_handler_stub(dasm_State **Dst) { +//#ifdef ZEND_VM_KIND_HYBRID +// const void *handler = zend_get_real_opcode_handler(EG(exception_op)); +//#else + const void *handler = EG(exception_op)->handler; +//#endif + |->exception_handler: +#if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID) + | add r4, HYBRID_SPAD +#else | add r4, SPAD // stack alignment - | EXT_JMP EG(exception_op)->handler, r0 +#endif + | EXT_JMP handler, r0 return 1; } @@ -1605,12 +1621,24 @@ static int zend_jit_exception_handler_undef_stub(dasm_State **Dst) static int zend_jit_leave_function_stub(dasm_State **Dst) { |->leave_function_handler: +#if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID) + | test FCARG1d, ZEND_CALL_TOP + | jnz >1 + | EXT_CALL zend_jit_leave_nested_func_helper, r0 + | add r4, HYBRID_SPAD // stack alignment + | jmp aword [IP] + |1: + | EXT_CALL zend_jit_leave_top_func_helper, r0 + | add r4, HYBRID_SPAD // stack alignment + | jmp aword [IP] +#else | add r4, SPAD | test FCARG1d, ZEND_CALL_TOP | jnz >1 | EXT_JMP zend_jit_leave_nested_func_helper, r0 |1: | EXT_JMP zend_jit_leave_top_func_helper, r0 +#endif return 1; } @@ -1976,7 +2004,11 @@ static int zend_jit_align_func(dasm_State **Dst) static int zend_jit_prologue(dasm_State **Dst) { +#if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID) + | sub r4, HYBRID_SPAD +#else | sub r4, SPAD // stack alignment +#endif return 1; } @@ -2063,10 +2095,16 @@ static int zend_jit_check_exception_undef_result(dasm_State **Dst, const zend_op static int zend_jit_handler(dasm_State **Dst, const zend_op *opline, int may_throw) { +#ifdef ZEND_VM_KIND_HYBRID + const void *handler = zend_get_real_opcode_handler(opline); +#else + const void *handler = opline->handler; +#endif + if (!zend_jit_set_valid_ip(Dst, opline)) { return 0; } - | EXT_CALL opline->handler, r0 + | EXT_CALL handler, r0 if (may_throw) { zend_jit_check_exception(Dst); } @@ -2103,11 +2141,21 @@ static int zend_jit_handler(dasm_State **Dst, const zend_op *opline, int may_thr static int zend_jit_tail_handler(dasm_State **Dst, const zend_op *opline) { +//#ifdef ZEND_VM_KIND_HYBRID +// const void *handler = zend_get_real_opcode_handler(opline); +//#else + const void *handler = opline->handler; +//#endif + if (!zend_jit_set_valid_ip(Dst, opline)) { return 0; } +#if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID) + | add r4, HYBRID_SPAD +#else | add r4, SPAD // stack alignment - | EXT_JMP opline->handler, r0 +#endif + | EXT_JMP handler, r0 last_valid_opline = NULL; return 1; } @@ -6938,8 +6986,12 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar if (!func) { | jmp >9 } +#else +#if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID) + | add r4, HYBRID_SPAD #else | add r4, SPAD // stack alignment +#endif | jmp aword [IP] #endif } @@ -8012,7 +8064,11 @@ static int zend_jit_leave_func(dasm_State **Dst, const zend_op *opline, zend_op_ | jne ->leave_throw_handler | // opline = EX(opline) + 1 | add IP, sizeof(zend_op) +#if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID) + | add r4, HYBRID_SPAD +#else | add r4, SPAD // stack alignment +#endif #ifdef CONTEXT_THREADED_JIT | ret #else From 015bb16baaf9a3dae8f6143d19ea3db9069f827c Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 10 May 2017 18:01:19 +0300 Subject: [PATCH 453/569] Run-time JIT for HYBRID VM --- ext/opcache/ZendAccelerator.c | 4 +- ext/opcache/jit/zend_jit.c | 72 ++++++++++++ ext/opcache/jit/zend_jit_x86.dasc | 183 ++++++++++++++++++++++++++++++ 3 files changed, 257 insertions(+), 2 deletions(-) diff --git a/ext/opcache/ZendAccelerator.c b/ext/opcache/ZendAccelerator.c index a7a2173658bd0..702d8e0733ae6 100644 --- a/ext/opcache/ZendAccelerator.c +++ b/ext/opcache/ZendAccelerator.c @@ -2844,6 +2844,8 @@ static int accel_startup(zend_extension *extension) /* Init auto-global strings */ zend_accel_init_auto_globals(); + zend_optimizer_startup(); + zend_shared_alloc_lock(); #ifdef HAVE_JIT if (ZCG(accel_directives).jit && @@ -2918,8 +2920,6 @@ static int accel_startup(zend_extension *extension) zend_accel_blacklist_load(&accel_blacklist, ZCG(accel_directives.user_blacklist_filename)); } - zend_optimizer_startup(); - return SUCCESS; } diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 6748cb9cb0054..9eb8dc1e7bde0 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -79,8 +79,16 @@ static void *dasm_buf = NULL; static void *dasm_end = NULL; static void **dasm_ptr = NULL; +#if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID) +static const void *hybrid_runtime_jit = NULL; +static const void *hybrid_profile_jit = NULL; +static const void *hybrid_func_counter = NULL; +static const void *hybrid_loop_counter = NULL; +#endif + static int zend_may_throw(const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa); static int zend_may_overflow(const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa); +static void ZEND_FASTCALL zend_runtime_jit(void); static zend_bool zend_ssa_is_last_use(zend_op_array *op_array, const zend_ssa *ssa, int var, int use) { @@ -2926,7 +2934,11 @@ void zend_jit_check_funcs(HashTable *function_table, zend_bool is_method) { while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) { opline++; } +#if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID) + if (opline->handler == hybrid_profile_jit) { +#else if (opline->handler == zend_jit_profile_helper) { +#endif zend_ulong counter = (zend_ulong)ZEND_COUNTER_INFO(op_array); ZEND_COUNTER_INFO(op_array) = 0; opline->handler = ZEND_FUNC_INFO(op_array); @@ -2990,13 +3002,22 @@ static int zend_jit_setup_hot_counters(zend_op_array *op_array) opline++; } +#if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID) + opline->handler = (const void*)hybrid_func_counter; +#else opline->handler = (const void*)zend_jit_func_counter_helper; +#endif for (i = 0; i < cfg.blocks_count; i++) { if ((cfg.blocks[i].flags & ZEND_BB_REACHABLE) && (cfg.blocks[i].flags & ZEND_BB_LOOP_HEADER)) { +#if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID) + op_array->opcodes[cfg.blocks[i].start].handler = + (const void*)hybrid_loop_counter; +#else op_array->opcodes[cfg.blocks[i].start].handler = (const void*)zend_jit_loop_counter_helper; +#endif } } @@ -3032,11 +3053,19 @@ ZEND_API int zend_jit_op_array(zend_op_array *op_array, zend_script *script) /* Set run-time JIT handler */ while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) { +#if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID) + opline->handler = (const void*)hybrid_runtime_jit; +#else opline->handler = (const void*)zend_runtime_jit; +#endif opline++; } ZEND_SET_FUNC_INFO(op_array, (void*)opline->handler); +#if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID) + opline->handler = (const void*)hybrid_runtime_jit; +#else opline->handler = (const void*)zend_runtime_jit; +#endif return SUCCESS; } else if (zend_jit_trigger == ZEND_JIT_ON_PROF_REQUEST) { @@ -3047,7 +3076,11 @@ ZEND_API int zend_jit_op_array(zend_op_array *op_array, zend_script *script) opline++; } ZEND_SET_FUNC_INFO(op_array, (void*)opline->handler); +#if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID) + opline->handler = (const void*)hybrid_profile_jit; +#else opline->handler = zend_jit_profile_helper; +#endif } return SUCCESS; @@ -3223,6 +3256,45 @@ static int zend_jit_make_stubs(void) return 0; } } + +#if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID) + dasm_setup(&dasm_state, dasm_actions); + if (!zend_jit_hybrid_runtime_jit_stub(&dasm_state)) { + return 0; + } + hybrid_runtime_jit = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, "JIT$$hybrid_runtime_jit"); + if (!hybrid_runtime_jit) { + return 0; + } + + dasm_setup(&dasm_state, dasm_actions); + if (!zend_jit_hybrid_profile_jit_stub(&dasm_state)) { + return 0; + } + hybrid_profile_jit = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, "JIT$$hybrid_profile_jit"); + if (!hybrid_profile_jit) { + return 0; + } + + dasm_setup(&dasm_state, dasm_actions); + if (!zend_jit_hybrid_func_counter_stub(&dasm_state)) { + return 0; + } + hybrid_func_counter = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, "JIT$$hybrid_func_counter"); + if (!hybrid_profile_jit) { + return 0; + } + + dasm_setup(&dasm_state, dasm_actions); + if (!zend_jit_hybrid_loop_counter_stub(&dasm_state)) { + return 0; + } + hybrid_loop_counter = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, "JIT$$hybrid_loop_counter"); + if (!hybrid_profile_jit) { + return 0; + } +#endif + dasm_free(&dasm_state); return 1; } diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index a9952aca6cc1a..533508cb96558 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -1947,6 +1947,189 @@ static int zend_jit_double_one_stub(dasm_State **Dst) return 1; } +#if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID) +static int zend_jit_hybrid_runtime_jit_stub(dasm_State **Dst) +{ + |->hybrid_runtime_jit: + | EXT_CALL zend_runtime_jit, r0 + | jmp aword [IP] + return 1; +} + +static int zend_jit_hybrid_profile_jit_stub(dasm_State **Dst) +{ + |->hybrid_profile_jit: + | // ++zend_jit_profile_counter; + | .if X64 + | LOAD_ADDR r0, &zend_jit_profile_counter + | inc aword [r0] + | .else + | inc aword [&zend_jit_profile_counter] + | .endif + | // op_array = (zend_op_array*)EX(func); + | mov r0, EX->func + | // ++(ZEND_COUNTER_INFO(op_array)); + | inc aword [r0 + offsetof(zend_op_array, reserved[zend_jit_profile_counter_rid])] + | // handler = (const void*)ZEND_FUNC_INFO(op_array); + | mov r0, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])] + | // return ((zend_vm_opcode_handler_t)handler)(); + | jmp r0 + return 1; +} + +static int zend_jit_hybrid_func_counter_stub(dasm_State **Dst) +{ + |->hybrid_func_counter: + | mov r0, EX->func + | mov r1, aword [r0 + offsetof(zend_op_array, function_name)] + | test r1, r1 + | jne >1 + | mov r0, aword [r0 + offsetof(zend_op_array, filename)] + |1: + | shr r0, 3 + | mov r1, r0 + | .if X64 + | shr r0, 30 + | xor r0, r1 + | mov64 r1, 0xbf58476d1ce4e5b9 + | imul r0, r1 + | mov r1, r0 + | shr r0, 27 + | xor r0, r1 + | mov64 r1, 0x94d049bb133111eb + | imul r0, r1 + | mov r1, r0 + | shr r0, 31 + | xor r1, r0 + | and r1, ZEND_HOT_COUNTERS_COUNT-1 + | LOAD_ADDR r0, &zend_jit_hot_counters + | sub word [r0+r1*2], ZEND_JIT_HOT_FUNC_COST + | .else + | shr r0, 16 + | xor r0, r1 + | imul r0, 0x45d9f3b + | mov r1, r0 + | shr r0, 16 + | xor r0, r1 + | imul r0, 0x45d9f3b + | mov r1, r0 + | shr r0, 16 + | xor r1, r0 + | and r1, ZEND_HOT_COUNTERS_COUNT-1 + | sub word [r1*2+&zend_jit_hot_counters], ZEND_JIT_HOT_FUNC_COST + | .endif + | jle >1 + | mov r1, EX->func + | mov r0, IP + | sub r0, aword [r1 + offsetof(zend_op_array, opcodes)] + | // divide by sizeof(zend_op) + | .if X64 + | sar r0, 5 + | .else + | sar r0, 2 + | imul r0, 0xb6db6db7 + | .endif + | mov r1, aword [r1 + offsetof(zend_op_array, reserved[zend_func_info_rid])] + | .if X64 + | jmp aword [r1+r0*8] + | .else + | jmp aword [r1+r0*4] + | .endif + |1: + | .if X64 + | mov word [r0+r1*2], ZEND_JIT_HOT_COUNTER_INIT + | mov CARG1, FP + | mov CARG2, IP + | EXT_CALL zend_jit_hot_func, r0 + | .else + | mov word [r1*2+&zend_jit_hot_counters], ZEND_JIT_HOT_COUNTER_INIT + | sub r4, 8 + | push IP + | push FP + | EXT_CALL zend_jit_hot_func, r0 + | add r4, 16 + | .endif + | jmp aword [IP] + return 1; +} + +static int zend_jit_hybrid_loop_counter_stub(dasm_State **Dst) +{ + |->hybrid_loop_counter: + | mov r0, EX->func + | mov r1, aword [r0 + offsetof(zend_op_array, function_name)] + | test r1, r1 + | jne >1 + | mov r0, aword [r0 + offsetof(zend_op_array, filename)] + |1: + | shr r0, 3 + | mov r1, r0 + | .if X64 + | shr r0, 30 + | xor r0, r1 + | mov64 r1, 0xbf58476d1ce4e5b9 + | imul r0, r1 + | mov r1, r0 + | shr r0, 27 + | xor r0, r1 + | mov64 r1, 0x94d049bb133111eb + | imul r0, r1 + | mov r1, r0 + | shr r0, 31 + | xor r1, r0 + | and r1, ZEND_HOT_COUNTERS_COUNT-1 + | LOAD_ADDR r0, &zend_jit_hot_counters + | sub word [r0+r1*2], ZEND_JIT_HOT_LOOP_COST + | .else + | shr r0, 16 + | xor r0, r1 + | imul r0, 0x45d9f3b + | mov r1, r0 + | shr r0, 16 + | xor r0, r1 + | imul r0, 0x45d9f3b + | mov r1, r0 + | shr r0, 16 + | xor r1, r0 + | and r1, ZEND_HOT_COUNTERS_COUNT-1 + | sub word [r1*2+&zend_jit_hot_counters], ZEND_JIT_HOT_LOOP_COST + | .endif + | jle >1 + | mov r1, EX->func + | mov r0, IP + | sub r0, aword [r1 + offsetof(zend_op_array, opcodes)] + | // divide by sizeof(zend_op) + | .if X64 + | sar r0, 5 + | .else + | sar r0, 2 + | imul r0, 0xb6db6db7 + | .endif + | mov r1, aword [r1 + offsetof(zend_op_array, reserved[zend_func_info_rid])] + | .if X64 + | jmp aword [r1+r0*8] + | .else + | jmp aword [r1+r0*4] + | .endif + |1: + | .if X64 + | mov word [r0+r1*2], ZEND_JIT_HOT_COUNTER_INIT + | mov CARG1, FP + | mov CARG2, IP + | EXT_CALL zend_jit_hot_func, r0 + | .else + | mov word [r1*2+&zend_jit_hot_counters], ZEND_JIT_HOT_COUNTER_INIT + | sub r4, 8 + | push IP + | push FP + | EXT_CALL zend_jit_hot_func, r0 + | add r4, 16 + | .endif + | jmp aword [IP] + return 1; +} +#endif + static const zend_jit_stub zend_jit_stubs[] = { JIT_STUB(interrupt_handler), JIT_STUB(exception_handler), From 094757c1515598d5fa074726e82027ba79361d28 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 16 May 2017 10:05:35 +0300 Subject: [PATCH 454/569] Improved HYBRID VM support --- ext/opcache/jit/zend_jit.c | 130 ++++++++++++-------------- ext/opcache/jit/zend_jit_internal.h | 2 + ext/opcache/jit/zend_jit_vm_helpers.c | 6 +- ext/opcache/jit/zend_jit_x86.dasc | 126 ++++++++++++------------- 4 files changed, 126 insertions(+), 138 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 9eb8dc1e7bde0..a912abdba02fc 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -75,16 +75,17 @@ int zend_jit_profile_counter_rid = -1; int16_t zend_jit_hot_counters[ZEND_HOT_COUNTERS_COUNT]; +const zend_op *zend_jit_halt_op = NULL; +static int zend_jit_vm_kind = 0; + static void *dasm_buf = NULL; static void *dasm_end = NULL; static void **dasm_ptr = NULL; -#if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID) -static const void *hybrid_runtime_jit = NULL; -static const void *hybrid_profile_jit = NULL; -static const void *hybrid_func_counter = NULL; -static const void *hybrid_loop_counter = NULL; -#endif +static const void *zend_jit_runtime_jit_handler = NULL; +static const void *zend_jit_profile_jit_handler = NULL; +static const void *zend_jit_func_counter_handler = NULL; +static const void *zend_jit_loop_counter_handler = NULL; static int zend_may_throw(const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa); static int zend_may_overflow(const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa); @@ -2934,11 +2935,7 @@ void zend_jit_check_funcs(HashTable *function_table, zend_bool is_method) { while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) { opline++; } -#if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID) - if (opline->handler == hybrid_profile_jit) { -#else - if (opline->handler == zend_jit_profile_helper) { -#endif + if (opline->handler == zend_jit_profile_jit_handler) { zend_ulong counter = (zend_ulong)ZEND_COUNTER_INFO(op_array); ZEND_COUNTER_INFO(op_array) = 0; opline->handler = ZEND_FUNC_INFO(op_array); @@ -3002,22 +2999,13 @@ static int zend_jit_setup_hot_counters(zend_op_array *op_array) opline++; } -#if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID) - opline->handler = (const void*)hybrid_func_counter; -#else - opline->handler = (const void*)zend_jit_func_counter_helper; -#endif + opline->handler = (const void*)zend_jit_func_counter_handler; for (i = 0; i < cfg.blocks_count; i++) { if ((cfg.blocks[i].flags & ZEND_BB_REACHABLE) && (cfg.blocks[i].flags & ZEND_BB_LOOP_HEADER)) { -#if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID) - op_array->opcodes[cfg.blocks[i].start].handler = - (const void*)hybrid_loop_counter; -#else op_array->opcodes[cfg.blocks[i].start].handler = - (const void*)zend_jit_loop_counter_helper; -#endif + (const void*)zend_jit_loop_counter_handler; } } @@ -3053,19 +3041,11 @@ ZEND_API int zend_jit_op_array(zend_op_array *op_array, zend_script *script) /* Set run-time JIT handler */ while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) { -#if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID) - opline->handler = (const void*)hybrid_runtime_jit; -#else - opline->handler = (const void*)zend_runtime_jit; -#endif + opline->handler = (const void*)zend_jit_runtime_jit_handler; opline++; } ZEND_SET_FUNC_INFO(op_array, (void*)opline->handler); -#if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID) - opline->handler = (const void*)hybrid_runtime_jit; -#else - opline->handler = (const void*)zend_runtime_jit; -#endif + opline->handler = (const void*)zend_jit_runtime_jit_handler; return SUCCESS; } else if (zend_jit_trigger == ZEND_JIT_ON_PROF_REQUEST) { @@ -3076,11 +3056,7 @@ ZEND_API int zend_jit_op_array(zend_op_array *op_array, zend_script *script) opline++; } ZEND_SET_FUNC_INFO(op_array, (void*)opline->handler); -#if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID) - opline->handler = (const void*)hybrid_profile_jit; -#else - opline->handler = zend_jit_profile_helper; -#endif + opline->handler = (const void*)zend_jit_profile_jit_handler; } return SUCCESS; @@ -3257,43 +3233,48 @@ static int zend_jit_make_stubs(void) } } -#if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID) - dasm_setup(&dasm_state, dasm_actions); - if (!zend_jit_hybrid_runtime_jit_stub(&dasm_state)) { - return 0; - } - hybrid_runtime_jit = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, "JIT$$hybrid_runtime_jit"); - if (!hybrid_runtime_jit) { - return 0; - } + if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { + dasm_setup(&dasm_state, dasm_actions); + if (!zend_jit_hybrid_runtime_jit_stub(&dasm_state)) { + return 0; + } + zend_jit_runtime_jit_handler = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, "JIT$$hybrid_runtime_jit"); + if (!zend_jit_runtime_jit_handler) { + return 0; + } - dasm_setup(&dasm_state, dasm_actions); - if (!zend_jit_hybrid_profile_jit_stub(&dasm_state)) { - return 0; - } - hybrid_profile_jit = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, "JIT$$hybrid_profile_jit"); - if (!hybrid_profile_jit) { - return 0; - } + dasm_setup(&dasm_state, dasm_actions); + if (!zend_jit_hybrid_profile_jit_stub(&dasm_state)) { + return 0; + } + zend_jit_profile_jit_handler = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, "JIT$$hybrid_profile_jit"); + if (!zend_jit_profile_jit_handler) { + return 0; + } - dasm_setup(&dasm_state, dasm_actions); - if (!zend_jit_hybrid_func_counter_stub(&dasm_state)) { - return 0; - } - hybrid_func_counter = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, "JIT$$hybrid_func_counter"); - if (!hybrid_profile_jit) { - return 0; - } + dasm_setup(&dasm_state, dasm_actions); + if (!zend_jit_hybrid_func_counter_stub(&dasm_state)) { + return 0; + } + zend_jit_func_counter_handler = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, "JIT$$hybrid_func_counter"); + if (!zend_jit_func_counter_handler) { + return 0; + } - dasm_setup(&dasm_state, dasm_actions); - if (!zend_jit_hybrid_loop_counter_stub(&dasm_state)) { - return 0; - } - hybrid_loop_counter = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, "JIT$$hybrid_loop_counter"); - if (!hybrid_profile_jit) { - return 0; + dasm_setup(&dasm_state, dasm_actions); + if (!zend_jit_hybrid_loop_counter_stub(&dasm_state)) { + return 0; + } + zend_jit_loop_counter_handler = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, "JIT$$hybrid_loop_counter"); + if (!zend_jit_loop_counter_handler) { + return 0; + } + } else { + zend_jit_runtime_jit_handler = (const void*)zend_runtime_jit; + zend_jit_profile_jit_handler = (const void*)zend_jit_profile_helper; + zend_jit_func_counter_handler = (const void*)zend_jit_func_counter_helper; + zend_jit_loop_counter_handler = (const void*)zend_jit_loop_counter_handler; } -#endif dasm_free(&dasm_state); return 1; @@ -3310,6 +3291,15 @@ ZEND_API int zend_jit_startup(zend_long jit, size_t size) zend_jit_reg_alloc = ZEND_JIT_REG_ALLOC(jit); zend_jit_cpu_flags = ZEND_JIT_CPU_FLAGS(jit); + zend_jit_vm_kind = zend_vm_kind(); + if (zend_jit_vm_kind != ZEND_VM_KIND_CALL && + zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) { + // TODO: error reporting and cleanup ??? + return FAILURE; + } + + zend_jit_halt_op = zend_get_halt_op(); + if (zend_jit_setup() != SUCCESS) { // TODO: error reporting and cleanup ??? return FAILURE; diff --git a/ext/opcache/jit/zend_jit_internal.h b/ext/opcache/jit/zend_jit_internal.h index f8776b5edb09b..47acf61a287cc 100644 --- a/ext/opcache/jit/zend_jit_internal.h +++ b/ext/opcache/jit/zend_jit_internal.h @@ -36,6 +36,8 @@ void zend_jit_hot_func(zend_execute_data *execute_data, const zend_op *opline); /* VM handlers */ typedef void (ZEND_FASTCALL *zend_vm_opcode_handler_t)(void); +extern const zend_op *zend_jit_halt_op; + /* VM helpers */ void ZEND_FASTCALL zend_jit_leave_nested_func_helper(uint32_t call_info); void ZEND_FASTCALL zend_jit_leave_top_func_helper(uint32_t call_info); diff --git a/ext/opcache/jit/zend_jit_vm_helpers.c b/ext/opcache/jit/zend_jit_vm_helpers.c index e0b338f972912..ad25df2a9019c 100644 --- a/ext/opcache/jit/zend_jit_vm_helpers.c +++ b/ext/opcache/jit/zend_jit_vm_helpers.c @@ -85,11 +85,7 @@ void ZEND_FASTCALL zend_jit_leave_top_func_helper(uint32_t call_info) OBJ_RELEASE((zend_object*)EX(func)->op_array.prototype); } execute_data = EG(current_execute_data); -#if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID) - opline = zend_get_real_exit_op(); -#else - opline = NULL; -#endif + opline = zend_jit_halt_op; } void ZEND_FASTCALL zend_jit_copy_extra_args_helper(void) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 533508cb96558..115455ceecc66 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -1580,11 +1580,11 @@ static int zend_jit_interrupt_handler_stub(dasm_State **Dst) | //} } | //ZEND_VM_CONTINUE() -#if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID) - | add r4, HYBRID_SPAD -#else - | add r4, SPAD // stack alignment -#endif + if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { + | add r4, HYBRID_SPAD + } else { + | add r4, SPAD // stack alignment + } | jmp aword [IP] return 1; @@ -1592,18 +1592,18 @@ static int zend_jit_interrupt_handler_stub(dasm_State **Dst) static int zend_jit_exception_handler_stub(dasm_State **Dst) { -//#ifdef ZEND_VM_KIND_HYBRID -// const void *handler = zend_get_real_opcode_handler(EG(exception_op)); -//#else const void *handler = EG(exception_op)->handler; -//#endif + +// if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { +// handler = zend_get_opcode_handler_func(EG(exception_op)); +// } |->exception_handler: -#if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID) - | add r4, HYBRID_SPAD -#else - | add r4, SPAD // stack alignment -#endif + if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { + | add r4, HYBRID_SPAD + } else { + | add r4, SPAD // stack alignment + } | EXT_JMP handler, r0 return 1; @@ -1621,24 +1621,24 @@ static int zend_jit_exception_handler_undef_stub(dasm_State **Dst) static int zend_jit_leave_function_stub(dasm_State **Dst) { |->leave_function_handler: -#if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID) - | test FCARG1d, ZEND_CALL_TOP - | jnz >1 - | EXT_CALL zend_jit_leave_nested_func_helper, r0 - | add r4, HYBRID_SPAD // stack alignment - | jmp aword [IP] - |1: - | EXT_CALL zend_jit_leave_top_func_helper, r0 - | add r4, HYBRID_SPAD // stack alignment - | jmp aword [IP] -#else - | add r4, SPAD - | test FCARG1d, ZEND_CALL_TOP - | jnz >1 - | EXT_JMP zend_jit_leave_nested_func_helper, r0 - |1: - | EXT_JMP zend_jit_leave_top_func_helper, r0 -#endif + if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { + | test FCARG1d, ZEND_CALL_TOP + | jnz >1 + | EXT_CALL zend_jit_leave_nested_func_helper, r0 + | add r4, HYBRID_SPAD // stack alignment + | jmp aword [IP] + |1: + | EXT_CALL zend_jit_leave_top_func_helper, r0 + | add r4, HYBRID_SPAD // stack alignment + | jmp aword [IP] + } else { + | add r4, SPAD + | test FCARG1d, ZEND_CALL_TOP + | jnz >1 + | EXT_JMP zend_jit_leave_nested_func_helper, r0 + |1: + | EXT_JMP zend_jit_leave_top_func_helper, r0 + } return 1; } @@ -1947,7 +1947,6 @@ static int zend_jit_double_one_stub(dasm_State **Dst) return 1; } -#if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID) static int zend_jit_hybrid_runtime_jit_stub(dasm_State **Dst) { |->hybrid_runtime_jit: @@ -2128,7 +2127,6 @@ static int zend_jit_hybrid_loop_counter_stub(dasm_State **Dst) | jmp aword [IP] return 1; } -#endif static const zend_jit_stub zend_jit_stubs[] = { JIT_STUB(interrupt_handler), @@ -2187,11 +2185,11 @@ static int zend_jit_align_func(dasm_State **Dst) static int zend_jit_prologue(dasm_State **Dst) { -#if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID) - | sub r4, HYBRID_SPAD -#else - | sub r4, SPAD // stack alignment -#endif + if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { + | sub r4, HYBRID_SPAD + } else { + | sub r4, SPAD // stack alignment + } return 1; } @@ -2278,11 +2276,13 @@ static int zend_jit_check_exception_undef_result(dasm_State **Dst, const zend_op static int zend_jit_handler(dasm_State **Dst, const zend_op *opline, int may_throw) { -#ifdef ZEND_VM_KIND_HYBRID - const void *handler = zend_get_real_opcode_handler(opline); -#else - const void *handler = opline->handler; -#endif + const void *handler; + + if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { + handler = zend_get_opcode_handler_func(opline); + } else { + handler = opline->handler; + } if (!zend_jit_set_valid_ip(Dst, opline)) { return 0; @@ -2324,20 +2324,20 @@ static int zend_jit_handler(dasm_State **Dst, const zend_op *opline, int may_thr static int zend_jit_tail_handler(dasm_State **Dst, const zend_op *opline) { -//#ifdef ZEND_VM_KIND_HYBRID -// const void *handler = zend_get_real_opcode_handler(opline); -//#else const void *handler = opline->handler; -//#endif + +// if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { +// handler = zend_get_opcode_handler_func(opline); +// } if (!zend_jit_set_valid_ip(Dst, opline)) { return 0; } -#if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID) - | add r4, HYBRID_SPAD -#else - | add r4, SPAD // stack alignment -#endif + if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { + | add r4, HYBRID_SPAD + } else { + | add r4, SPAD // stack alignment + } | EXT_JMP handler, r0 last_valid_opline = NULL; return 1; @@ -7170,11 +7170,11 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar | jmp >9 } #else -#if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID) - | add r4, HYBRID_SPAD -#else - | add r4, SPAD // stack alignment -#endif + if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { + | add r4, HYBRID_SPAD + } else { + | add r4, SPAD // stack alignment + } | jmp aword [IP] #endif } @@ -8247,11 +8247,11 @@ static int zend_jit_leave_func(dasm_State **Dst, const zend_op *opline, zend_op_ | jne ->leave_throw_handler | // opline = EX(opline) + 1 | add IP, sizeof(zend_op) -#if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID) - | add r4, HYBRID_SPAD -#else - | add r4, SPAD // stack alignment -#endif + if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { + | add r4, HYBRID_SPAD + } else { + | add r4, SPAD // stack alignment + } #ifdef CONTEXT_THREADED_JIT | ret #else From 814350bebc58bca53fe55e4776181253247c870b Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 24 May 2017 23:38:43 +0300 Subject: [PATCH 455/569] Minimal JIT support for ZEND_IN_ARRAY --- ext/opcache/jit/zend_jit.c | 20 ++------------------ 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index a912abdba02fc..609b4f94b6b43 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -2584,15 +2584,7 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa, const zend_op *rt_op /* skip */ } else if ((opline->opcode == ZEND_JMPZ || (opline->opcode == ZEND_JMPNZ)) && - ((opline-1)->opcode == ZEND_IS_IDENTICAL || - (opline-1)->opcode == ZEND_IS_NOT_IDENTICAL || - (opline-1)->opcode == ZEND_ISSET_ISEMPTY_VAR || - (opline-1)->opcode == ZEND_ISSET_ISEMPTY_STATIC_PROP || - (opline-1)->opcode == ZEND_ISSET_ISEMPTY_DIM_OBJ || - (opline-1)->opcode == ZEND_ISSET_ISEMPTY_PROP_OBJ || - (opline-1)->opcode == ZEND_INSTANCEOF || - (opline-1)->opcode == ZEND_TYPE_CHECK || - (opline-1)->opcode == ZEND_DEFINED)) { + zend_is_smart_branch(opline-1)) { /* smart branch */ if (!zend_jit_cond_jmp(&dasm_state, opline + 1, ssa->cfg.blocks[b].successors[0])) { goto jit_failure; @@ -2732,15 +2724,7 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa, const zend_op *rt_op } break; } - } else if ((opline-1)->opcode == ZEND_IS_IDENTICAL || - (opline-1)->opcode == ZEND_IS_NOT_IDENTICAL || - (opline-1)->opcode == ZEND_ISSET_ISEMPTY_VAR || - (opline-1)->opcode == ZEND_ISSET_ISEMPTY_STATIC_PROP || - (opline-1)->opcode == ZEND_ISSET_ISEMPTY_DIM_OBJ || - (opline-1)->opcode == ZEND_ISSET_ISEMPTY_PROP_OBJ || - (opline-1)->opcode == ZEND_INSTANCEOF || - (opline-1)->opcode == ZEND_TYPE_CHECK || - (opline-1)->opcode == ZEND_DEFINED) { + } else if (zend_is_smart_branch(opline-1)) { /* smart branch */ if (!zend_jit_cond_jmp(&dasm_state, opline + 1, ssa->cfg.blocks[b].successors[0])) { goto jit_failure; From 1b8583b940333556157b28f8b2e586e72ddcffc6 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Mon, 5 Jun 2017 18:52:23 +0800 Subject: [PATCH 456/569] Use LVAL directly --- ext/opcache/jit/zend_jit_x86.dasc | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 115455ceecc66..cc12e63e71ec2 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -383,10 +383,19 @@ static void* dasm_labels[zend_lb_MAX]; |.macro SSE_GET_ZVAL_LVAL, reg, addr || if (Z_MODE(addr) == IS_CONST_ZVAL) { +|.if X64 +|| if (!IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(addr)))) { +| mov64 r0, Z_LVAL_P(Z_ZV(addr)) +|| } else { +| mov r0, Z_LVAL_P(Z_ZV(addr)) +|| } +|.else +| mov r0, Z_LVAL_P(Z_ZV(addr)) +|.endif || if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { -| MEM_OP3_3 vcvtsi2sd, xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), aword, Z_ZV(addr), r0 +| vcvtsi2sd, xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), r0 || } else { -| MEM_OP2_2 cvtsi2sd, xmm(reg-ZREG_XMM0), aword, Z_ZV(addr), r0 +| cvtsi2sd, xmm(reg-ZREG_XMM0), r0 || } || } else if (Z_MODE(addr) == IS_MEM_ZVAL) { || if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { From 09cd2c8bce924ee1e4a39ccad8a631973e2072b9 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 17 Jul 2017 15:04:31 +0300 Subject: [PATCH 457/569] Fixed license --- Zend/zend_gdb.c | 12 ++++++------ Zend/zend_gdb.h | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/Zend/zend_gdb.c b/Zend/zend_gdb.c index dec0e6289f1f5..91c887a1e9ad1 100644 --- a/Zend/zend_gdb.c +++ b/Zend/zend_gdb.c @@ -1,16 +1,16 @@ /* +----------------------------------------------------------------------+ - | Zend JIT | + | Zend Engine | +----------------------------------------------------------------------+ - | Copyright (c) 1998-2016 The PHP Group | + | Copyright (c) 1998-2017 Zend Technologies Ltd. (http://www.zend.com) | +----------------------------------------------------------------------+ - | This source file is subject to version 3.01 of the PHP license, | + | This source file is subject to version 2.00 of the Zend license, | | that is bundled with this package in the file LICENSE, and is | | available through the world-wide-web at the following url: | - | http://www.php.net/license/3_01.txt | - | If you did not receive a copy of the PHP license and are unable to | + | http://www.zend.com/license/2_00.txt. | + | If you did not receive a copy of the Zend license and are unable to | | obtain it through the world-wide-web, please send a note to | - | license@php.net so we can mail you a copy immediately. | + | license@zend.com so we can mail you a copy immediately. | +----------------------------------------------------------------------+ | Authors: Dmitry Stogov | | Xinchen Hui | diff --git a/Zend/zend_gdb.h b/Zend/zend_gdb.h index f2dddb05d4e3a..203bc9556e6df 100644 --- a/Zend/zend_gdb.h +++ b/Zend/zend_gdb.h @@ -1,16 +1,16 @@ /* +----------------------------------------------------------------------+ - | Zend JIT | + | Zend Engine | +----------------------------------------------------------------------+ - | Copyright (c) 1998-2016 The PHP Group | + | Copyright (c) 1998-2017 Zend Technologies Ltd. (http://www.zend.com) | +----------------------------------------------------------------------+ - | This source file is subject to version 3.01 of the PHP license, | + | This source file is subject to version 2.00 of the Zend license, | | that is bundled with this package in the file LICENSE, and is | | available through the world-wide-web at the following url: | - | http://www.php.net/license/3_01.txt | - | If you did not receive a copy of the PHP license and are unable to | + | http://www.zend.com/license/2_00.txt. | + | If you did not receive a copy of the Zend license and are unable to | | obtain it through the world-wide-web, please send a note to | - | license@php.net so we can mail you a copy immediately. | + | license@zend.com so we can mail you a copy immediately. | +----------------------------------------------------------------------+ | Authors: Dmitry Stogov | | Xinchen Hui | From d4535ceabda1cd75043c1da9555894dd78c69375 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 17 Jul 2017 15:06:21 +0300 Subject: [PATCH 458/569] enable global register allocator and AVX instruction usage by default --- ext/opcache/jit/zend_jit.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit.h b/ext/opcache/jit/zend_jit.h index bb5cce2604b2f..d8a072e6cbf2d 100644 --- a/ext/opcache/jit/zend_jit.h +++ b/ext/opcache/jit/zend_jit.h @@ -49,7 +49,7 @@ #define ZEND_JIT_CPU_FLAGS(n) (((n) / 1000) % 10) -#define ZEND_JIT_DEFAULT "5" +#define ZEND_JIT_DEFAULT "1205" /* Makes profile based JIT (opcache.jit=2*) to generate code only for most From a8da544d2150bd57dc6141c98465897645c10dd4 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 20 Jul 2017 13:31:12 +0300 Subject: [PATCH 459/569] Optimized long zero to double conversion --- ext/opcache/jit/zend_jit_x86.dasc | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index cc12e63e71ec2..6362ee2d393de 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -383,19 +383,27 @@ static void* dasm_labels[zend_lb_MAX]; |.macro SSE_GET_ZVAL_LVAL, reg, addr || if (Z_MODE(addr) == IS_CONST_ZVAL) { -|.if X64 -|| if (!IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(addr)))) { -| mov64 r0, Z_LVAL_P(Z_ZV(addr)) +|| if (Z_LVAL_P(Z_ZV(addr)) == 0) { +|| if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { +| vxorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0) +|| } else { +| xorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0) +|| } || } else { -| mov r0, Z_LVAL_P(Z_ZV(addr)) -|| } +|.if X64 +|| if (!IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(addr)))) { +| mov64 r0, Z_LVAL_P(Z_ZV(addr)) +|| } else { +| mov r0, Z_LVAL_P(Z_ZV(addr)) +|| } |.else -| mov r0, Z_LVAL_P(Z_ZV(addr)) +| mov r0, Z_LVAL_P(Z_ZV(addr)) |.endif -|| if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { -| vcvtsi2sd, xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), r0 -|| } else { -| cvtsi2sd, xmm(reg-ZREG_XMM0), r0 +|| if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { +| vcvtsi2sd, xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), r0 +|| } else { +| cvtsi2sd, xmm(reg-ZREG_XMM0), r0 +|| } || } || } else if (Z_MODE(addr) == IS_MEM_ZVAL) { || if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { From b56c0ff36ea18cb43e4fd4676cc9a1a12afabecc Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 20 Jul 2017 13:46:56 +0300 Subject: [PATCH 460/569] Generate code for constructor call --- ext/opcache/jit/zend_jit.c | 2 +- ext/opcache/jit/zend_jit_x86.dasc | 80 ++++++++++++++++--------------- 2 files changed, 43 insertions(+), 39 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 42e03ff476e73..8e9b234dc26eb 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -2439,7 +2439,7 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa, const zend_op *rt_op } break; case ZEND_NEW: - if (!zend_jit_new(&dasm_state, opline, &i, op_array, ssa)) { + if (!zend_jit_new(&dasm_state, opline, &i, op_array, ssa, call_level)) { goto jit_failure; } break; diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 6362ee2d393de..8d67c8df2d32c 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -2441,44 +2441,6 @@ static int zend_jit_call(dasm_State **Dst, const zend_op *opline) #endif } -static int zend_jit_new(dasm_State **Dst, const zend_op *opline, int *opnum, zend_op_array *op_array, zend_ssa *ssa) -{ - if (!zend_jit_handler(Dst, opline, 1)) { - return 0; - } - if (opline->extended_value == 0 && (opline+1)->opcode == ZEND_DO_FCALL) { - zend_class_entry *ce = NULL; - - if (zend_jit_level >= ZEND_JIT_LEVEL_OPT_FUNC) { - if (ssa->ops && ssa->var_info) { - zend_ssa_var_info *res_ssa = &ssa->var_info[ssa->ops[opline - op_array->opcodes].result_def]; - if (res_ssa->ce && !res_ssa->is_instanceof) { - ce = res_ssa->ce; - } - } - } else { - if (opline->op1_type == IS_CONST) { - zval *zv = RT_CONSTANT(op_array, opline->op1); - if (Z_TYPE_P(zv) == IS_STRING) { - zval *lc = zv + 1; - ce = (zend_class_entry*)zend_hash_find_ptr(EG(class_table), Z_STR_P(lc)); - } - } - } - - (*opnum)++; - if (!ce || ce->constructor) { - const zend_op *next_opline = opline + 1; - - | cmp IPl, next_opline - | jne >1 - zend_jit_call(Dst, next_opline); - |1: - } - } - return 1; -} - static int zend_jit_spill_store(dasm_State **Dst, zend_jit_addr src, zend_jit_addr dst, uint32_t info, zend_bool set_type) { ZEND_ASSERT(Z_MODE(src) == IS_REG); @@ -7400,6 +7362,48 @@ fallback: } } +static int zend_jit_new(dasm_State **Dst, const zend_op *opline, int *opnum, zend_op_array *op_array, zend_ssa *ssa, int call_level) +{ + if (!zend_jit_handler(Dst, opline, 1)) { + return 0; + } + if (opline->extended_value == 0 && (opline+1)->opcode == ZEND_DO_FCALL) { + zend_class_entry *ce = NULL; + + if (zend_jit_level >= ZEND_JIT_LEVEL_OPT_FUNC) { + if (ssa->ops && ssa->var_info) { + zend_ssa_var_info *res_ssa = &ssa->var_info[ssa->ops[opline - op_array->opcodes].result_def]; + if (res_ssa->ce && !res_ssa->is_instanceof) { + ce = res_ssa->ce; + } + } + } else { + if (opline->op1_type == IS_CONST) { + zval *zv = RT_CONSTANT(op_array, opline->op1); + if (Z_TYPE_P(zv) == IS_STRING) { + zval *lc = zv + 1; + ce = (zend_class_entry*)zend_hash_find_ptr(EG(class_table), Z_STR_P(lc)); + } + } + } + + (*opnum)++; + if (!ce || ce->constructor) { + const zend_op *next_opline = opline + 1; + + | cmp IPl, next_opline + | jne >6 +#if 0 + zend_jit_call(Dst, next_opline); +#else + zend_jit_do_fcall(Dst, next_opline, op_array, ssa, call_level); +#endif + |6: + } + } + return 1; +} + static int zend_jit_send_val(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, zend_lifetime_interval **ra) { uint32_t op1_info; From 1d4c025ec6b41598ef21a018a6b77b80519e99ac Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Sat, 5 Aug 2017 19:37:20 +0800 Subject: [PATCH 461/569] Fixed error message --- ext/opcache/jit/zend_jit_helpers.c | 4 +++- ext/opcache/jit/zend_jit_x86.dasc | 8 +++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/ext/opcache/jit/zend_jit_helpers.c b/ext/opcache/jit/zend_jit_helpers.c index 3433b4b20d2f3..e693b1707efc0 100644 --- a/ext/opcache/jit/zend_jit_helpers.c +++ b/ext/opcache/jit/zend_jit_helpers.c @@ -1951,7 +1951,9 @@ static void ZEND_FASTCALL zend_jit_fetch_obj_r_slow(zend_object *zobj, zval *off zval tmp; if (UNEXPECTED(zobj->handlers->read_property == NULL)) { - zend_error(E_NOTICE, "Trying to get property of non-object"); + zend_string *property_name = zval_get_string(offset); + zend_error(E_NOTICE, "Trying to get property '%s' of non-object", ZSTR_VAL(property_name)); + zend_string_release(property_name); ZVAL_NULL(result); } else { ZVAL_OBJ(&tmp, zobj); diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 8d67c8df2d32c..21613e6269b78 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -9176,11 +9176,13 @@ static int zend_jit_fetch_obj_read(dasm_State **Dst, zend_op *opline, zend_op_ar } |.if X64 | mov CARG1, E_NOTICE - | LOAD_ADDR CARG2, "Trying to get property of non-object" + | LOAD_ADDR CARG2, "Trying to get property '%s' of non-object" + | LOAD_ADDR CARG3, Z_STRVAL_P(member) | EXT_CALL zend_error, r0 |.else - | sub r4, 8 - | push "Trying to get property of non-object" + | sub r4, 4 + | push Z_STRVAL_P(member) + | push "Trying to get property '%s' of non-object" | push E_NOTICE | EXT_CALL zend_error, r0 | add r4, 16 From 8b15dfe1a967ed3ed889a0097a12dfbf55b95877 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Thu, 10 Aug 2017 19:48:26 +0800 Subject: [PATCH 462/569] Fixed typo --- ext/opcache/jit/zend_jit_x86.dasc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 21613e6269b78..11ac023e9fd48 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -9359,7 +9359,7 @@ static zend_bool zend_jit_opline_supports_reg(zend_op_array *op_array, zend_ssa case ZEND_SUB: case ZEND_MUL: op1_info = OP1_INFO(); - op2_info = OP1_INFO(); + op2_info = OP2_INFO(); return !(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) && !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))); @@ -9370,7 +9370,7 @@ static zend_bool zend_jit_opline_supports_reg(zend_op_array *op_array, zend_ssa case ZEND_SR: case ZEND_MOD: op1_info = OP1_INFO(); - op2_info = OP1_INFO(); + op2_info = OP2_INFO(); return !(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG)) && !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG)); From 0251860ff8536cbf2ac710b43ffab056063ddd04 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Fri, 11 Aug 2017 10:29:06 +0800 Subject: [PATCH 463/569] Fixed block_order on op_array has unreachable_free block --- ext/opcache/jit/zend_jit.c | 8 +++---- ext/opcache/tests/jit/unreachable_block.phpt | 25 ++++++++++++++++++++ 2 files changed, 29 insertions(+), 4 deletions(-) create mode 100644 ext/opcache/tests/jit/unreachable_block.phpt diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 8e9b234dc26eb..28e8df7e6b1eb 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -986,10 +986,11 @@ static int *zend_jit_compute_block_order_int(zend_ssa *ssa, int n, int *block_or return block_order; } -static void zend_jit_compute_block_order(zend_ssa *ssa, int *block_order) +static int zend_jit_compute_block_order(zend_ssa *ssa, int *block_order) { int *end = zend_jit_compute_block_order_int(ssa, 0, block_order); - ZEND_ASSERT(end - block_order == ssa->cfg.blocks_count); + + return end - block_order; } static zend_bool zend_jit_in_loop(zend_ssa *ssa, int header, zend_basic_block *b) @@ -1086,13 +1087,12 @@ static int zend_jit_compute_liveness(zend_op_array *op_array, zend_ssa *ssa, zen memset(intervals, 0, ssa->vars_count * sizeof(zend_lifetime_interval*)); zend_bitset_clear(live_in, set_size * ssa->cfg.blocks_count); - zend_jit_compute_block_order(ssa, block_order); /* TODO: Provide a linear block order where all dominators of a block * are before this block, and where all blocks belonging to the same loop * are contiguous ??? */ - for (l = ssa->cfg.blocks_count - 1; l >= 0; l--) { + for (l = zend_jit_compute_block_order(ssa, block_order) - 1; l >= 0; l--) { zend_basic_block *b; i = block_order[l]; diff --git a/ext/opcache/tests/jit/unreachable_block.phpt b/ext/opcache/tests/jit/unreachable_block.phpt new file mode 100644 index 0000000000000..9ccc3d7895f88 --- /dev/null +++ b/ext/opcache/tests/jit/unreachable_block.phpt @@ -0,0 +1,25 @@ +--TEST-- +JIT unreachable_block with block order +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +opcache.protect_memory=1 +--SKIPIF-- + +--FILE-- +returnType) { + case 'float': return $this->returnTypeNullable ? null : 0; + default: return; + } + } +} +?> +okey +--EXPECT-- +okey From d4bef887365fe20b70108b960b2640f0127d5910 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Fri, 11 Aug 2017 10:29:36 +0800 Subject: [PATCH 464/569] Fixed test due to warning message chagned --- ext/opcache/tests/jit/fetch_obj_001.phpt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opcache/tests/jit/fetch_obj_001.phpt b/ext/opcache/tests/jit/fetch_obj_001.phpt index c02ee07fedbcf..d68dd94a6c052 100644 --- a/ext/opcache/tests/jit/fetch_obj_001.phpt +++ b/ext/opcache/tests/jit/fetch_obj_001.phpt @@ -115,7 +115,7 @@ object(stdClass)#%d (2) { } } -Warning: Attempt to modify property of non-object in %sfetch_obj_001.php on line 40 +Warning: Attempt to modify property 'abc' of non-object in %sfetch_obj_001.php on line 40 array(0) { } object(stdClass)#%d (1) { From a6cad7e6abeafc9c8b9e880d5cef9fdc3a7dc89f Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 14 Aug 2017 10:26:33 +0300 Subject: [PATCH 465/569] Fixed register spilling --- ext/opcache/jit/zend_jit_x86.dasc | 14 +++++++++ ext/opcache/tests/jit/reg_alloc_001.phpt | 39 ++++++++++++++++++++++++ 2 files changed, 53 insertions(+) create mode 100644 ext/opcache/tests/jit/reg_alloc_001.phpt diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 11ac023e9fd48..723341f759019 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -5990,9 +5990,23 @@ static int zend_jit_identical(dasm_State **Dst, const zend_op *opline, int b, in } } else { if (opline->op1_type != IS_CONST) { + if (Z_MODE(op1_addr) == IS_REG) { + zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); + if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { + return 0; + } + op1_addr = real_addr; + } | LOAD_ZVAL_ADDR FCARG1a, op1_addr } if (opline->op2_type != IS_CONST) { + if (Z_MODE(op2_addr) == IS_REG) { + zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var); + if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) { + return 0; + } + op2_addr = real_addr; + } | LOAD_ZVAL_ADDR FCARG2a, op2_addr } } diff --git a/ext/opcache/tests/jit/reg_alloc_001.phpt b/ext/opcache/tests/jit/reg_alloc_001.phpt new file mode 100644 index 0000000000000..52932988d30e4 --- /dev/null +++ b/ext/opcache/tests/jit/reg_alloc_001.phpt @@ -0,0 +1,39 @@ +--TEST-- +Register Alloction 001: Spilling in "identical" code +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +opcache.protect_memory=1 +--SKIPIF-- + +--FILE-- + $v) { + if (!isset($k[1])) { + $type |= self::EXCLUDE_PUBLIC; + } else { + $type |= self::EXCLUDE_PRIVATE; + } + + if ((self::EXCLUDE_STRICT & $filter) ? $type === $filter : $type) { + } + } + + return $a; + } + +} +?> +OK +--EXPECT-- +OK From 92e5d31218b843477d8584bcfb0865fca4f19949 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 14 Aug 2017 10:54:31 +0300 Subject: [PATCH 466/569] Fixed JIT for comparison for comparison of integer and double --- ext/opcache/jit/zend_jit_x86.dasc | 14 +++++++------- ext/opcache/tests/jit/cmp_001.phpt | 24 ++++++++++++++++++++++++ ext/opcache/tests/jit/cmp_002.phpt | 24 ++++++++++++++++++++++++ 3 files changed, 55 insertions(+), 7 deletions(-) create mode 100644 ext/opcache/tests/jit/cmp_001.phpt create mode 100644 ext/opcache/tests/jit/cmp_002.phpt diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 723341f759019..1926af55f8671 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -5187,7 +5187,7 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, i break; case ZEND_IS_SMALLER: if (swap) { - | jnae => target_label + | jna => target_label } else { | jae => target_label } @@ -5222,7 +5222,7 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, i break; case ZEND_IS_SMALLER: if (swap) { - | jae => target_label + | ja => target_label } else { | jnae => target_label } @@ -5257,7 +5257,7 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, i break; case ZEND_IS_SMALLER: if (swap) { - | jnae => target_label + | jna => target_label } else { | jae => target_label } @@ -5300,7 +5300,7 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, i break; case ZEND_IS_SMALLER: if (swap) { - | setae al + | seta al | movzx eax, al | lea eax, [eax + 2] | SET_ZVAL_TYPE_INFO res_addr, eax @@ -5357,11 +5357,11 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, i break; case ZEND_IS_SMALLER: if (swap) { - | setae al + | seta al | movzx eax, al | lea eax, [eax + 2] | SET_ZVAL_TYPE_INFO res_addr, eax - | jae => target_label + | ja => target_label } else { | setnae al | movzx eax, al @@ -5413,7 +5413,7 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, i break; case ZEND_IS_SMALLER: if (swap) { - | setae al + | seta al | movzx eax, al | add eax, 2 } else { diff --git a/ext/opcache/tests/jit/cmp_001.phpt b/ext/opcache/tests/jit/cmp_001.phpt new file mode 100644 index 0000000000000..d73974c7258d4 --- /dev/null +++ b/ext/opcache/tests/jit/cmp_001.phpt @@ -0,0 +1,24 @@ +--TEST-- +JIT CMP: 001 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +opcache.protect_memory=1 +--SKIPIF-- + +--FILE-- + $b); +var_dump($a <= $b); +var_dump($a >= $b); +?> +--EXPECT-- +bool(false) +bool(false) +bool(true) +bool(true) diff --git a/ext/opcache/tests/jit/cmp_002.phpt b/ext/opcache/tests/jit/cmp_002.phpt new file mode 100644 index 0000000000000..dab1be12215dc --- /dev/null +++ b/ext/opcache/tests/jit/cmp_002.phpt @@ -0,0 +1,24 @@ +--TEST-- +JIT CMP: 001 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +opcache.protect_memory=1 +--SKIPIF-- + +--FILE-- + $b ? 1 : 0); +var_dump($a <= $b ? 1 : 0); +var_dump($a >= $b ? 1 : 0); +?> +--EXPECT-- +int(0) +int(0) +int(1) +int(1) From e595f6c4918cf3b7728ec0857e05372122155a38 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 14 Aug 2017 17:45:20 +0300 Subject: [PATCH 467/569] Use jb/setb instead of jna/setna --- ext/opcache/jit/zend_jit_x86.dasc | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 1926af55f8671..a15a085fbbf7f 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -5187,14 +5187,14 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, i break; case ZEND_IS_SMALLER: if (swap) { - | jna => target_label + | jbe => target_label } else { | jae => target_label } break; case ZEND_IS_SMALLER_OR_EQUAL: if (swap) { - | jnae => target_label + | jb => target_label } else { | ja => target_label } @@ -5224,14 +5224,14 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, i if (swap) { | ja => target_label } else { - | jnae => target_label + | jb => target_label } break; case ZEND_IS_SMALLER_OR_EQUAL: if (swap) { | jae => target_label } else { - | jna => target_label + | jbe => target_label } break; default: @@ -5257,14 +5257,14 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, i break; case ZEND_IS_SMALLER: if (swap) { - | jna => target_label + | jbe => target_label } else { | jae => target_label } break; case ZEND_IS_SMALLER_OR_EQUAL: if (swap) { - | jnae => target_label + | jb => target_label } else { | ja => target_label } @@ -5306,7 +5306,7 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, i | SET_ZVAL_TYPE_INFO res_addr, eax | jnae => target_label } else { - | setnae al + | setb al | movzx eax, al | lea eax, [eax + 2] | SET_ZVAL_TYPE_INFO res_addr, eax @@ -5321,7 +5321,7 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, i | SET_ZVAL_TYPE_INFO res_addr, eax | jnae => target_label } else { - | setna al + | setbe al | movzx eax, al | lea eax, [eax + 2] | SET_ZVAL_TYPE_INFO res_addr, eax @@ -5363,7 +5363,7 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, i | SET_ZVAL_TYPE_INFO res_addr, eax | ja => target_label } else { - | setnae al + | setb al | movzx eax, al | lea eax, [eax + 2] | SET_ZVAL_TYPE_INFO res_addr, eax @@ -5378,7 +5378,7 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, i | SET_ZVAL_TYPE_INFO res_addr, eax | jae => target_label } else { - | setna al + | setbe al | movzx eax, al | lea eax, [eax + 2] | SET_ZVAL_TYPE_INFO res_addr, eax @@ -5417,7 +5417,7 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, i | movzx eax, al | add eax, 2 } else { - | setnae al + | setb al | movzx eax, al | add eax, 2 } @@ -5428,7 +5428,7 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, i | movzx eax, al | add eax, 2 } else { - | setna al + | setbe al | movzx eax, al | add eax, 2 } From f736d5b0b59b5863660ba82ec9e6084d8cabffae Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 14 Aug 2017 22:08:32 +0300 Subject: [PATCH 468/569] Fixed comparison with NAN --- ext/opcache/jit/zend_jit_x86.dasc | 104 ++++++++++-------- ext/opcache/tests/jit/cmp_003.phpt | 169 +++++++++++++++++++++++++++++ 2 files changed, 226 insertions(+), 47 deletions(-) create mode 100644 ext/opcache/tests/jit/cmp_003.phpt diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index a15a085fbbf7f..3e4b0bb755810 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -5175,9 +5175,8 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, i case ZEND_IS_EQUAL: case ZEND_IS_IDENTICAL: case ZEND_CASE: - | jp >1 + | jp => target_label | jne => target_label - |1: break; case ZEND_IS_NOT_EQUAL: case ZEND_IS_NOT_IDENTICAL: @@ -5187,15 +5186,19 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, i break; case ZEND_IS_SMALLER: if (swap) { + | jp => target_label | jbe => target_label } else { + | jp => target_label | jae => target_label } break; case ZEND_IS_SMALLER_OR_EQUAL: if (swap) { + | jp => target_label | jb => target_label } else { + | jp => target_label | ja => target_label } break; @@ -5216,22 +5219,25 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, i break; case ZEND_IS_NOT_EQUAL: case ZEND_IS_NOT_IDENTICAL: - | jp >1 + | jp => target_label | jne => target_label - |1: break; case ZEND_IS_SMALLER: if (swap) { | ja => target_label } else { + | jp >1 | jb => target_label + |1: } break; case ZEND_IS_SMALLER_OR_EQUAL: if (swap) { | jae => target_label } else { + | jp >1 | jbe => target_label + |1: } break; default: @@ -5240,40 +5246,44 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, i } else if ((opline+1)->opcode == ZEND_JMPZNZ && (opline+1)->op1_type == IS_TMP_VAR && (opline+1)->op1.var == opline->result.var) { + unsigned int target_label2 = ssa->cfg.blocks[b].successors[1]; + target_label = ssa->cfg.blocks[b].successors[0]; switch (opline->opcode) { case ZEND_IS_EQUAL: case ZEND_IS_IDENTICAL: case ZEND_CASE: - | jp >1 + | jp => target_label | jne => target_label - |1: break; case ZEND_IS_NOT_EQUAL: case ZEND_IS_NOT_IDENTICAL: - | jp >1 + | jp => target_label2 | je => target_label |1: break; case ZEND_IS_SMALLER: if (swap) { + | jp => target_label | jbe => target_label } else { + | jp => target_label | jae => target_label } break; case ZEND_IS_SMALLER_OR_EQUAL: if (swap) { + | jp => target_label | jb => target_label } else { + | jp => target_label | ja => target_label } break; default: ZEND_ASSERT(0); } - target_label = ssa->cfg.blocks[b].successors[1]; - | jmp => target_label + | jmp => target_label2 } else if ((opline+1)->opcode == ZEND_JMPZ_EX && (opline+1)->op1_type == IS_TMP_VAR && (opline+1)->op1.var == opline->result.var) { @@ -5285,10 +5295,9 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, i case ZEND_IS_IDENTICAL: case ZEND_CASE: | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE - | jp >1 + | jp => target_label | jne => target_label | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE - |1: break; case ZEND_IS_NOT_EQUAL: case ZEND_IS_NOT_IDENTICAL: @@ -5300,32 +5309,28 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, i break; case ZEND_IS_SMALLER: if (swap) { - | seta al - | movzx eax, al - | lea eax, [eax + 2] - | SET_ZVAL_TYPE_INFO res_addr, eax - | jnae => target_label + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE + | jp => target_label + | jbe => target_label + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE } else { - | setb al - | movzx eax, al - | lea eax, [eax + 2] - | SET_ZVAL_TYPE_INFO res_addr, eax + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE + | jp => target_label | jae => target_label + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE } break; case ZEND_IS_SMALLER_OR_EQUAL: if (swap) { - | setae al - | movzx eax, al - | lea eax, [eax + 2] - | SET_ZVAL_TYPE_INFO res_addr, eax - | jnae => target_label + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE + | jp => target_label + | jb => target_label + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE } else { - | setbe al - | movzx eax, al - | lea eax, [eax + 2] - | SET_ZVAL_TYPE_INFO res_addr, eax + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE + | jp => target_label | ja => target_label + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE } break; default: @@ -5350,10 +5355,9 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, i case ZEND_IS_NOT_EQUAL: case ZEND_IS_NOT_IDENTICAL: | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE - | jp >1 + | jp => target_label | jne => target_label | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE - |1: break; case ZEND_IS_SMALLER: if (swap) { @@ -5363,11 +5367,11 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, i | SET_ZVAL_TYPE_INFO res_addr, eax | ja => target_label } else { - | setb al - | movzx eax, al - | lea eax, [eax + 2] - | SET_ZVAL_TYPE_INFO res_addr, eax - | jnae => target_label + | jp >1 + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE + | jb => target_label + |1: + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE } break; case ZEND_IS_SMALLER_OR_EQUAL: @@ -5378,11 +5382,11 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, i | SET_ZVAL_TYPE_INFO res_addr, eax | jae => target_label } else { - | setbe al - | movzx eax, al - | lea eax, [eax + 2] - | SET_ZVAL_TYPE_INFO res_addr, eax - | jna => target_label + | jp >1 + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE + | jbe => target_label + |1: + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE } break; default: @@ -5417,9 +5421,12 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, i | movzx eax, al | add eax, 2 } else { - | setb al - | movzx eax, al - | add eax, 2 + | jp >1 + | mov eax, IS_TRUE + | jb >2 + |1: + | mov eax, IS_FALSE + |2: } break; case ZEND_IS_SMALLER_OR_EQUAL: @@ -5428,9 +5435,12 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, i | movzx eax, al | add eax, 2 } else { - | setbe al - | movzx eax, al - | add eax, 2 + | jp >1 + | mov eax, IS_TRUE + | jbe >2 + |1: + | mov eax, IS_FALSE + |2: } break; default: diff --git a/ext/opcache/tests/jit/cmp_003.phpt b/ext/opcache/tests/jit/cmp_003.phpt new file mode 100644 index 0000000000000..4623030ed2ce6 --- /dev/null +++ b/ext/opcache/tests/jit/cmp_003.phpt @@ -0,0 +1,169 @@ +--TEST-- +JIT CMP: 003 Comparisoin with NaN +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +opcache.protect_memory=1 +--SKIPIF-- + +--FILE-- + $b); +var_dump($a <= $b); +var_dump($a >= $b); +var_dump($a == $b ? 1 : 0); +var_dump($a != $b ? 1 : 0); +var_dump($a < $b ? 1 : 0); +var_dump($a > $b ? 1 : 0); +var_dump($a <= $b ? 1 : 0); +var_dump($a >= $b ? 1 : 0); +if ($a == $b) { +} else { + echo "1\n"; +} +if ($a != $b) { +} else { + echo "2\n"; +} +if ($a < $b) { +} else { + echo "3\n"; +} +if ($a > $b) { +} else { + echo "4\n"; +} +if ($a <= $b) { +} else { + echo "5\n"; +} +if ($a >= $b) { +} else { + echo "6\n"; +} +var_dump($i == $b ? 1 : 0); +var_dump($i != $b ? 1 : 0); +var_dump($i < $b ? 1 : 0); +var_dump($i > $b ? 1 : 0); +var_dump($i <= $b ? 1 : 0); +var_dump($i >= $b ? 1 : 0); +if ($i == $b) { +} else { + echo "1\n"; +} +if ($i != $b) { +} else { + echo "2\n"; +} +if ($i < $b) { +} else { + echo "3\n"; +} +if ($i > $b) { +} else { + echo "4\n"; +} +if ($i <= $b) { +} else { + echo "5\n"; +} +if ($i >= $b) { +} else { + echo "6\n"; +} +var_dump($a == $b && t()); +var_dump($a != $b && t()); +var_dump($a < $b && t()); +var_dump($a > $b && t()); +var_dump($a <= $b && t()); +var_dump($a >= $b && t()); +var_dump($a == $b || f()); +var_dump($a != $b || f()); +var_dump($a < $b || f()); +var_dump($a > $b || f()); +var_dump($a <= $b || f()); +var_dump($a >= $b || f()); +var_dump($i == $b && t()); +var_dump($i != $b && t()); +var_dump($i < $b && t()); +var_dump($i > $b && t()); +var_dump($i <= $b && t()); +var_dump($i >= $b && t()); +var_dump($i == $b || f()); +var_dump($i != $b || f()); +var_dump($i < $b || f()); +var_dump($i > $b || f()); +var_dump($i <= $b || f()); +var_dump($i >= $b || f()); +?> +--EXPECT-- +bool(false) +bool(true) +bool(false) +bool(false) +bool(false) +bool(false) +int(0) +int(1) +int(0) +int(0) +int(0) +int(0) +1 +3 +4 +5 +6 +int(0) +int(1) +int(0) +int(0) +int(0) +int(0) +1 +3 +4 +5 +6 +bool(false) +!bool(true) +bool(false) +bool(false) +bool(false) +bool(false) +!bool(false) +bool(true) +!bool(false) +!bool(false) +!bool(false) +!bool(false) +bool(false) +!bool(true) +bool(false) +bool(false) +bool(false) +bool(false) +!bool(false) +bool(true) +!bool(false) +!bool(false) +!bool(false) +!bool(false) From acf57157798ca7a43136b9b6b035c7f57305f659 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Wed, 16 Aug 2017 01:26:42 +0800 Subject: [PATCH 469/569] Fixed bug of continue jumps --- ext/opcache/jit/zend_jit_x86.dasc | 10 ++++++++++ ext/opcache/tests/jit/continue_jmp.phpt | 26 +++++++++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 ext/opcache/tests/jit/continue_jmp.phpt diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 3e4b0bb755810..a60c2add0ed13 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -6236,9 +6236,19 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, int b, } else if (opline->opcode == ZEND_JMPZ_EX) { set_bool = 1; false_label = ssa->cfg.blocks[b].successors[0]; + if ((op_array->opcodes + ssa->cfg.blocks[false_label].start)->opcode == ZEND_JMPZ) { + false_label = ssa->cfg.blocks[false_label].successors[0]; + } else if ((op_array->opcodes + ssa->cfg.blocks[false_label].start)->opcode == ZEND_JMPNZ) { + false_label++; + } } else if (opline->opcode == ZEND_JMPNZ_EX) { set_bool = 1; true_label = ssa->cfg.blocks[b].successors[0]; + if ((op_array->opcodes + ssa->cfg.blocks[true_label].start)->opcode == ZEND_JMPNZ) { + true_label = ssa->cfg.blocks[true_label].successors[0]; + } else if ((op_array->opcodes + ssa->cfg.blocks[true_label].start)->opcode == ZEND_JMPZ) { + true_label++; + } } else { ZEND_ASSERT(0); } diff --git a/ext/opcache/tests/jit/continue_jmp.phpt b/ext/opcache/tests/jit/continue_jmp.phpt new file mode 100644 index 0000000000000..72475d58b588e --- /dev/null +++ b/ext/opcache/tests/jit/continue_jmp.phpt @@ -0,0 +1,26 @@ +--TEST-- +JIT continue jmps +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +opcache.protect_memory=1 +--SKIPIF-- + +--FILE-- + +--EXPECT-- +bool(true) From cab7da63db607f438dbdac022b7d48b0015b46e0 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 16 Aug 2017 09:54:18 +0300 Subject: [PATCH 470/569] Fixed register allocation (SEND_VAL_EX uses %r0 as temporary register) --- ext/opcache/jit/zend_jit_x86.dasc | 2 +- ext/opcache/tests/jit/reg_alloc_002.phpt | 28 ++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 ext/opcache/tests/jit/reg_alloc_002.phpt diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index a60c2add0ed13..688f31d7c5652 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -9518,7 +9518,7 @@ static zend_regset zend_jit_get_scratch_regset(zend_op_array *op_array, zend_ssa case ZEND_SEND_VAL: case ZEND_SEND_VAL_EX: if (ssa->ops[line].op1_use == current_var) { - regset = ZEND_REGSET_EMPTY; + regset = ZEND_REGSET(ZREG_R0); break; } op1_info = OP1_INFO(); diff --git a/ext/opcache/tests/jit/reg_alloc_002.phpt b/ext/opcache/tests/jit/reg_alloc_002.phpt new file mode 100644 index 0000000000000..cb032f782264e --- /dev/null +++ b/ext/opcache/tests/jit/reg_alloc_002.phpt @@ -0,0 +1,28 @@ +--TEST-- +Register Alloction 002: SEND_VAL_EX uses %r0 as a temporay register +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +opcache.protect_memory=1 +--SKIPIF-- + +--FILE-- +process(function($i, $v) { var_dump($i); }); +?> +--EXPECT-- +int(0) +int(1) +int(2) From f9566c1d1601832619ea59d8a477c67cda15c7a1 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 16 Aug 2017 10:46:43 +0300 Subject: [PATCH 471/569] Revert incomplete fix, but keep the test --- ext/opcache/jit/zend_jit_x86.dasc | 10 ---------- .../tests/jit/{continue_jmp.phpt => jmpz_001.phpt} | 2 +- 2 files changed, 1 insertion(+), 11 deletions(-) rename ext/opcache/tests/jit/{continue_jmp.phpt => jmpz_001.phpt} (80%) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 688f31d7c5652..e34331405c931 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -6236,19 +6236,9 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, int b, } else if (opline->opcode == ZEND_JMPZ_EX) { set_bool = 1; false_label = ssa->cfg.blocks[b].successors[0]; - if ((op_array->opcodes + ssa->cfg.blocks[false_label].start)->opcode == ZEND_JMPZ) { - false_label = ssa->cfg.blocks[false_label].successors[0]; - } else if ((op_array->opcodes + ssa->cfg.blocks[false_label].start)->opcode == ZEND_JMPNZ) { - false_label++; - } } else if (opline->opcode == ZEND_JMPNZ_EX) { set_bool = 1; true_label = ssa->cfg.blocks[b].successors[0]; - if ((op_array->opcodes + ssa->cfg.blocks[true_label].start)->opcode == ZEND_JMPNZ) { - true_label = ssa->cfg.blocks[true_label].successors[0]; - } else if ((op_array->opcodes + ssa->cfg.blocks[true_label].start)->opcode == ZEND_JMPZ) { - true_label++; - } } else { ZEND_ASSERT(0); } diff --git a/ext/opcache/tests/jit/continue_jmp.phpt b/ext/opcache/tests/jit/jmpz_001.phpt similarity index 80% rename from ext/opcache/tests/jit/continue_jmp.phpt rename to ext/opcache/tests/jit/jmpz_001.phpt index 72475d58b588e..99baa9331dc29 100644 --- a/ext/opcache/tests/jit/continue_jmp.phpt +++ b/ext/opcache/tests/jit/jmpz_001.phpt @@ -1,5 +1,5 @@ --TEST-- -JIT continue jmps +JIT JMPZ: JMPZ may require code for "smart branch" and at the same time be a target of another JMP. --INI-- opcache.enable=1 opcache.enable_cli=1 From b46a6bc2512ff4dfb60e21a1d2ed16702e53b151 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 16 Aug 2017 13:01:29 +0300 Subject: [PATCH 472/569] Fixed handling of smart branch instruction pairs splitted across basic blocks. --- ext/opcache/jit/zend_jit.c | 68 ++++++++++++++++++++++++++++++++++---- 1 file changed, 61 insertions(+), 7 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 28e8df7e6b1eb..373712500bd02 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -2071,6 +2071,58 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa, const zend_op *rt_op #endif } + if (ssa->cfg.blocks[b].len == 1 && + ssa->cfg.blocks[b].start != 0 && + zend_is_smart_branch(&op_array->opcodes[ssa->cfg.blocks[b].start-1])) { + + /* Special handling for splitted smart branch instruction pairs */ + opline = &op_array->opcodes[ssa->cfg.blocks[b].start]; + if (zend_jit_level >= ZEND_JIT_LEVEL_INLINE) { + if ((opline->opcode == ZEND_JMPZ || + opline->opcode == ZEND_JMPNZ || + opline->opcode == ZEND_JMPZNZ || + opline->opcode == ZEND_JMPZ_EX || + opline->opcode == ZEND_JMPNZ_EX) && + ((opline-1)->opcode == ZEND_IS_EQUAL || + (opline-1)->opcode == ZEND_IS_NOT_EQUAL || + (opline-1)->opcode == ZEND_IS_SMALLER || + (opline-1)->opcode == ZEND_IS_SMALLER_OR_EQUAL || + (opline-1)->opcode == ZEND_CASE)) { + zend_jit_jmp(&dasm_state, b + 1); + } else if (opline->opcode == ZEND_JMPZ || + opline->opcode == ZEND_JMPNZ) { + /* smart branch */ + if (!zend_jit_cond_jmp(&dasm_state, opline + 1, ssa->cfg.blocks[b].successors[0])) { + goto jit_failure; + } + zend_jit_jmp(&dasm_state, b + 1); + } + } else { + if (opline->opcode == ZEND_JMPZ || + opline->opcode == ZEND_JMPNZ) { + opline--; + if ((opline->opcode == ZEND_IS_EQUAL || + opline->opcode == ZEND_IS_NOT_EQUAL || + opline->opcode == ZEND_IS_SMALLER || + opline->opcode == ZEND_IS_SMALLER_OR_EQUAL || + opline->opcode == ZEND_CASE) && + ((OP1_INFO() & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) || + (OP2_INFO() & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))))) { + /* might be smart branch */ + if (!zend_jit_smart_branch(&dasm_state, opline + 1, (b + 1), ssa->cfg.blocks[b].successors[0])) { + goto jit_failure; + } + } else { + /* smart branch */ + if (!zend_jit_cond_jmp(&dasm_state, opline + 2, ssa->cfg.blocks[b].successors[0])) { + goto jit_failure; + } + zend_jit_jmp(&dasm_state, b + 1); + } + } + } + } + zend_jit_label(&dasm_state, b); if (ssa->cfg.blocks[b].flags & ZEND_BB_TARGET) { if (!zend_jit_reset_opline(&dasm_state, op_array->opcodes + ssa->cfg.blocks[b].start)) { @@ -2269,13 +2321,15 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa, const zend_op *rt_op case ZEND_JMPZNZ: case ZEND_JMPZ_EX: case ZEND_JMPNZ_EX: - if ((opline-1)->opcode == ZEND_IS_EQUAL || - (opline-1)->opcode == ZEND_IS_NOT_EQUAL || - (opline-1)->opcode == ZEND_IS_SMALLER || - (opline-1)->opcode == ZEND_IS_SMALLER_OR_EQUAL || - (opline-1)->opcode == ZEND_CASE) { + if (i != ssa->cfg.blocks[b].start && + ((opline-1)->opcode == ZEND_IS_EQUAL || + (opline-1)->opcode == ZEND_IS_NOT_EQUAL || + (opline-1)->opcode == ZEND_IS_SMALLER || + (opline-1)->opcode == ZEND_IS_SMALLER_OR_EQUAL || + (opline-1)->opcode == ZEND_CASE)) { /* skip */ - } else if ((opline->opcode == ZEND_JMPZ || + } else if (i != ssa->cfg.blocks[b].start && + (opline->opcode == ZEND_JMPZ || (opline->opcode == ZEND_JMPNZ)) && zend_is_smart_branch(opline-1)) { /* smart branch */ @@ -2393,7 +2447,7 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa, const zend_op *rt_op break; case ZEND_JMPZ: case ZEND_JMPNZ: - if (i != 0) { + if (i != ssa->cfg.blocks[b].start) { if ((opline-1)->opcode == ZEND_IS_EQUAL || (opline-1)->opcode == ZEND_IS_NOT_EQUAL || (opline-1)->opcode == ZEND_IS_SMALLER || From d463fc149931678b1cc70eb68528998ebddb72f8 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 16 Aug 2017 23:00:16 +0300 Subject: [PATCH 473/569] unused label --- ext/opcache/jit/zend_jit_x86.dasc | 1 - 1 file changed, 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index e34331405c931..6bb6c9c3824a9 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -5260,7 +5260,6 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, i case ZEND_IS_NOT_IDENTICAL: | jp => target_label2 | je => target_label - |1: break; case ZEND_IS_SMALLER: if (swap) { From e45fb6e7e4594dd9e030db622b8f1c396ad0c640 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Thu, 17 Aug 2017 16:51:02 +0800 Subject: [PATCH 474/569] Fixed checking of should_send_by_ref in zend_send_var_ex --- ext/opcache/jit/zend_jit_x86.dasc | 8 +----- ext/opcache/tests/jit/send_var_ex_001.phpt | 30 ++++++++++++++++++++++ 2 files changed, 31 insertions(+), 7 deletions(-) create mode 100644 ext/opcache/tests/jit/send_var_ex_001.phpt diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 6bb6c9c3824a9..baa4a4155982c 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -7619,13 +7619,7 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, zend_op_ar } if (opline->opcode == ZEND_SEND_VAR_EX || opline->opcode == ZEND_SEND_VAR_NO_REF_EX) { - uint32_t mask; - - if (opline->opcode == ZEND_SEND_VAR_EX) { - mask = ZEND_SEND_BY_REF << ((arg_num + 3) * 2); - } else { - mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2); - } + uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2); | mov r0, EX:RX->func if (arg_num <= MAX_ARG_FLAG_NUM) { diff --git a/ext/opcache/tests/jit/send_var_ex_001.phpt b/ext/opcache/tests/jit/send_var_ex_001.phpt new file mode 100644 index 0000000000000..dc1d02e60ade1 --- /dev/null +++ b/ext/opcache/tests/jit/send_var_ex_001.phpt @@ -0,0 +1,30 @@ +--TEST-- +JIT ASSIGN: 007 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +;opcache.jit_debug=257 +--SKIPIF-- + +--FILE-- +evalParameters = array("a" => "okey"); + extract($this->evalParameters, EXTR_SKIP); + echo $a; + return false; + } +} + +$a = new A(); + +$a->evaluate(); +?> +--EXPECT-- +okey From b204fa51ff492f9a3aeb3b45496fc41131b7cd7d Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Thu, 17 Aug 2017 16:53:55 +0800 Subject: [PATCH 475/569] Fixed test title --- ext/opcache/tests/jit/send_var_ex_001.phpt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opcache/tests/jit/send_var_ex_001.phpt b/ext/opcache/tests/jit/send_var_ex_001.phpt index dc1d02e60ade1..369f9759a8090 100644 --- a/ext/opcache/tests/jit/send_var_ex_001.phpt +++ b/ext/opcache/tests/jit/send_var_ex_001.phpt @@ -1,5 +1,5 @@ --TEST-- -JIT ASSIGN: 007 +JIT SEND_VAR_EX fails on SHOULD_SEND_BY_REF checking --INI-- opcache.enable=1 opcache.enable_cli=1 From 61aa6116b6e48eb2c2b7d644b8228d4ce5124fce Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Thu, 17 Aug 2017 22:01:39 +0800 Subject: [PATCH 476/569] Added test for FETCH_DIM_FUNC_ARG fix --- .../tests/jit/fetch_dim_func_args_001.phpt | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 ext/opcache/tests/jit/fetch_dim_func_args_001.phpt diff --git a/ext/opcache/tests/jit/fetch_dim_func_args_001.phpt b/ext/opcache/tests/jit/fetch_dim_func_args_001.phpt new file mode 100644 index 0000000000000..65762bb4f4b38 --- /dev/null +++ b/ext/opcache/tests/jit/fetch_dim_func_args_001.phpt @@ -0,0 +1,26 @@ +--TEST-- +JIT FETCH_DIM_FUNC_ARG: 001 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +--SKIPIF-- + +--FILE-- +change($a = array("a" => range(1, 5))); +?> +okey +--EXPECT-- +okey From 49d43426782a2db81d00f58ea854afb93b0c7ab8 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 18 Aug 2017 14:07:06 +0300 Subject: [PATCH 477/569] Added missing symbols --- ext/opcache/jit/zend_jit_disasm_x86.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ext/opcache/jit/zend_jit_disasm_x86.c b/ext/opcache/jit/zend_jit_disasm_x86.c index 54ec42ec50d84..571df581e127b 100644 --- a/ext/opcache/jit/zend_jit_disasm_x86.c +++ b/ext/opcache/jit/zend_jit_disasm_x86.c @@ -432,6 +432,8 @@ static int zend_jit_disasm_init(void) REGISTER_HELPER(zend_jit_copy_extra_args_helper); REGISTER_HELPER(zend_jit_deprecated_or_abstract_helper); REGISTER_HELPER(zend_jit_verify_internal_arg_types_helper); + REGISTER_HELPER(zend_runtime_jit); + REGISTER_HELPER(zend_jit_hot_func); #undef REGISTER_HELPER zend_elf_load_symbols(); From 86f85451b6692c0d020ce10e9a475ee2d2b2dde6 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 18 Aug 2017 14:33:36 +0300 Subject: [PATCH 478/569] Make JIT for HYBRID VM to use only function opcode handlers. --- ext/opcache/jit/zend_jit_x86.dasc | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index baa4a4155982c..450259ebfce9c 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -1609,19 +1609,19 @@ static int zend_jit_interrupt_handler_stub(dasm_State **Dst) static int zend_jit_exception_handler_stub(dasm_State **Dst) { - const void *handler = EG(exception_op)->handler; - -// if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { -// handler = zend_get_opcode_handler_func(EG(exception_op)); -// } - |->exception_handler: if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { + const void *handler = zend_get_opcode_handler_func(EG(exception_op)); + | add r4, HYBRID_SPAD + | EXT_CALL handler, r0 + | jmp aword [IP] } else { + const void *handler = EG(exception_op)->handler; + | add r4, SPAD // stack alignment + | EXT_JMP handler, r0 } - | EXT_JMP handler, r0 return 1; } @@ -2341,21 +2341,21 @@ static int zend_jit_handler(dasm_State **Dst, const zend_op *opline, int may_thr static int zend_jit_tail_handler(dasm_State **Dst, const zend_op *opline) { - const void *handler = opline->handler; - -// if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { -// handler = zend_get_opcode_handler_func(opline); -// } - if (!zend_jit_set_valid_ip(Dst, opline)) { return 0; } if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { + const void *handler = zend_get_opcode_handler_func(opline); + | add r4, HYBRID_SPAD + | EXT_CALL handler, r0 + | jmp aword [IP] } else { + const void *handler = opline->handler; + | add r4, SPAD // stack alignment + | EXT_JMP handler, r0 } - | EXT_JMP handler, r0 last_valid_opline = NULL; return 1; } From ee8849ef7c695c5ee5a721ce5f769284c0ef1fa3 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Sat, 19 Aug 2017 18:42:05 +0800 Subject: [PATCH 479/569] Fixed segfault caused by FETCH_DIM_UNSET type infer(from drupal tests) --- ext/opcache/Optimizer/zend_inference.c | 7 +++++ ext/opcache/jit/zend_jit_x86.dasc | 1 + ext/opcache/tests/jit/assign_035.phpt | 39 ++++++++++++++++++++++++++ 3 files changed, 47 insertions(+) create mode 100644 ext/opcache/tests/jit/assign_035.phpt diff --git a/ext/opcache/Optimizer/zend_inference.c b/ext/opcache/Optimizer/zend_inference.c index 9d5d6cfea5406..39ff7d8d49d42 100644 --- a/ext/opcache/Optimizer/zend_inference.c +++ b/ext/opcache/Optimizer/zend_inference.c @@ -3004,6 +3004,13 @@ static int zend_update_type_info(const zend_op_array *op_array, tmp |= MAY_BE_ARRAY_KEY_STRING; } } + } else if (opline->opcode == ZEND_FETCH_DIM_UNSET) { + if (t1 & MAY_BE_ARRAY) { + tmp |= MAY_BE_RC1; + } + if (t1 & MAY_BE_OBJECT) { + tmp |= t1 & MAY_BE_RCN; + } } j = ssa_vars[ssa_ops[i].result_def].use_chain; while (j >= 0) { diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 450259ebfce9c..278c2c5f58c68 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -4302,6 +4302,7 @@ static int zend_jit_assign_to_variable(dasm_State **Dst, |.code } } else if (in_cold) { + ZEND_ASSERT(RC_MAY_BE_1(var_info)); |.code } |5: diff --git a/ext/opcache/tests/jit/assign_035.phpt b/ext/opcache/tests/jit/assign_035.phpt new file mode 100644 index 0000000000000..967748db79bde --- /dev/null +++ b/ext/opcache/tests/jit/assign_035.phpt @@ -0,0 +1,39 @@ +--TEST-- +JIT ASSIGN: Segfault & memleak if no RC info +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +opcache.protect_memory=1 +;opcache.jit_debug=257 +--SKIPIF-- + +--FILE-- + $closure(), + 'b' => [$closure() => [],], + ]; + + $x = $arr; + unset($x['b'][$closure()]['d']); + + $x = $arr; + $x['a'] = $closure(); + + return "okey"; + } +} + +$a = new A(); +echo $a->test(); +?> +--EXPECT-- +okey From 1eec65699bf31e081569287b9dff7e5df174f664 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Sun, 20 Aug 2017 23:21:18 +0800 Subject: [PATCH 480/569] Fixed incomplete fix --- ext/opcache/jit/zend_jit_x86.dasc | 4 ++- ext/opcache/tests/jit/assign_036.phpt | 43 +++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 ext/opcache/tests/jit/assign_036.phpt diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 278c2c5f58c68..d2aedf0110536 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -4302,7 +4302,9 @@ static int zend_jit_assign_to_variable(dasm_State **Dst, |.code } } else if (in_cold) { - ZEND_ASSERT(RC_MAY_BE_1(var_info)); + if (!RC_MAY_BE_1(var_info)) { + | jmp >5 + } |.code } |5: diff --git a/ext/opcache/tests/jit/assign_036.phpt b/ext/opcache/tests/jit/assign_036.phpt new file mode 100644 index 0000000000000..079fae1769f97 --- /dev/null +++ b/ext/opcache/tests/jit/assign_036.phpt @@ -0,0 +1,43 @@ +--TEST-- +JIT ASSIGN: Assign with INTERNED string(no RC) +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +opcache.protect_memory=1 +;opcache.jit_debug=257 +--SKIPIF-- + +--FILE-- +$propName); + $newType = \gettype($propValue); + if ($propValue === 'false') + { + $newType = 'boolean'; + $propValue = \false; + } + elseif ($propValue === 'true') + { + $newType = 'boolean'; + $propValue = \true; + } + if ($oldType !== $newType) + { + $tmp = $propValue; + \settype($tmp, $newType); + } + $this->propName = $propValue; + } +} +$a = new A; +$a->result = "okey"; +echo $a->result; +?> +--EXPECT-- +okey From f591d3196170877844c3de1da5e8e78002e8eaea Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Tue, 22 Aug 2017 11:56:13 +0800 Subject: [PATCH 481/569] Change to assertion (could make potential problems more visible) --- ext/opcache/jit/zend_jit.c | 2 +- ext/opcache/jit/zend_jit_x86.dasc | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 373712500bd02..a785a9c44a188 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -1248,7 +1248,7 @@ static int zend_jit_compute_liveness(zend_op_array *op_array, zend_ssa *ssa, zen } if (zend_jit_reg_alloc >= ZEND_JIT_REG_ALLOC_GLOBAL) { - /* Register hinting (a cheap way for register coalesing) */ + /* Register hinting (a cheap way for register coalescing) */ for (i = 0; i < ssa->vars_count; i++) { if (intervals[i]) { int var = intervals[i]->ssa_var; diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index d2aedf0110536..278c2c5f58c68 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -4302,9 +4302,7 @@ static int zend_jit_assign_to_variable(dasm_State **Dst, |.code } } else if (in_cold) { - if (!RC_MAY_BE_1(var_info)) { - | jmp >5 - } + ZEND_ASSERT(RC_MAY_BE_1(var_info)); |.code } |5: From a60bc06df23d061edecf8aadeb28ad544c456941 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 4 Sep 2017 15:39:59 +0300 Subject: [PATCH 482/569] Allow registr allocation for math instructions, if at least one of operands is known to be numeric. --- ext/opcache/jit/zend_jit_x86.dasc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 278c2c5f58c68..4201bbe061b67 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -9379,7 +9379,7 @@ static zend_bool zend_jit_opline_supports_reg(zend_op_array *op_array, zend_ssa op1_info = OP1_INFO(); op2_info = OP2_INFO(); return - !(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) && + !(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) || !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))); case ZEND_BW_OR: case ZEND_BW_AND: @@ -9456,7 +9456,8 @@ static zend_bool zend_jit_may_be_in_reg(zend_op_array *op_array, zend_ssa *ssa, int use = ssa->vars[var].use_chain; do { - if (!zend_jit_opline_supports_reg(op_array, ssa, op_array->opcodes + use, var)) { + if (!zend_ssa_is_no_val_use(op_array->opcodes + use, ssa->ops + use, var) && + !zend_jit_opline_supports_reg(op_array, ssa, op_array->opcodes + use, var)) { return 0; } use = zend_ssa_next_use(ssa->ops, var, use); From 56b1dd7e0981cb09f9ef0120eb1a219f6fb99673 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 18 Sep 2017 13:27:20 +0300 Subject: [PATCH 483/569] Adopt JIT for property access changes in master branch --- ext/opcache/jit/zend_jit_x86.dasc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 4201bbe061b67..5de7b103a0617 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -9001,6 +9001,8 @@ static int zend_jit_recv_init(dasm_State **Dst, const zend_op *opline, zend_op_a return 1; } +#define ZEND_WRONG_PROPERTY_OFFSET 0 + static uint32_t zend_get_known_property_offset(zend_class_entry *ce, zend_string *member, zend_bool on_this, zend_string *filename) { zend_property_info *info; @@ -9120,8 +9122,8 @@ static int zend_jit_fetch_obj_read(dasm_State **Dst, zend_op *opline, zend_op_ar | cmp r2, aword [FCARG1a + offsetof(zend_object, ce)] | jne >5 | mov r0, aword [r0 + Z_CACHE_SLOT_P(member) + sizeof(void*)] - | cmp eax, (uint32_t)ZEND_DYNAMIC_PROPERTY_OFFSET - | je >8 // dynamic property + | test eax, eax + | jl >8 // dynamic property | mov edx, dword [FCARG1a + r0 + 8] | IF_TYPE dl, IS_UNDEF, >5 | add FCARG1a, r0 From 79b7598ade68f9233cdafb95a1aee2561ee9764d Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 20 Sep 2017 02:28:58 +0300 Subject: [PATCH 484/569] JIT support for new array creation API. --- ext/opcache/jit/zend_jit_x86.dasc | 134 +++++++++++------------------- 1 file changed, 49 insertions(+), 85 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 5de7b103a0617..bba395a998a5a 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -4358,42 +4358,33 @@ static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, zend_op_ | CMP_ZVAL_TYPE op1_addr, IS_FALSE | jg >7 } - | // ZVAL_NEW_ARR(container); - | // zend_hash_init(Z_ARRVAL_P(container), 8, NULL, ZVAL_PTR_DTOR, 0); + | // ZVAL_ARR(container, zend_new_array(8)); if (Z_REG(op1_addr) != ZREG_FP) { | mov [r4], Ra(Z_REG(op1_addr)) // save } if (ZEND_DEBUG) { const char *filename = op_array->filename ? op_array->filename->val : NULL; |.if X64 - | LOAD_ADDR CARG3, filename - | xor CARG4d, opline->lineno + | mov CARG3d, opline->lineno |.else + | sub r4, 12 | push opline->lineno - | push filename |.endif - } else { + | LOAD_ADDR FCARG2a, filename + } + | mov FCARG1d, 8 + | EXT_CALL _zend_new_array, r0 + if (ZEND_DEBUG) { |.if not X64 - | sub r4, 8 + | add r4, 12 |.endif } - |.if X64 - | LOAD_ADDR CARG2, 8 - if (Z_REG(op1_addr) != ZREG_FCARG1a) { - | LOAD_ZVAL_ADDR FCARG1a, op1_addr - } - |.else - | push 8 - | PUSH_ZVAL_ADDR op1_addr, r0 - |.endif - | EXT_CALL _array_init, r0 - |.if not X64 - | add r4, 16 - |.endif if (Z_REG(op1_addr) != ZREG_FP) { | mov Ra(Z_REG(op1_addr)), [r4] // restore } - | GET_ZVAL_LVAL ZREG_FCARG1a, op1_addr + | SET_ZVAL_LVAL op1_addr, r0 + | SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX + | mov FCARG1a, r0 } if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { @@ -4465,42 +4456,33 @@ static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, zend_op_ | CMP_ZVAL_TYPE op1_addr, IS_FALSE | jg >2 } - | // ZVAL_NEW_ARR(container); - | // zend_hash_init(Z_ARRVAL_P(container), 8, NULL, ZVAL_PTR_DTOR, 0); + | // ZVAL_ARR(container, zend_new_array(8)); if (Z_REG(op1_addr) != ZREG_FP) { | mov [r4], Ra(Z_REG(op1_addr)) // save } if (ZEND_DEBUG) { const char *filename = op_array->filename ? op_array->filename->val : NULL; |.if X64 - | LOAD_ADDR CARG3, filename - | xor CARG4d, opline->lineno + | mov CARG3d, opline->lineno |.else + | sub r4, 12 | push opline->lineno - | push filename |.endif - } else { + | LOAD_ADDR FCARG2a, filename + } + | mov FCARG1d, 8 + | EXT_CALL _zend_new_array, r0 + if (ZEND_DEBUG) { |.if not X64 - | sub r4, 8 + | add r4, 12 |.endif } - |.if X64 - | LOAD_ADDR CARG2, 8 - if (Z_REG(op1_addr) != ZREG_FCARG1a) { - | LOAD_ZVAL_ADDR FCARG1a, op1_addr - } - |.else - | push 8 - | PUSH_ZVAL_ADDR op1_addr, r0 - |.endif - | EXT_CALL _array_init, r0 - |.if not X64 - | add r4, 16 - |.endif if (Z_REG(op1_addr) != ZREG_FP) { | mov Ra(Z_REG(op1_addr)), [r4] // restore } - | GET_ZVAL_LVAL ZREG_FCARG1a, op1_addr + | SET_ZVAL_LVAL op1_addr, r0 + | SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX + | mov FCARG1a, r0 | // ZEND_VM_C_GOTO(assign_dim_op_new_array); | jmp <6 |2: @@ -4624,42 +4606,33 @@ static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, zend_ | EXT_CALL zend_jit_undefined_op_helper, r0 |1: } - | // ZVAL_NEW_ARR(container); - | // zend_hash_init(Z_ARRVAL_P(container), 8, NULL, ZVAL_PTR_DTOR, 0); + | // ZVAL_ARR(container, zend_new_array(8)); if (Z_REG(op1_addr) != ZREG_FP) { | mov [r4], Ra(Z_REG(op1_addr)) // save } if (ZEND_DEBUG) { const char *filename = op_array->filename ? op_array->filename->val : NULL; |.if X64 - | LOAD_ADDR CARG3, filename - | xor CARG4d, opline->lineno + | mov CARG3d, opline->lineno |.else + | sub r4, 12 | push opline->lineno - | push filename |.endif - } else { + | LOAD_ADDR FCARG2a, filename + } + | mov FCARG1d, 8 + | EXT_CALL _zend_new_array, r0 + if (ZEND_DEBUG) { |.if not X64 - | sub r4, 8 + | add r4, 12 |.endif - } - |.if X64 - | LOAD_ADDR CARG2, 8 - if (Z_REG(op1_addr) != ZREG_FCARG1a) { - | LOAD_ZVAL_ADDR FCARG1a, op1_addr - } - |.else - | push 8 - | PUSH_ZVAL_ADDR op1_addr, r0 - |.endif - | EXT_CALL _array_init, r0 - |.if not X64 - | add r4, 16 - |.endif + } if (Z_REG(op1_addr) != ZREG_FP) { | mov Ra(Z_REG(op1_addr)), [r4] // restore } - | GET_ZVAL_LVAL ZREG_FCARG1a, op1_addr + | SET_ZVAL_LVAL op1_addr, r0 + | SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX + | mov FCARG1a, r0 } if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { @@ -4757,42 +4730,33 @@ static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, zend_ | EXT_CALL zend_jit_undefined_op_helper, r0 |1: } - | // ZVAL_NEW_ARR(container); - | // zend_hash_init(Z_ARRVAL_P(container), 8, NULL, ZVAL_PTR_DTOR, 0); + | // ZVAL_ARR(container, zend_new_array(8)); if (Z_REG(op1_addr) != ZREG_FP) { | mov [r4], Ra(Z_REG(op1_addr)) // save } if (ZEND_DEBUG) { const char *filename = op_array->filename ? op_array->filename->val : NULL; |.if X64 - | LOAD_ADDR CARG3, filename - | xor CARG4d, opline->lineno + | mov CARG3d, opline->lineno |.else + | sub r4, 12 | push opline->lineno - | push filename |.endif - } else { + | LOAD_ADDR FCARG2a, filename + } + | mov FCARG1d, 8 + | EXT_CALL _zend_new_array, r0 + if (ZEND_DEBUG) { |.if not X64 - | sub r4, 8 + | add r4, 12 |.endif } - |.if X64 - | LOAD_ADDR CARG2, 8 - if (Z_REG(op1_addr) != ZREG_FCARG1a) { - | LOAD_ZVAL_ADDR FCARG1a, op1_addr - } - |.else - | push 8 - | PUSH_ZVAL_ADDR op1_addr, r0 - |.endif - | EXT_CALL _array_init, r0 - |.if not X64 - | add r4, 16 - |.endif if (Z_REG(op1_addr) != ZREG_FP) { | mov Ra(Z_REG(op1_addr)), [r4] // restore } - | GET_ZVAL_LVAL ZREG_FCARG1a, op1_addr + | SET_ZVAL_LVAL op1_addr, r0 + | SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX + | mov FCARG1a, r0 | // ZEND_VM_C_GOTO(assign_dim_op_new_array); | jmp <6 |2: From fd7abe3a7f81de0cba80e906f77b0075c59d1efb Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 26 Sep 2017 16:06:50 +0300 Subject: [PATCH 485/569] Generate better code for functions with type hints --- ext/opcache/jit/zend_jit_x86.dasc | 81 ++++++++++++++++++++++++++++--- 1 file changed, 74 insertions(+), 7 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index bba395a998a5a..841aa99682189 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -1924,6 +1924,7 @@ static int zend_jit_not_obj_stub(dasm_State **Dst) static int zend_jit_negative_shift_stub(dasm_State **Dst) { |->negative_shift: + | mov aword EX->opline, IP |.if X64 | LOAD_ADDR CARG1, zend_ce_arithmetic_error | LOAD_ADDR CARG2, "Bit shift by negative number" @@ -1942,6 +1943,7 @@ static int zend_jit_negative_shift_stub(dasm_State **Dst) static int zend_jit_mod_by_zero_stub(dasm_State **Dst) { |->mod_by_zero: + | mov aword EX->opline, IP |.if X64 | LOAD_ADDR CARG1, zend_ce_division_by_zero_error | LOAD_ADDR CARG2, "Modulo by zero" @@ -3387,6 +3389,9 @@ static int zend_jit_long_math_helper(dasm_State **Dst, if (EXPECTED(op2_lval > 0)) { | xor Ra(result_reg), Ra(result_reg) } else { + if (!zend_jit_set_valid_ip(Dst, opline)) { + return 0; + } | jmp ->negative_shift } } else { @@ -3401,6 +3406,9 @@ static int zend_jit_long_math_helper(dasm_State **Dst, !ssa->var_info[op2_ssa_var].has_range || ssa->var_info[op2_ssa_var].range.min < 0 || ssa->var_info[op2_ssa_var].range.max >= SIZEOF_ZEND_LONG * 8) { + if (!zend_jit_set_valid_ip(Dst, opline)) { + return 0; + } | cmp r1, (SIZEOF_ZEND_LONG*8) | jae >1 |.cold_code @@ -3423,6 +3431,9 @@ static int zend_jit_long_math_helper(dasm_State **Dst, if (EXPECTED(op2_lval > 0)) { | sar Ra(result_reg), (SIZEOF_ZEND_LONG * 8) - 1 } else { + if (!zend_jit_set_valid_ip(Dst, opline)) { + return 0; + } | jmp ->negative_shift } } else { @@ -3436,6 +3447,9 @@ static int zend_jit_long_math_helper(dasm_State **Dst, !ssa->var_info[op2_ssa_var].has_range || ssa->var_info[op2_ssa_var].range.min < 0 || ssa->var_info[op2_ssa_var].range.max >= SIZEOF_ZEND_LONG * 8) { + if (!zend_jit_set_valid_ip(Dst, opline)) { + return 0; + } | cmp r1, (SIZEOF_ZEND_LONG*8) | jae >1 |.cold_code @@ -3453,6 +3467,9 @@ static int zend_jit_long_math_helper(dasm_State **Dst, zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr)); if (op2_lval == 0) { + if (!zend_jit_set_valid_ip(Dst, opline)) { + return 0; + } | jmp ->mod_by_zero } else if (op2_lval == -1) { | xor Ra(result_reg), Ra(result_reg) @@ -3472,6 +3489,9 @@ static int zend_jit_long_math_helper(dasm_State **Dst, !ssa->var_info[op2_ssa_var].has_range || (ssa->var_info[op2_ssa_var].range.min <= 0 && ssa->var_info[op2_ssa_var].range.max >= 0)) { + if (!zend_jit_set_valid_ip(Dst, opline)) { + return 0; + } if (Z_MODE(op2_addr) == IS_MEM_ZVAL) { | cmp aword [Ra(Z_REG(op2_addr))+Z_OFFSET(op2_addr)], 0 } else if (Z_MODE(op2_addr) == IS_REG) { @@ -6894,6 +6914,39 @@ static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, uint32_t return 1; } +static uint32_t skip_valid_arguments(zend_op_array *op_array, zend_ssa *ssa, zend_call_info *call_info) +{ + uint32_t num_args = 0; + zend_function *func = call_info->callee_func; + + while (num_args < call_info->num_args) { + zend_arg_info *arg_info = func->op_array.arg_info + num_args; + + if (ZEND_TYPE_IS_SET(arg_info->type)) { + if (!ZEND_TYPE_IS_CLASS(arg_info->type)) { + unsigned char code = ZEND_TYPE_CODE(arg_info->type); + uint32_t info = _ssa_op1_info(op_array, ssa, call_info->arg_info[num_args].opline); + + if (code == _IS_BOOL) { + if (info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_FALSE|MAY_BE_TRUE))) { + break; + } + } else if (code <= IS_RESOURCE) { + if (info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (1 << code))) { + break; + } + } else { + break; + } + } else { + break; + } + } + num_args++; + } + return num_args; +} + static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, int call_level) { zend_func_info *info = ZEND_FUNC_INFO(op_array); @@ -6923,8 +6976,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar } else if (func->type == ZEND_USER_FUNCTION) { if (call_info->num_args > func->op_array.num_args || (opline-1)->opcode == ZEND_SEND_UNPACK || - (opline-1)->opcode == ZEND_SEND_ARRAY || - (func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) != 0) { + (opline-1)->opcode == ZEND_SEND_ARRAY) { goto fallback; } } else if (func->type == ZEND_INTERNAL_FUNCTION) { @@ -7058,15 +7110,22 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar | // opline = op_array->opcodes; if (func) { + uint32_t num_args; + + if (func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) { + num_args = skip_valid_arguments(op_array, ssa, call_info); + } else { + num_args = call_info->num_args; + } if (func && zend_accel_in_shm(func->op_array.opcodes)) { - | LOAD_ADDR IP, (func->op_array.opcodes + call_info->num_args) + | LOAD_ADDR IP, (func->op_array.opcodes + num_args) } else { if (func) { | mov r0, EX->func } | mov IP, aword [r0 + offsetof(zend_op_array, opcodes)] - if (call_info->num_args) { - | add IP, (call_info->num_args * sizeof(zend_op)) + if (num_args) { + | add IP, (num_args * sizeof(zend_op)) } } } else { @@ -7118,15 +7177,23 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar if (func && op_array == &func->op_array) { /* recursive call */ + uint32_t num_args; + + if (func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) { + num_args = skip_valid_arguments(op_array, ssa, call_info); + } else { + num_args = call_info->num_args; + } + #ifdef CONTEXT_THREADED_JIT - | call =>(call_info->num_args+ssa->cfg.blocks_count) + | call =>(num_args+ssa->cfg.blocks_count) | MEM_OP2_1 cmp, aword, &EG(exception), 0, r0 | jne ->exception_handler if (!func) { | jmp >9 } #else - | jmp =>call_info->num_args + | jmp =>num_args #endif } else { #ifdef CONTEXT_THREADED_JIT From e2bebba9f54036328570a9351761aaa87fd1c4be Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 26 Sep 2017 22:12:06 +0300 Subject: [PATCH 486/569] JIT for RECV handler --- ext/opcache/jit/zend_jit.c | 5 ++ ext/opcache/jit/zend_jit_helpers.c | 13 ++- ext/opcache/jit/zend_jit_x86.dasc | 124 +++++++++++++++++++++++++++++ 3 files changed, 138 insertions(+), 4 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index a785a9c44a188..eab649e642a55 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -2364,6 +2364,11 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa, const zend_op *rt_op goto jit_failure; } goto done; + case ZEND_RECV: + if (!zend_jit_recv(&dasm_state, opline, op_array, ssa)) { + goto jit_failure; + } + goto done; case ZEND_RECV_INIT: if (!zend_jit_recv_init(&dasm_state, opline, op_array, (opline + 1)->opcode != ZEND_RECV_INIT, ssa)) { goto jit_failure; diff --git a/ext/opcache/jit/zend_jit_helpers.c b/ext/opcache/jit/zend_jit_helpers.c index e693b1707efc0..f40d44f106b52 100644 --- a/ext/opcache/jit/zend_jit_helpers.c +++ b/ext/opcache/jit/zend_jit_helpers.c @@ -1822,6 +1822,10 @@ static ZEND_COLD void zend_verify_type_error_common( *need_msg = "be iterable"; *need_kind = ""; break; + case IS_OBJECT: + *need_msg = "be an object"; + *need_kind = ""; + break; default: *need_msg = "be of the type "; *need_kind = zend_get_type_by_const(ZEND_TYPE_CODE(arg_info->type)); @@ -1857,7 +1861,7 @@ static ZEND_COLD void zend_verify_arg_error( const char *fname, *fsep, *fclass; const char *need_msg, *need_kind, *need_or_null, *given_msg, *given_kind; - if (value) { + if (value && !Z_ISUNDEF_P(value)) { zend_verify_type_error_common( zf, arg_info, ce, value, &fname, &fsep, &fclass, &need_msg, &need_kind, &need_or_null, &given_msg, &given_kind); @@ -1874,7 +1878,7 @@ static ZEND_COLD void zend_verify_arg_error( zend_type_error("Argument %d passed to %s%s%s() must %s%s%s, %s%s given", arg_num, fclass, fsep, fname, need_msg, need_kind, need_or_null, given_msg, given_kind); } } else { - zend_missing_arg_error(ptr); + zend_missing_arg_error(EG(current_execute_data)); } } @@ -1928,8 +1932,9 @@ static void ZEND_FASTCALL zend_jit_verify_arg_slow(zval *arg, zend_op_array *op_ } else if (ZEND_TYPE_CODE(arg_info->type) == _IS_BOOL && EXPECTED(Z_TYPE_P(arg) == IS_FALSE || Z_TYPE_P(arg) == IS_TRUE)) { return; - } else { - if (zend_verify_scalar_type_hint(ZEND_TYPE_CODE(arg_info->type), arg, ZEND_RET_USES_STRICT_TYPES()) == 0) { + } else if (ZEND_TYPE_CODE(arg_info->type) != Z_TYPE_P(arg)) { + if (Z_ISUNDEF_P(arg) || + zend_verify_scalar_type_hint(ZEND_TYPE_CODE(arg_info->type), arg, ZEND_ARG_USES_STRICT_TYPES()) == 0) { goto err; } } diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 841aa99682189..99d8029abd79b 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -8914,6 +8914,125 @@ static int zend_jit_bind_global(dasm_State **Dst, const zend_op *opline, zend_op return 1; } +static int zend_jit_recv(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) +{ + uint32_t arg_num = opline->op1.num; + + if (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) { + zend_arg_info *arg_info = NULL; + + if (EXPECTED(arg_num <= op_array->num_args)) { + arg_info = &op_array->arg_info[arg_num-1]; + } else if (UNEXPECTED(op_array->fn_flags & ZEND_ACC_VARIADIC)) { + arg_info = &op_array->arg_info[op_array->num_args]; + } + if (arg_info) { + zend_type type = arg_info->type; + + if (ZEND_TYPE_IS_SET(type)) { + // Type check + zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1); + + | LOAD_ZVAL_ADDR r0, res_addr + if (arg_info->pass_by_reference) { + | GET_Z_PTR r0, r0 + | add r0, offsetof(zend_reference, val) + } + if (!ZEND_TYPE_IS_CLASS(type)) { + unsigned char code = ZEND_TYPE_CODE(type); + + if (type == _IS_BOOL) { + | cmp byte [r0 + 8], IS_FALSE + | je >1 + | cmp byte [r0 + 8], IS_TRUE + | jne >8 + |1: + } else { + | cmp byte [r0 + 8], code + | jne >8 + } + } else { + | SAVE_VALID_OPLINE opline + | cmp byte [r0 + 8], IS_OBJECT + | jne >9 + | mov FCARG1a, r0 + | mov r0, EX->run_time_cache + | add r0, opline->op2.num + | LOAD_ADDR FCARG2a, (ptrdiff_t)op_array + |.if X64 + | mov CARG3, arg_num + | LOAD_ADDR CARG4, (ptrdiff_t)arg_info + | mov CARG5, r0 + |.else + | push r0 + | push (ptrdiff_t)arg_info + | push arg_num + |.endif + | EXT_CALL zend_jit_verify_arg_object, r0 + if (!zend_jit_check_exception(Dst)) { + return 0; + } + } + + |.cold_code + |8: + | SAVE_VALID_OPLINE opline + |9: + | mov FCARG1a, r0 + | mov r0, EX->run_time_cache + | add r0, opline->op2.num + | LOAD_ADDR FCARG2a, (ptrdiff_t)op_array + |.if X64 + | mov CARG3, arg_num + | LOAD_ADDR CARG4, (ptrdiff_t)arg_info + | mov CARG5, r0 + | xor CARG6, CARG6 + |.else + | push 0 + | push r0 + | push (ptrdiff_t)arg_info + | push arg_num + |.endif + | EXT_CALL zend_jit_verify_arg_slow, r0 + if (!zend_jit_check_exception(Dst)) { + return 0; + } + | jmp >1 + |.code + |1: + + if ((opline+1)->opcode != ZEND_RECV && (opline+1)->opcode != ZEND_RECV_INIT) { + last_valid_opline = NULL; + if (!zend_jit_set_valid_ip(Dst, opline + 1)) { + return 0; + } + } + + return 1; + } + } + } + + | cmp dword EX->This.u2.num_args, arg_num + | jb >1 + |.cold_code + |1: + | SAVE_VALID_OPLINE opline + | mov FCARG1a, FP + | EXT_CALL zend_missing_arg_error, r0 + | jmp ->exception_handler + |.code + + if ((opline+1)->opcode != ZEND_RECV && (opline+1)->opcode != ZEND_RECV_INIT) { + last_valid_opline = NULL; + if (!zend_jit_set_valid_ip(Dst, opline + 1)) { + return 0; + } + } + + return 1; +} + static int zend_jit_recv_init(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_bool is_last, zend_ssa *ssa) { zend_arg_info *arg_info; @@ -9004,6 +9123,11 @@ static int zend_jit_recv_init(dasm_State **Dst, const zend_op *opline, zend_op_a |7: | ZVAL_PTR_DTOR res_addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN, 1, 0, 0, op_array->filename, opline | SET_ZVAL_TYPE_INFO res_addr, IS_UNDEF + if (zend_may_throw(opline, op_array, ssa)) { + if (!zend_jit_check_exception(Dst)) { + return 0; + } + } | jmp <5 } if (has_slow & 2) { From cdf36215ef50d2c9fca8ae6ad86ae45a3f8f0796 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 27 Sep 2017 12:51:11 +0300 Subject: [PATCH 487/569] Avoid useless stores --- ext/opcache/jit/zend_jit_x86.dasc | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 99d8029abd79b..5aa529241760e 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -1226,9 +1226,6 @@ static void* dasm_labels[zend_lb_MAX]; // zval should be in FCARG1a |.macro ZVAL_DTOR_FUNC, var_info, filename, opline // arg1 must be in FCARG1a || do { -|| if (opline) { -| SAVE_VALID_OPLINE opline -|| } || if (has_concrete_type((var_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { || zend_uchar type = concrete_type((var_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)); || if (type == IS_STRING && !ZEND_DEBUG) { @@ -1241,9 +1238,15 @@ static void* dasm_labels[zend_lb_MAX]; |2: || break; || } else if (type == IS_ARRAY) { +|| if (opline) { +| SAVE_VALID_OPLINE opline +|| } | EXT_CALL zend_array_destroy, r0 || break; || } else if (type == IS_OBJECT) { +|| if (opline) { +| SAVE_VALID_OPLINE opline +|| } | .if X64 | EXT_CALL zend_objects_store_del, r0 | .else @@ -1255,6 +1258,9 @@ static void* dasm_labels[zend_lb_MAX]; || break; || } || } +|| if (opline) { +| SAVE_VALID_OPLINE opline +|| } || if (ZEND_DEBUG) { || uint32_t lineno = opline ? ((zend_op*)opline)->lineno : 0; || if (filename) { @@ -8381,7 +8387,6 @@ static int zend_jit_return(dasm_State **Dst, const zend_op *opline, zend_op_arra } } | //SAVE_OPLINE() - | SAVE_VALID_OPLINE opline | ZVAL_DTOR_FUNC op1_info, op_array->filename, opline | //????mov r1, EX->return_value // reload ??? } From 01126618523a9ab88a3f731cff8b739d4377c53e Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 3 Oct 2017 14:13:34 +0300 Subject: [PATCH 488/569] Removed experemental code --- ext/opcache/jit/zend_jit.c | 39 --------------------------- ext/opcache/jit/zend_jit_internal.h | 2 -- ext/opcache/jit/zend_jit_vm_helpers.c | 29 -------------------- ext/opcache/jit/zend_jit_x86.dasc | 17 ------------ 4 files changed, 87 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index eab649e642a55..8a6cef9af71fb 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -36,8 +36,6 @@ //#define CONTEXT_THREADED_JIT #define PREFER_MAP_32BIT -//#define ZEND_JIT_RECORD -//#define ZEND_JIT_FILTER #define ZEND_JIT_USE_RC_INFERENCE #ifdef ZEND_JIT_USE_RC_INFERENCE @@ -1970,35 +1968,6 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa, const zend_op *rt_op void *checkpoint = NULL; zend_lifetime_interval **ra = NULL; -#ifdef ZEND_JIT_FILTER - const char *names[] = { -#include "zend_jit_filter.c" - }; - - for (i = 0; i < sizeof(names)/sizeof(names[0]); i++) { - const char *name = names[i]; - const char *sep = strstr(name, "::"); - - if (sep) { - if (op_array->scope && - op_array->scope->name && - op_array->function_name && - strncmp(ZSTR_VAL(op_array->scope->name), name, sep-name) == 0 && - strcmp(ZSTR_VAL(op_array->function_name), sep+2) == 0) { - goto pass; - } - } else { - if (op_array->scope == NULL && - op_array->function_name && - strcmp(ZSTR_VAL(op_array->function_name), name) == 0) { - goto pass; - } - } - } - return SUCCESS; -pass: -#endif - if (zend_jit_reg_alloc) { checkpoint = zend_arena_checkpoint(CG(arena)); ra = zend_jit_allocate_registers(op_array, ssa); @@ -2064,11 +2033,6 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa, const zend_op *rt_op zend_jit_label(&dasm_state, ssa->cfg.blocks_count + b); zend_jit_prologue(&dasm_state); } -#ifdef ZEND_JIT_RECORD - if (opline->opcode != ZEND_RECV && opline->opcode != ZEND_RECV_INIT) { - zend_jit_func_header(&dasm_state, op_array); - } -#endif } if (ssa->cfg.blocks[b].len == 1 && @@ -2134,9 +2098,6 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa, const zend_op *rt_op } } if (ssa->cfg.blocks[b].flags & ZEND_BB_LOOP_HEADER) { -#ifdef ZEND_JIT_RECORD - zend_jit_loop_header(&dasm_state, op_array, op_array->opcodes + ssa->cfg.blocks[b].start); -#endif if (!zend_jit_check_timeout(&dasm_state, op_array->opcodes + ssa->cfg.blocks[b].start)) { goto jit_failure; } diff --git a/ext/opcache/jit/zend_jit_internal.h b/ext/opcache/jit/zend_jit_internal.h index 47acf61a287cc..dda0135bb6f58 100644 --- a/ext/opcache/jit/zend_jit_internal.h +++ b/ext/opcache/jit/zend_jit_internal.h @@ -43,8 +43,6 @@ void ZEND_FASTCALL zend_jit_leave_nested_func_helper(uint32_t call_info); void ZEND_FASTCALL zend_jit_leave_top_func_helper(uint32_t call_info); void ZEND_FASTCALL zend_jit_copy_extra_args_helper(void); void ZEND_FASTCALL zend_jit_deprecated_or_abstract_helper(void); -void ZEND_FASTCALL zend_jit_func_header_helper(void); -void ZEND_FASTCALL zend_jit_loop_header_helper(void); void ZEND_FASTCALL zend_jit_profile_helper(void); void ZEND_FASTCALL zend_jit_func_counter_helper(void); void ZEND_FASTCALL zend_jit_loop_counter_helper(void); diff --git a/ext/opcache/jit/zend_jit_vm_helpers.c b/ext/opcache/jit/zend_jit_vm_helpers.c index ad25df2a9019c..5f49c4890a0d1 100644 --- a/ext/opcache/jit/zend_jit_vm_helpers.c +++ b/ext/opcache/jit/zend_jit_vm_helpers.c @@ -139,35 +139,6 @@ void ZEND_FASTCALL zend_jit_deprecated_or_abstract_helper(void) } } -/* The recorded log may be postprocessed to identify the hot functions and - * loops. - * - * To get the top functions: - * sed 's/^\(.*\), (.:\(.*\):.*$/\1,\2/' | sort | uniq -c | sort -nr | \ - * head -n25 | sed 's/^\s*[0-9]*\s*\(.*\),.*$/"\1",/' > zend_jit_filter.c - * - */ - -void ZEND_FASTCALL zend_jit_func_header_helper(void) -{ - fprintf(stderr, "%s%s%s, (F:%s:%d)\n", - EX(func)->op_array.scope ? ZSTR_VAL(EX(func)->op_array.scope->name) : "", - EX(func)->op_array.scope ? "::" : "", - EX(func)->op_array.function_name ? ZSTR_VAL(EX(func)->op_array.function_name) : "", - ZSTR_VAL(EX(func)->op_array.filename), - EX(func)->op_array.line_start); -} - -void ZEND_FASTCALL zend_jit_loop_header_helper(void) -{ - fprintf(stderr, "%s%s%s, (L:%s:%d)\n", - EX(func)->op_array.scope ? ZSTR_VAL(EX(func)->op_array.scope->name) : "", - EX(func)->op_array.scope ? "::" : "", - EX(func)->op_array.function_name ? ZSTR_VAL(EX(func)->op_array.function_name) : "", - ZSTR_VAL(EX(func)->op_array.filename), - opline->lineno); -} - void ZEND_FASTCALL zend_jit_profile_helper(void) { zend_op_array *op_array = (zend_op_array*)EX(func); diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 5aa529241760e..510d3543b7045 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -9913,23 +9913,6 @@ static zend_regset zend_jit_get_scratch_regset(zend_op_array *op_array, zend_ssa return regset; } -#ifdef ZEND_JIT_RECORD -static int zend_jit_func_header(dasm_State **Dst, zend_op_array *op_array) -{ - | EXT_CALL zend_jit_func_header_helper, r0 - return 1; -} - -static int zend_jit_loop_header(dasm_State **Dst, zend_op_array *op_array, zend_op *opline) -{ - if (!zend_jit_set_valid_ip(Dst, opline)) { - return 0; - } - | EXT_CALL zend_jit_loop_header_helper, r0 - return 1; -} -#endif - /* * Local variables: * tab-width: 4 From 7f1a2b11c15afad1c9a9214e2c850cb6d7512767 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 4 Oct 2017 18:58:21 +0300 Subject: [PATCH 489/569] Fixed support for new run-time constant operands addressing --- ext/opcache/jit/zend_jit_x86.dasc | 60 ++++++++++++------------------- ext/opcache/jit/zend_jit_x86.h | 2 +- 2 files changed, 24 insertions(+), 38 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 510d3543b7045..fcf250e83b88f 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -1769,8 +1769,8 @@ static int zend_jit_undefined_offset_stub(dasm_State **Dst) | SET_Z_TYPE_INFO FP + r1, IS_NULL | jne >2 |.if X64 - | movsxd r0, dword OP:r0->op2.constant - | add r0, aword EX->literals + | movsxd r1, dword OP:r0->op2.constant + | add r0, r1 |.else | mov r0, aword OP:r0->op2.zv |.endif @@ -1829,8 +1829,8 @@ static int zend_jit_undefined_index_stub(dasm_State **Dst) | SET_Z_TYPE_INFO FP + r1, IS_NULL | jne >2 |.if X64 - | movsxd r0, dword OP:r0->op2.constant - | add r0, aword EX->literals + | movsxd r1, dword OP:r0->op2.constant + | add r0, r1 |.else | mov r0, aword OP:r0->op2.zv |.endif @@ -4361,7 +4361,7 @@ static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, zend_op_ op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, NULL, -1); op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2, opline, NULL, -1); - op3_addr = zend_jit_decode_op(op_array, (opline+1)->op1_type, (opline+1)->op1, opline, NULL, -1); + op3_addr = zend_jit_decode_op(op_array, (opline+1)->op1_type, (opline+1)->op1, opline+1, NULL, -1); if (opline->result_type == IS_UNUSED) { res_addr = 0; } else { @@ -4605,7 +4605,7 @@ static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, zend_ op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, NULL, -1); op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2, opline, NULL, -1); - op3_addr = zend_jit_decode_op(op_array, (opline+1)->op1_type, (opline+1)->op1, opline, NULL, -1); + op3_addr = zend_jit_decode_op(op_array, (opline+1)->op1_type, (opline+1)->op1, opline+1, NULL, -1); if (op1_info & MAY_BE_REF) { | LOAD_ZVAL_ADDR FCARG1a, op1_addr @@ -6870,7 +6870,7 @@ static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, uint32_t /* recursive call */ | mov r0, EX->func } else { - zval *zv = RT_CONSTANT(op_array, opline->op2); + zval *zv = RT_CONSTANT(opline, opline->op2); | // if (CACHED_PTR(Z_CACHE_SLOT_P(fname))) | mov r0, EX->run_time_cache @@ -7096,20 +7096,6 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar } } - | //EX_LOAD_LITERALS(op_array); - |.if X64 - if (func && zend_accel_in_shm(func->op_array.literals)) { - | LOAD_ADDR r2, func->op_array.literals - | mov EX:RX->literals, r2 - } else { - if (func) { - | mov r0, EX:RX->func - } - | mov r2, aword [r0 + offsetof(zend_op_array, literals)] - | mov EX:RX->literals, r2 - } - |.endif - | // EG(current_execute_data) = execute_data; | MEM_OP2_1 mov, aword, &EG(current_execute_data), RX, r1 | mov FP, RX @@ -7440,7 +7426,7 @@ static int zend_jit_new(dasm_State **Dst, const zend_op *opline, int *opnum, zen } } else { if (opline->op1_type == IS_CONST) { - zval *zv = RT_CONSTANT(op_array, opline->op1); + zval *zv = RT_CONSTANT(opline, opline->op1); if (Z_TYPE_P(zv) == IS_STRING) { zval *lc = zv + 1; ce = (zend_class_entry*)zend_hash_find_ptr(EG(class_table), Z_STR_P(lc)); @@ -7504,7 +7490,7 @@ static int zend_jit_send_val(dasm_State **Dst, const zend_op *opline, zend_op_ar arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var); if (opline->op1_type == IS_CONST) { - zval *zv = RT_CONSTANT(op_array, opline->op1); + zval *zv = RT_CONSTANT(opline, opline->op1); | ZVAL_COPY_CONST arg_addr, -1, zv, r0 if (Z_REFCOUNTED_P(zv)) { @@ -7874,7 +7860,7 @@ static int zend_jit_defined(dasm_State **Dst, const zend_op *opline, int b, int zend_bool smart_branch = 0; uint32_t defined_label = (uint32_t)-1; uint32_t undefined_label = (uint32_t)-1; - zval *zv = RT_CONSTANT(op_array, opline->op1); + zval *zv = RT_CONSTANT(opline, opline->op1); zend_jit_addr res_addr; if (((opline+1)->opcode == ZEND_JMPZ || @@ -8405,7 +8391,7 @@ static int zend_jit_return(dasm_State **Dst, const zend_op *opline, zend_op_arra } if (opline->op1_type == IS_CONST) { - zval *zv = RT_CONSTANT(op_array, opline->op1); + zval *zv = RT_CONSTANT(opline, opline->op1); | ZVAL_COPY_CONST ret_addr, -1, zv, r0 if (Z_REFCOUNTED_P(zv)) { | ADDREF_CONST zv, r0 @@ -8778,7 +8764,7 @@ static int zend_jit_bind_global(dasm_State **Dst, const zend_op *opline, zend_op { uint32_t op1_info; zend_jit_addr op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, NULL, -1); - zval *varname = RT_CONSTANT(op_array, opline->op2); + zval *varname = RT_CONSTANT(opline, opline->op2); if (!ssa->ops || !ssa->var_info) { op1_info = MAY_BE_ANY|MAY_BE_REF; @@ -9043,7 +9029,7 @@ static int zend_jit_recv_init(dasm_State **Dst, const zend_op *opline, zend_op_a zend_arg_info *arg_info; zend_bool has_slow = 0; uint32_t arg_num = opline->op1.num; - zval *zv = RT_CONSTANT(op_array, opline->op2); + zval *zv = RT_CONSTANT(opline, opline->op2); zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1); | cmp dword EX->This.u2.num_args, arg_num @@ -9230,7 +9216,7 @@ static int zend_jit_fetch_obj_read(dasm_State **Dst, zend_op *opline, zend_op_ar goto fallback; } - member = RT_CONSTANT(op_array, opline->op2); + member = RT_CONSTANT(opline, opline->op2); if (Z_TYPE_P(member) != IS_STRING || Z_STRVAL_P(member)[0] == '\0') { goto fallback; } @@ -9439,7 +9425,7 @@ static int zend_jit_free(dasm_State **Dst, const zend_op *opline, zend_op_array static int zend_jit_echo(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) { if (opline->op1_type == IS_CONST) { - zval *zv = RT_CONSTANT(op_array, opline->op1); + zval *zv = RT_CONSTANT(opline, opline->op1); if (Z_TYPE_P(zv) == IS_STRING) { size_t len = Z_STRLEN_P(zv); @@ -9629,11 +9615,11 @@ static zend_bool zend_jit_may_be_in_reg(zend_op_array *op_array, zend_ssa *ssa, return 1; } -static zend_bool zend_needs_extra_reg_for_const(zend_op_array *op_array, zend_uchar op_type, znode_op op) +static zend_bool zend_needs_extra_reg_for_const(zend_op_array *op_array, const zend_op *opline, zend_uchar op_type, znode_op op) { |.if X64 || if (op_type == IS_CONST) { -|| zval *zv = RT_CONSTANT(op_array, op); +|| zval *zv = RT_CONSTANT(opline, op); || if (Z_TYPE_P(zv) == IS_DOUBLE && Z_DVAL_P(zv) != 0 && !IS_32BIT(zv)) { || return 1; || } @@ -9784,8 +9770,8 @@ static zend_regset zend_jit_get_scratch_regset(zend_op_array *op_array, zend_ssa ZEND_REGSET_INCL(regset, ZREG_XMM0); } } - if (zend_needs_extra_reg_for_const(op_array, opline->op1_type, opline->op1) || - zend_needs_extra_reg_for_const(op_array, opline->op1_type, opline->op2)) { + if (zend_needs_extra_reg_for_const(op_array, opline, opline->op1_type, opline->op1) || + zend_needs_extra_reg_for_const(op_array, opline, opline->op1_type, opline->op2)) { ZEND_REGSET_INCL(regset, ZREG_R0); } } @@ -9827,8 +9813,8 @@ static zend_regset zend_jit_get_scratch_regset(zend_op_array *op_array, zend_ssa !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG))) { regset = ZEND_REGSET_EMPTY; if (opline->op2_type == IS_CONST && - Z_TYPE_P(RT_CONSTANT(op_array, opline->op2)) == IS_LONG && - zend_long_is_power_of_two(Z_LVAL_P(RT_CONSTANT(op_array, opline->op2)))) { + Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_LONG && + zend_long_is_power_of_two(Z_LVAL_P(RT_CONSTANT(opline, opline->op2)))) { if (ssa->ops[line].result_def != current_var && (ssa->ops[line].op1_use != current_var || !zend_ssa_is_last_use(op_array, ssa, current_var, opline-op_array->opcodes))) { ZEND_REGSET_INCL(regset, ZREG_R0); @@ -9870,8 +9856,8 @@ static zend_regset zend_jit_get_scratch_regset(zend_op_array *op_array, zend_ssa ZEND_REGSET_INCL(regset, ZREG_XMM0); } } - if (zend_needs_extra_reg_for_const(op_array, opline->op1_type, opline->op1) || - zend_needs_extra_reg_for_const(op_array, opline->op1_type, opline->op2)) { + if (zend_needs_extra_reg_for_const(op_array, opline, opline->op1_type, opline->op1) || + zend_needs_extra_reg_for_const(op_array, opline, opline->op1_type, opline->op2)) { ZEND_REGSET_INCL(regset, ZREG_R0); } } diff --git a/ext/opcache/jit/zend_jit_x86.h b/ext/opcache/jit/zend_jit_x86.h index 3fe55e9349430..3bd812c4fb915 100644 --- a/ext/opcache/jit/zend_jit_x86.h +++ b/ext/opcache/jit/zend_jit_x86.h @@ -192,7 +192,7 @@ static zend_always_inline zend_jit_addr zend_jit_decode_op(const zend_op_array * #if ZEND_USE_ABS_CONST_ADDR return ZEND_ADDR_CONST_ZVAL(op.zv); #else - return ZEND_ADDR_CONST_ZVAL(RT_CONSTANT(op_array, op)); + return ZEND_ADDR_CONST_ZVAL(RT_CONSTANT(opline, op)); #endif } else { if (ra && ssa_var >= 0 && ra[ssa_var]) { From c7e6a3cccd685b59db3ec90e3333ef011aeda0ea Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 4 Oct 2017 19:12:25 +0300 Subject: [PATCH 490/569] typo --- ext/opcache/jit/zend_jit_x86.dasc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index fcf250e83b88f..957b0dd069d80 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -2597,7 +2597,7 @@ static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, zend_op_arr | ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, MAY_BE_LONG, ZREG_R0, ZREG_R1 } if (op1_addr != op1_def_addr) { - if (Z_MODE(op1_addr) != IS_REG && Z_MODE(res_addr) == IS_REG && + if (Z_MODE(op1_addr) != IS_REG && Z_MODE(op1_def_addr) == IS_REG && (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC)) { if (!zend_jit_update_regs(Dst, res_addr, op1_def_addr, MAY_BE_LONG, ra[ssa->ops[opline - op_array->opcodes].op1_use])) { return 0; From 1561f831f03e8783362519942e0abd19649ab398 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Sun, 15 Oct 2017 20:48:36 +0800 Subject: [PATCH 491/569] Fixed infinite loop after enable RECV(typo?) --- ext/opcache/jit/zend_jit.c | 2 +- ext/opcache/tests/jit/recv_001.phpt | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 ext/opcache/tests/jit/recv_001.phpt diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 8a6cef9af71fb..e9251d612d8f0 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -2018,7 +2018,7 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa, const zend_op *rt_op } } else { if (opline != op_array->opcodes) { - zend_jit_jmp(&dasm_state, 1); + zend_jit_jmp(&dasm_state, b); } zend_jit_label(&dasm_state, ssa->cfg.blocks_count + b); for (i = 1; (opline+i)->opcode == ZEND_RECV_INIT; i++) { diff --git a/ext/opcache/tests/jit/recv_001.phpt b/ext/opcache/tests/jit/recv_001.phpt new file mode 100644 index 0000000000000..9794deac81b06 --- /dev/null +++ b/ext/opcache/tests/jit/recv_001.phpt @@ -0,0 +1,23 @@ +--TEST-- +JIT RECV: infinite loop +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +opcache.protect_memory=1 +;opcache.jit_debug=257 +--SKIPIF-- + +--FILE-- + Date: Tue, 17 Oct 2017 17:45:29 +0800 Subject: [PATCH 492/569] Fixed buggy ZEND_DEFINED handler --- ext/opcache/jit/zend_jit_x86.dasc | 18 ++++++++++-------- ext/opcache/tests/jit/const_001.phpt | 26 ++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 8 deletions(-) create mode 100644 ext/opcache/tests/jit/const_001.phpt diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 957b0dd069d80..bde32d0357d68 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -7902,20 +7902,22 @@ static int zend_jit_defined(dasm_State **Dst, const zend_op *opline, int b, int } else { | jz >3 } - } else { - res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1); - } - | mov r0, EX->run_time_cache - | mov r0, aword [r0 + Z_CACHE_SLOT_P(zv)] - if (smart_branch) { + | mov r1, EX->run_time_cache + | mov aword [r1 + Z_CACHE_SLOT_P(zv)], r0 if (defined_label != (uint32_t)-1) { | jmp =>defined_label } else { | jmp >3 } } else { - | add eax, IS_FALSE - | SET_ZVAL_TYPE_INFO res_addr, eax + res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1); + | jz >2 + | mov r1, EX->run_time_cache + | mov aword [r1 + Z_CACHE_SLOT_P(zv)], r0 + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE + | jmp >3 + |2: + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE | jmp >3 } |.code diff --git a/ext/opcache/tests/jit/const_001.phpt b/ext/opcache/tests/jit/const_001.phpt new file mode 100644 index 0000000000000..55e6878496ff0 --- /dev/null +++ b/ext/opcache/tests/jit/const_001.phpt @@ -0,0 +1,26 @@ +--TEST-- +JIT CONST: defined +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +opcache.protect_memory=1 +--SKIPIF-- + +--FILE-- + +--EXPECT-- +bool(false) +bool(true) From 4fe8666183a6a784abaccc7ce2e52c1d1fabc621 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Tue, 17 Oct 2017 18:04:32 +0800 Subject: [PATCH 493/569] Fixed segfault in X64 --- ext/opcache/jit/zend_jit_x86.dasc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index bde32d0357d68..725f28bce2211 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -8811,7 +8811,7 @@ static int zend_jit_bind_global(dasm_State **Dst, const zend_op *opline, zend_op | mov CARG1, r1 | LOAD_ADDR CARG2, Z_STRVAL_P(varname) | mov CARG3, Z_STRLEN_P(varname) - | call &memcmp + | EXT_CALL memcmp, r0 |.else | sub r4, 4 | push Z_STRLEN_P(varname) From 5d9e0f7768c68890aa58cb0b3b1cc31ebddaecb8 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 18 Oct 2017 10:34:40 +0300 Subject: [PATCH 494/569] Avoid code generation for dynamic property access if the property is know (but offset is not known) --- ext/opcache/jit/zend_jit_x86.dasc | 38 ++++++++++++++++++++++++++++--- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 725f28bce2211..edcd6eede2904 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -9203,12 +9203,41 @@ static uint32_t zend_get_known_property_offset(zend_class_entry *ce, zend_string return info->offset; } +static zend_bool zend_may_be_dynamic_property(zend_class_entry *ce, zend_string *member, zend_bool on_this, zend_string *filename) +{ + zend_property_info *info; + + if (!ce || (ce->ce_flags & ZEND_ACC_TRAIT)) { + return 1; + } + + if (ce->info.user.filename != filename) { + /* class declaration might be changed infdependently */ + return 1; + } + + info = (zend_property_info*)zend_hash_find_ptr(&ce->properties_info, member); + if (info == NULL || + info->offset == ZEND_WRONG_PROPERTY_OFFSET || + (info->flags & ZEND_ACC_STATIC)) { + return 1; + } + + if (!(info->flags & ZEND_ACC_PUBLIC) && + (!on_this || info->ce != ce)) { + return 1; + } + + return 0; +} + static int zend_jit_fetch_obj_read(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) { uint32_t op1_info; zend_class_entry *ce = NULL; zval *member; uint32_t offset; + zend_bool may_be_dynamic = 1; zend_jit_addr op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, NULL, -1); zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1); zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); @@ -9270,8 +9299,11 @@ static int zend_jit_fetch_obj_read(dasm_State **Dst, zend_op *opline, zend_op_ar | cmp r2, aword [FCARG1a + offsetof(zend_object, ce)] | jne >5 | mov r0, aword [r0 + Z_CACHE_SLOT_P(member) + sizeof(void*)] - | test eax, eax - | jl >8 // dynamic property + may_be_dynamic = zend_may_be_dynamic_property(ce, Z_STR_P(member), opline->op1_type == IS_UNUSED, op_array->filename); + if (may_be_dynamic) { + | test eax, eax + | jl >8 // dynamic property + } | mov edx, dword [FCARG1a + r0 + 8] | IF_TYPE dl, IS_UNDEF, >5 | add FCARG1a, r0 @@ -9360,7 +9392,7 @@ static int zend_jit_fetch_obj_read(dasm_State **Dst, zend_op *opline, zend_op_ar | jmp >9 } - if (offset == ZEND_WRONG_PROPERTY_OFFSET) { + if (offset == ZEND_WRONG_PROPERTY_OFFSET && may_be_dynamic) { |8: | LOAD_ADDR FCARG2a, member |.if X64 From 461d0231b429a048f7dfc9590cc3fbd680dd7aaf Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 18 Oct 2017 11:42:45 +0300 Subject: [PATCH 495/569] Avoid repeatable checks for $this. --- ext/opcache/jit/zend_jit.c | 80 ++++++++++++++++++++++++++++++- ext/opcache/jit/zend_jit_x86.dasc | 16 ++++--- 2 files changed, 88 insertions(+), 8 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index e9251d612d8f0..820c42e5f4440 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -1958,6 +1958,80 @@ static zend_lifetime_interval** zend_jit_allocate_registers(zend_op_array *op_ar return NULL; } +static void zend_calc_checked_this_r(zend_bitset checked_this, zend_op_array *op_array, zend_cfg *cfg, int b, int checked) +{ + zend_op *opline = &op_array->opcodes[cfg->blocks[b].start]; + zend_op *end = opline + cfg->blocks[b].len; + int old_checked = checked; + int i; + + for (; opline < end; opline++) { + switch (opline->opcode) { + case ZEND_ASSIGN_ADD: + case ZEND_ASSIGN_SUB: + case ZEND_ASSIGN_MUL: + case ZEND_ASSIGN_DIV: + case ZEND_ASSIGN_MOD: + case ZEND_ASSIGN_SL: + case ZEND_ASSIGN_SR: + case ZEND_ASSIGN_CONCAT: + case ZEND_ASSIGN_BW_OR: + case ZEND_ASSIGN_BW_AND: + case ZEND_ASSIGN_BW_XOR: + case ZEND_ASSIGN_POW: + if (opline->extended_value != ZEND_ASSIGN_OBJ) { + break; + } + case ZEND_PRE_INC_OBJ: + case ZEND_PRE_DEC_OBJ: + case ZEND_POST_INC_OBJ: + case ZEND_POST_DEC_OBJ: + case ZEND_FETCH_OBJ_R: + case ZEND_FETCH_OBJ_W: + case ZEND_FETCH_OBJ_RW: + case ZEND_FETCH_OBJ_IS: + case ZEND_FETCH_OBJ_FUNC_ARG: + case ZEND_FETCH_OBJ_UNSET: + case ZEND_ASSIGN_OBJ: + case ZEND_INIT_METHOD_CALL: + case ZEND_CLONE: + case ZEND_UNSET_OBJ: + case ZEND_ISSET_ISEMPTY_PROP_OBJ: + if (opline->op1_type != IS_UNUSED) { + break; + } + case ZEND_FETCH_THIS: + if (checked) { + zend_bitset_incl(checked_this, (opline - op_array->opcodes)); + } else { + checked = 1; + } + break; + default: + break; + } + opline++; + } + + if (cfg->blocks[b].flags && ZEND_BB_TRY) { + checked = old_checked; + } + + for (i = cfg->blocks[b].children; i >= 0; i = cfg->blocks[i].next_child) { + zend_calc_checked_this_r(checked_this, op_array, cfg, i, checked); + } +} + +static zend_bitset zend_calc_checked_this(zend_arena **arena, zend_op_array *op_array, zend_cfg *cfg) +{ + uint32_t bitset_len = zend_bitset_len(op_array->last); + zend_bitset checked_this = zend_arena_calloc(arena, bitset_len, ZEND_BITSET_ELM_SIZE); + + zend_calc_checked_this_r(checked_this, op_array, cfg, 0, 0); + + return checked_this; +} + static int zend_jit(zend_op_array *op_array, zend_ssa *ssa, const zend_op *rt_opline) { int b, i, end; @@ -1967,6 +2041,7 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa, const zend_op *rt_op int call_level = 0; void *checkpoint = NULL; zend_lifetime_interval **ra = NULL; + zend_bitset checked_this = NULL; if (zend_jit_reg_alloc) { checkpoint = zend_arena_checkpoint(CG(arena)); @@ -2316,7 +2391,10 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa, const zend_op *rt_op goto done; case ZEND_FETCH_OBJ_R: case ZEND_FETCH_OBJ_IS: - if (!zend_jit_fetch_obj_read(&dasm_state, opline, op_array, ssa)) { + if (opline->op1_type == IS_UNUSED && !checked_this) { + checked_this = zend_calc_checked_this(&CG(arena), op_array, &ssa->cfg); + } + if (!zend_jit_fetch_obj_read(&dasm_state, opline, op_array, ssa, checked_this)) { goto jit_failure; } goto done; diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index edcd6eede2904..96392caf6286d 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -9231,7 +9231,7 @@ static zend_bool zend_may_be_dynamic_property(zend_class_entry *ce, zend_string return 0; } -static int zend_jit_fetch_obj_read(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) +static int zend_jit_fetch_obj_read(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, zend_bitset checked_this) { uint32_t op1_info; zend_class_entry *ce = NULL; @@ -9274,12 +9274,14 @@ static int zend_jit_fetch_obj_read(dasm_State **Dst, zend_op *opline, zend_op_ar offset = zend_get_known_property_offset(ce, Z_STR_P(member), opline->op1_type == IS_UNUSED, op_array->filename); if (opline->op1_type == IS_UNUSED) { - | IF_ZVAL_TYPE this_addr, IS_UNDEF, >1 - |.cold_code - |1: - | SAVE_VALID_OPLINE opline - | jmp ->not_obj - |.code + if (!checked_this || !zend_bitset_in(checked_this, (opline - op_array->opcodes))) { + | IF_ZVAL_TYPE this_addr, IS_UNDEF, >1 + |.cold_code + |1: + | SAVE_VALID_OPLINE opline + | jmp ->not_obj + |.code + } | GET_ZVAL_PTR FCARG1a, this_addr } else { if (op1_info & MAY_BE_REF) { From 7e31b13c06561d06611f38d70ec724e716214b84 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Wed, 18 Oct 2017 17:21:10 +0800 Subject: [PATCH 496/569] Disable JIT for functions with extened_infos --- ext/opcache/jit/zend_jit.c | 41 +++++++++++++++++++++----------------- 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 820c42e5f4440..e083225444f36 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -649,9 +649,9 @@ static int zend_may_overflow(const zend_op *opline, zend_op_array *op_array, zen } } -static int zend_jit_build_cfg(zend_op_array *op_array, zend_cfg *cfg, uint32_t *flags) +static int zend_jit_build_cfg(zend_op_array *op_array, zend_cfg *cfg) { - if (zend_build_cfg(&CG(arena), op_array, ZEND_CFG_STACKLESS | ZEND_CFG_RECV_ENTRY | ZEND_RT_CONSTANTS | ZEND_CFG_NO_ENTRY_PREDECESSORS | ZEND_SSA_RC_INFERENCE_FLAG | ZEND_SSA_USE_CV_RESULTS, cfg, flags) != SUCCESS) { + if (zend_build_cfg(&CG(arena), op_array, ZEND_CFG_STACKLESS | ZEND_CFG_RECV_ENTRY | ZEND_RT_CONSTANTS | ZEND_CFG_NO_ENTRY_PREDECESSORS | ZEND_SSA_RC_INFERENCE_FLAG | ZEND_SSA_USE_CV_RESULTS, cfg) != SUCCESS) { return FAILURE; } @@ -665,16 +665,21 @@ static int zend_jit_build_cfg(zend_op_array *op_array, zend_cfg *cfg, uint32_t * } /* Identify reducible and irreducible loops */ - if (zend_cfg_identify_loops(op_array, cfg, flags) != SUCCESS) { + if (zend_cfg_identify_loops(op_array, cfg) != SUCCESS) { return FAILURE; } return SUCCESS; } -static int zend_jit_op_array_analyze1(zend_op_array *op_array, zend_script *script, zend_ssa *ssa, uint32_t *flags) +static int zend_jit_op_array_analyze1(zend_op_array *op_array, zend_script *script, zend_ssa *ssa) { - if (zend_jit_build_cfg(op_array, &ssa->cfg, flags) != SUCCESS) { + if (zend_jit_build_cfg(op_array, &ssa->cfg) != SUCCESS) { + return FAILURE; + } + + /* TODO: debugger and profiler supports? */ + if ((ssa->cfg.flags & ZEND_FUNC_HAS_EXTENDED_INFO)) { return FAILURE; } @@ -682,8 +687,8 @@ static int zend_jit_op_array_analyze1(zend_op_array *op_array, zend_script *scri && ssa->cfg.blocks && op_array->last_try_catch == 0 && !(op_array->fn_flags & ZEND_ACC_GENERATOR) - && !(*flags & ZEND_FUNC_INDIRECT_VAR_ACCESS)) { - if (zend_build_ssa(&CG(arena), script, op_array, ZEND_RT_CONSTANTS | ZEND_SSA_RC_INFERENCE | ZEND_SSA_USE_CV_RESULTS, ssa, flags) != SUCCESS) { + && !(ssa->cfg.flags & ZEND_FUNC_INDIRECT_VAR_ACCESS)) { + if (zend_build_ssa(&CG(arena), script, op_array, ZEND_RT_CONSTANTS | ZEND_SSA_RC_INFERENCE | ZEND_SSA_USE_CV_RESULTS, ssa) != SUCCESS) { return FAILURE; } @@ -705,13 +710,13 @@ static int zend_jit_op_array_analyze1(zend_op_array *op_array, zend_script *scri return SUCCESS; } -static int zend_jit_op_array_analyze2(zend_op_array *op_array, zend_script *script, zend_ssa *ssa, uint32_t *flags) +static int zend_jit_op_array_analyze2(zend_op_array *op_array, zend_script *script, zend_ssa *ssa) { if ((zend_jit_level >= ZEND_JIT_LEVEL_OPT_FUNC) && ssa->cfg.blocks && op_array->last_try_catch == 0 && !(op_array->fn_flags & ZEND_ACC_GENERATOR) - && !(*flags & ZEND_FUNC_INDIRECT_VAR_ACCESS)) { + && !(ssa->cfg.flags & ZEND_FUNC_INDIRECT_VAR_ACCESS)) { if (zend_ssa_inference(&CG(arena), op_array, script, ssa) != SUCCESS) { return FAILURE; @@ -2083,7 +2088,7 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa, const zend_op *rt_op #endif if (ssa->cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_RECV_ENTRY)) { opline = op_array->opcodes + ssa->cfg.blocks[b].start; - if (ssa->cfg.split_at_recv && opline->opcode == ZEND_RECV_INIT) { + if ((ssa->cfg.flags & ZEND_CFG_RECV_ENTRY) && opline->opcode == ZEND_RECV_INIT) { if (opline > op_array->opcodes && (opline-1)->opcode == ZEND_RECV_INIT) { if (zend_jit_level < ZEND_JIT_LEVEL_INLINE) { @@ -2431,7 +2436,7 @@ static int zend_jit(zend_op_array *op_array, zend_ssa *ssa, const zend_op *rt_op switch (opline->opcode) { case ZEND_RECV_INIT: - if (ssa->cfg.split_at_recv) { + if ((ssa->cfg.flags & ZEND_CFG_RECV_ENTRY)) { if (!zend_jit_handler(&dasm_state, opline, zend_may_throw(opline, op_array, ssa))) { goto jit_failure; } @@ -2600,7 +2605,6 @@ static int zend_jit_collect_calls(zend_op_array *op_array, zend_script *script) static int zend_real_jit_func(zend_op_array *op_array, zend_script *script, const zend_op *rt_opline) { - uint32_t flags = 0; zend_ssa ssa; void *checkpoint; @@ -2613,11 +2617,11 @@ static int zend_real_jit_func(zend_op_array *op_array, zend_script *script, cons /* Build SSA */ memset(&ssa, 0, sizeof(zend_ssa)); - if (zend_jit_op_array_analyze1(op_array, script, &ssa, &flags) != SUCCESS) { + if (zend_jit_op_array_analyze1(op_array, script, &ssa) != SUCCESS) { goto jit_failure; } - if (zend_jit_op_array_analyze2(op_array, script, &ssa, &flags) != SUCCESS) { + if (zend_jit_op_array_analyze2(op_array, script, &ssa) != SUCCESS) { goto jit_failure; } @@ -2757,10 +2761,9 @@ static int zend_jit_setup_hot_counters(zend_op_array *op_array) zend_op *opline = op_array->opcodes; const void **orig_handlers; zend_cfg cfg; - uint32_t flags = 0; uint32_t i; - if (zend_jit_build_cfg(op_array, &cfg, &flags) != SUCCESS) { + if (zend_jit_build_cfg(op_array, &cfg) != SUCCESS) { return FAILURE; } @@ -2895,9 +2898,10 @@ ZEND_API int zend_jit_script(zend_script *script) for (i = 0; i < call_graph.op_arrays_count; i++) { info = ZEND_FUNC_INFO(call_graph.op_arrays[i]); if (info) { - if (zend_jit_op_array_analyze1(call_graph.op_arrays[i], script, &info->ssa, &info->flags) != SUCCESS) { + if (zend_jit_op_array_analyze1(call_graph.op_arrays[i], script, &info->ssa) != SUCCESS) { goto jit_failure; } + info->flags = info->ssa.cfg.flags; } } @@ -2918,9 +2922,10 @@ ZEND_API int zend_jit_script(zend_script *script) } info = ZEND_FUNC_INFO(call_graph.op_arrays[i]); if (info) { - if (zend_jit_op_array_analyze2(call_graph.op_arrays[i], script, &info->ssa, &info->flags) != SUCCESS) { + if (zend_jit_op_array_analyze2(call_graph.op_arrays[i], script, &info->ssa) != SUCCESS) { goto jit_failure; } + info->flags = info->ssa.cfg.flags; } } From 8f382701f7437000ea029f5036c1d65831ccdcb4 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 24 Oct 2017 12:03:50 +0300 Subject: [PATCH 497/569] JIT support for dynamic property caching --- ext/opcache/jit/zend_jit_helpers.c | 54 +++++++++++++++++++++++++++--- ext/opcache/jit/zend_jit_x86.dasc | 8 +++-- 2 files changed, 55 insertions(+), 7 deletions(-) diff --git a/ext/opcache/jit/zend_jit_helpers.c b/ext/opcache/jit/zend_jit_helpers.c index f40d44f106b52..b97b4dd27b1bb 100644 --- a/ext/opcache/jit/zend_jit_helpers.c +++ b/ext/opcache/jit/zend_jit_helpers.c @@ -1969,12 +1969,35 @@ static void ZEND_FASTCALL zend_jit_fetch_obj_r_slow(zend_object *zobj, zval *off } } -static void ZEND_FASTCALL zend_jit_fetch_obj_r_dynamic(zend_object *zobj, zval *offset, zval *result) +static void ZEND_FASTCALL zend_jit_fetch_obj_r_dynamic(zend_object *zobj, intptr_t prop_offset, zval *offset, zval *result) { if (zobj->properties) { - zval *retval = zend_hash_find(zobj->properties, Z_STR_P(offset)); + zval *retval; + + if (!IS_UNKNOWN_DYNAMIC_PROPERTY_OFFSET(prop_offset)) { + intptr_t idx = ZEND_DECODE_DYN_PROP_OFFSET(prop_offset); + + if (EXPECTED(idx < zobj->properties->nNumUsed * sizeof(Bucket))) { + Bucket *p = (Bucket*)((char*)zobj->properties->arData + idx); + + if (EXPECTED(Z_TYPE(p->val) != IS_UNDEF) && + (EXPECTED(p->key == Z_STR_P(offset)) || + (EXPECTED(p->h == ZSTR_H(Z_STR_P(offset))) && + EXPECTED(p->key != NULL) && + EXPECTED(ZSTR_LEN(p->key) == Z_STRLEN_P(offset)) && + EXPECTED(memcmp(ZSTR_VAL(p->key), Z_STRVAL_P(offset), Z_STRLEN_P(offset)) == 0)))) { + ZVAL_COPY_UNREF(result, &p->val); + return; + } + } + CACHE_PTR_EX((void**)((char*)EG(current_execute_data)->run_time_cache + Z_CACHE_SLOT_P(offset)) + 1, (void*)ZEND_DYNAMIC_PROPERTY_OFFSET); + } + + retval = zend_hash_find(zobj->properties, Z_STR_P(offset)); if (EXPECTED(retval)) { + intptr_t idx = (char*)retval - (char*)zobj->properties->arData; + CACHE_PTR_EX((void**)((char*)EG(current_execute_data)->run_time_cache + Z_CACHE_SLOT_P(offset)) + 1, (void*)ZEND_ENCODE_DYN_PROP_OFFSET(idx)); ZVAL_COPY_UNREF(result, retval); return; } @@ -1999,12 +2022,35 @@ static void ZEND_FASTCALL zend_jit_fetch_obj_is_slow(zend_object *zobj, zval *of } } -static void ZEND_FASTCALL zend_jit_fetch_obj_is_dynamic(zend_object *zobj, zval *offset, zval *result) +static void ZEND_FASTCALL zend_jit_fetch_obj_is_dynamic(zend_object *zobj, intptr_t prop_offset, zval *offset, zval *result) { if (zobj->properties) { - zval *retval = zend_hash_find(zobj->properties, Z_STR_P(offset)); + zval *retval; + + if (!IS_UNKNOWN_DYNAMIC_PROPERTY_OFFSET(prop_offset)) { + intptr_t idx = ZEND_DECODE_DYN_PROP_OFFSET(prop_offset); + + if (EXPECTED(idx < zobj->properties->nNumUsed * sizeof(Bucket))) { + Bucket *p = (Bucket*)((char*)zobj->properties->arData + idx); + + if (EXPECTED(Z_TYPE(p->val) != IS_UNDEF) && + (EXPECTED(p->key == Z_STR_P(offset)) || + (EXPECTED(p->h == ZSTR_H(Z_STR_P(offset))) && + EXPECTED(p->key != NULL) && + EXPECTED(ZSTR_LEN(p->key) == Z_STRLEN_P(offset)) && + EXPECTED(memcmp(ZSTR_VAL(p->key), Z_STRVAL_P(offset), Z_STRLEN_P(offset)) == 0)))) { + ZVAL_COPY_UNREF(result, &p->val); + return; + } + } + CACHE_PTR_EX((void**)((char*)EG(current_execute_data)->run_time_cache + Z_CACHE_SLOT_P(offset)) + 1, (void*)ZEND_DYNAMIC_PROPERTY_OFFSET); + } + + retval = zend_hash_find(zobj->properties, Z_STR_P(offset)); if (EXPECTED(retval)) { + intptr_t idx = (char*)retval - (char*)zobj->properties->arData; + CACHE_PTR_EX((void**)((char*)EG(current_execute_data)->run_time_cache + Z_CACHE_SLOT_P(offset)) + 1, (void*)ZEND_ENCODE_DYN_PROP_OFFSET(idx)); ZVAL_COPY(result, retval); return; } diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 96392caf6286d..22ac49b5b6041 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -9303,7 +9303,7 @@ static int zend_jit_fetch_obj_read(dasm_State **Dst, zend_op *opline, zend_op_ar | mov r0, aword [r0 + Z_CACHE_SLOT_P(member) + sizeof(void*)] may_be_dynamic = zend_may_be_dynamic_property(ce, Z_STR_P(member), opline->op1_type == IS_UNUSED, op_array->filename); if (may_be_dynamic) { - | test eax, eax + | test r0, r0 | jl >8 // dynamic property } | mov edx, dword [FCARG1a + r0 + 8] @@ -9396,11 +9396,13 @@ static int zend_jit_fetch_obj_read(dasm_State **Dst, zend_op *opline, zend_op_ar if (offset == ZEND_WRONG_PROPERTY_OFFSET && may_be_dynamic) { |8: - | LOAD_ADDR FCARG2a, member + | mov FCARG2a, r0 |.if X64 - | LOAD_ZVAL_ADDR CARG3, res_addr + | LOAD_ADDR CARG3, member + | LOAD_ZVAL_ADDR CARG4, res_addr |.else | PUSH_ZVAL_ADDR res_addr, r0 + | PUSH_ADDR member, r0 |.endif | SAVE_VALID_OPLINE opline if (opline->opcode == ZEND_FETCH_OBJ_R) { From 373e3f93e3863d7cca5fd813938175d834a999d2 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Tue, 31 Oct 2017 15:28:30 +0800 Subject: [PATCH 498/569] Fixed build --- ext/opcache/jit/zend_jit_helpers.c | 2 +- ext/opcache/jit/zend_jit_vm_helpers.c | 2 +- ext/opcache/jit/zend_jit_x86.dasc | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ext/opcache/jit/zend_jit_helpers.c b/ext/opcache/jit/zend_jit_helpers.c index b97b4dd27b1bb..3cdbcae0cdf2e 100644 --- a/ext/opcache/jit/zend_jit_helpers.c +++ b/ext/opcache/jit/zend_jit_helpers.c @@ -1663,7 +1663,7 @@ static void ZEND_FASTCALL zend_jit_free_call_frame(zend_execute_data *call) static zval* ZEND_FASTCALL zend_jit_new_ref_helper(zval *value) { zend_reference *ref = (zend_reference*)emalloc(sizeof(zend_reference)); - GC_REFCOUNT(ref) = 1; + GC_SET_REFCOUNT(ref, 1); GC_TYPE_INFO(ref) = IS_REFERENCE; ZVAL_COPY_VALUE(&ref->val, value); Z_REF_P(value) = ref; diff --git a/ext/opcache/jit/zend_jit_vm_helpers.c b/ext/opcache/jit/zend_jit_vm_helpers.c index 5f49c4890a0d1..934474780de7b 100644 --- a/ext/opcache/jit/zend_jit_vm_helpers.c +++ b/ext/opcache/jit/zend_jit_vm_helpers.c @@ -47,7 +47,7 @@ void ZEND_FASTCALL zend_jit_leave_nested_func_helper(uint32_t call_info) if (UNEXPECTED(call_info & ZEND_CALL_RELEASE_THIS)) { zend_object *object = Z_OBJ(execute_data->This); if (UNEXPECTED(EG(exception) != NULL) && (call_info & ZEND_CALL_CTOR)) { - GC_REFCOUNT(object)--; + GC_DELREF(object); zend_object_store_ctor_failed(object); } OBJ_RELEASE(object); diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 22ac49b5b6041..d8190d9924b97 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -7334,7 +7334,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar | // if (UNEXPECTED(EG(exception) | MEM_OP2_1 cmp, aword, &EG(exception), 0, r1 | je >1 - | // GC_REFCOUNT(object)--; + | // GC_DELREF(object); | GC_DELREF r0 | // zend_object_store_ctor_failed(object); | // GC_FLAGS(obj) |= IS_OBJ_DESTRUCTOR_CALLED; @@ -8260,7 +8260,7 @@ static int zend_jit_leave_func(dasm_State **Dst, const zend_op *opline, zend_op_ | // if (call_info & ZEND_CALL_CTOR) | test FCARG1d, ZEND_CALL_CTOR | jz >5 - | // GC_REFCOUNT(object)--; + | // GC_DELREF(object); | GC_DELREF r0 | // zend_object_store_ctor_failed(object); | // GC_FLAGS(obj) |= IS_OBJ_DESTRUCTOR_CALLED; From 36fcb38c0d5e4ef665c0eb57a4cb5c8593a88759 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 11 Dec 2017 12:14:11 +0300 Subject: [PATCH 499/569] Fixed JIT for TYPE_CHECK --- ext/opcache/jit/zend_jit_x86.dasc | 37 ++++++++++++++++++------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index d8190d9924b97..00253111788c7 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -7941,7 +7941,7 @@ static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, int b, i zend_bool smart_branch = 0; zend_jit_addr op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, NULL, -1); - if (opline->extended_value == IS_RESOURCE) { + if (opline->extended_value & MAY_BE_RESOURCE) { // TODO: support for is_resource() ??? goto fallback; } @@ -7965,7 +7965,7 @@ static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, int b, i | SAVE_VALID_OPLINE opline | mov FCARG1d, opline->op1.var | EXT_CALL zend_jit_undefined_op_helper, r0 - if (opline->extended_value == IS_NULL) { + if (opline->extended_value & MAY_BE_NULL) { if (!zend_jit_smart_true(Dst, opline, b, op_array, ssa, (op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0)) { return 0; } @@ -7980,11 +7980,18 @@ static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, int b, i } if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { - type = opline->extended_value; - if (type == _IS_BOOL) { - mask = MAY_BE_FALSE | MAY_BE_TRUE; - } else { - mask = (1 << type); + mask = opline->extended_value; + switch (mask) { + case MAY_BE_NULL: type = IS_NULL; break; + case MAY_BE_FALSE: type = IS_FALSE; break; + case MAY_BE_TRUE: type = IS_TRUE; break; + case MAY_BE_LONG: type = IS_LONG; break; + case MAY_BE_DOUBLE: type = IS_DOUBLE; break; + case MAY_BE_STRING: type = IS_STRING; break; + case MAY_BE_ARRAY: type = IS_ARRAY; break; + case MAY_BE_OBJECT: type = IS_OBJECT; break; + default: + type = 0; } if (!(op1_info & (MAY_BE_ANY - mask))) { @@ -8002,7 +8009,7 @@ static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, int b, i | LOAD_ZVAL_ADDR r0, op1_addr | ZVAL_DEREF r0, op1_info } - if (opline->extended_value == _IS_BOOL) { + if (type == 0) { if (smart_branch && (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { @@ -8050,9 +8057,9 @@ static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, int b, i | mov cl, byte [FP + opline->op1.var + 8] } } - | mov al, 1 - | shl al, cl - | test al, 0xc + | mov eax, 1 + | shl eax, cl + | test eax, mask if ((opline+1)->opcode == ZEND_JMPZ && (opline+1)->op1_type == IS_TMP_VAR && (opline+1)->op1.var == opline->result.var) { @@ -8079,8 +8086,6 @@ static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, int b, i | SET_ZVAL_TYPE_INFO res_addr, eax | FREE_OP opline->op1_type, opline->op1, op1_info, 1, op_array, opline } - } else if (opline->extended_value == IS_RESOURCE) { - ZEND_ASSERT(0); } else { if (smart_branch && (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && @@ -8122,12 +8127,12 @@ static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, int b, i | mov cl, byte [FP + opline->op1.var + 8] } |2: - | cmp cl, opline->extended_value + | cmp cl, type } else { if (op1_info & MAY_BE_REF) { - | cmp byte [r0 + 8], opline->extended_value + | cmp byte [r0 + 8], type } else { - | cmp byte [FP + opline->op1.var + 8], opline->extended_value + | cmp byte [FP + opline->op1.var + 8], type } } if ((opline+1)->opcode == ZEND_JMPZ && From b92fd1755e7d377b0a57530a7567f73a0e8141b1 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 11 Dec 2017 12:15:01 +0300 Subject: [PATCH 500/569] Fixed error message --- ext/opcache/jit/zend_jit_helpers.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_helpers.c b/ext/opcache/jit/zend_jit_helpers.c index 3cdbcae0cdf2e..73ca3de80f577 100644 --- a/ext/opcache/jit/zend_jit_helpers.c +++ b/ext/opcache/jit/zend_jit_helpers.c @@ -97,7 +97,7 @@ static zval* ZEND_FASTCALL zend_jit_hash_index_lookup_rw(HashTable *ht, zend_lon zval *retval = zend_hash_index_find(ht, idx); if (!retval) { - zend_error(E_NOTICE,"Undefined index: %s", idx); + zend_error(E_NOTICE,"Undefined offset: " ZEND_LONG_FMT, idx); retval = zend_hash_index_update(ht, idx, &EG(uninitialized_zval)); } return retval; From abb10597f4a013f326cf2dc2a36ab67b79599386 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 11 Dec 2017 12:22:43 +0300 Subject: [PATCH 501/569] Use "fastcal" calling convention for internal PHP functions on x86 --- ext/opcache/jit/zend_jit_x86.dasc | 24 ++++++------------------ 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 00253111788c7..033367efdbea0 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -7288,24 +7288,12 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar zend_jit_reset_opline(Dst, NULL); | // fbc->internal_function.handler(call, ret); - |.if X64 - | mov FCARG1a, RX - if (func) { - | EXT_CALL func->internal_function.handler, r0 - } else { - | call aword [r0 + offsetof(zend_internal_function, handler)] - } - |.else - | sub r4, 8 - | push FCARG2a - | push RX - if (func) { - | EXT_CALL func->internal_function.handler, r0 - } else { - | call aword [r0 + offsetof(zend_internal_function, handler)] - } - | add r4, 16 - |.endif + | mov FCARG1a, RX + if (func) { + | EXT_CALL func->internal_function.handler, r0 + } else { + | call aword [r0 + offsetof(zend_internal_function, handler)] + } | // EG(current_execute_data) = execute_data; | MEM_OP2_1 mov, aword, &EG(current_execute_data), FP, r0 From 56bf8808f6508dbf2e2d8159d0c6a643103d966b Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 12 Jan 2018 13:34:05 +0300 Subject: [PATCH 502/569] Fixed JIT support for recent "master" changes --- ext/opcache/jit/zend_jit_vm_helpers.c | 5 +++-- ext/opcache/jit/zend_jit_x86.dasc | 24 +++++------------------- 2 files changed, 8 insertions(+), 21 deletions(-) diff --git a/ext/opcache/jit/zend_jit_vm_helpers.c b/ext/opcache/jit/zend_jit_vm_helpers.c index 934474780de7b..3e37f04c3b3a6 100644 --- a/ext/opcache/jit/zend_jit_vm_helpers.c +++ b/ext/opcache/jit/zend_jit_vm_helpers.c @@ -20,6 +20,7 @@ #include "Zend/zend_execute.h" #include "Zend/zend_exceptions.h" #include "Zend/zend_vm.h" +#include "Zend/zend_closures.h" #include #include "Optimizer/zend_func_info.h" @@ -52,7 +53,7 @@ void ZEND_FASTCALL zend_jit_leave_nested_func_helper(uint32_t call_info) } OBJ_RELEASE(object); } else if (UNEXPECTED(call_info & ZEND_CALL_CLOSURE)) { - OBJ_RELEASE((zend_object*)execute_data->func->op_array.prototype); + OBJ_RELEASE(ZEND_CLOSURE_OBJECT(EX(func))); } zend_vm_stack_free_extra_args_ex(call_info, execute_data); @@ -82,7 +83,7 @@ void ZEND_FASTCALL zend_jit_leave_top_func_helper(uint32_t call_info) } EG(current_execute_data) = EX(prev_execute_data); if (UNEXPECTED(call_info & ZEND_CALL_CLOSURE)) { - OBJ_RELEASE((zend_object*)EX(func)->op_array.prototype); + OBJ_RELEASE(ZEND_CLOSURE_OBJECT(EX(func))); } execute_data = EG(current_execute_data); opline = zend_jit_halt_op; diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 033367efdbea0..b4a4a6bce14d0 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -1247,14 +1247,7 @@ static void* dasm_labels[zend_lb_MAX]; || if (opline) { | SAVE_VALID_OPLINE opline || } -| .if X64 -| EXT_CALL zend_objects_store_del, r0 -| .else -| sub r4, 12 -| push FCARG1a -| EXT_CALL zend_objects_store_del, r0 -| add r4, 16 -| .endif +| EXT_CALL zend_objects_store_del, r0 || break; || } || } @@ -1467,15 +1460,8 @@ static void* dasm_labels[zend_lb_MAX]; | GC_DELREF reg | jne >1 | // zend_objects_store_del(obj); -| .if X64 -| mov CARG1, reg -| EXT_CALL zend_objects_store_del, r0 -| .else -| sub r4, 12 -| push reg -| EXT_CALL zend_objects_store_del, r0 -| add r4, 16 -| .endif +| mov FCARG1a, reg +| EXT_CALL zend_objects_store_del, r0 | jmp exit_label |1: | IF_GC_MAY_NOT_LEAK reg, tmp_reg, >1 @@ -8278,9 +8264,9 @@ static int zend_jit_leave_func(dasm_State **Dst, const zend_op *opline, zend_op_ | jnz >3 |.cold_code |3: - | // OBJ_RELEASE((zend_object*)execute_data->func->op_array.prototype); + | // OBJ_RELEASE(ZEND_CLOSURE_OBJECT(EX(func))); | mov r0, EX->func - | mov r0, aword [r0 + offsetof(zend_op_array, prototype)] + | sub r0, sizeof(zend_object) | OBJ_RELEASE r0, ecx, >4 | jmp >4 |.code From 3e4d68777e30def699fb7171189a6a73e1aa7e73 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Sat, 13 Jan 2018 14:30:15 +0800 Subject: [PATCH 503/569] Disable file cache while JIT effects --- ext/opcache/ZendAccelerator.c | 10 ++++++---- ext/opcache/zend_file_cache.c | 5 +++++ 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/ext/opcache/ZendAccelerator.c b/ext/opcache/ZendAccelerator.c index ba1246056a10d..42a43906f5adb 100644 --- a/ext/opcache/ZendAccelerator.c +++ b/ext/opcache/ZendAccelerator.c @@ -2767,14 +2767,18 @@ static int accel_post_startup(void) } } + /* Initialize zend_func_info_rid */ + zend_optimizer_startup(); + /********************************************/ /* End of non-SHM dependent initializations */ /********************************************/ #ifdef HAVE_OPCACHE_FILE_CACHE - if (!ZCG(accel_directives).file_cache_only) { + if (!ZCG(accel_directives).file_cache_only) #else - if (1) { + if (1) #endif + { switch (zend_shared_alloc_startup(ZCG(accel_directives).memory_consumption)) { case ALLOC_SUCCESS: if (zend_accel_init_shm() == FAILURE) { @@ -2819,8 +2823,6 @@ static int accel_post_startup(void) /* Init auto-global strings */ zend_accel_init_auto_globals(); - zend_optimizer_startup(); - zend_shared_alloc_lock(); #ifdef HAVE_JIT if (ZCG(accel_directives).jit && diff --git a/ext/opcache/zend_file_cache.c b/ext/opcache/zend_file_cache.c index 047abcb91c0a0..329c1bf5e38b2 100644 --- a/ext/opcache/zend_file_cache.c +++ b/ext/opcache/zend_file_cache.c @@ -777,6 +777,11 @@ int zend_file_cache_script_store(zend_persistent_script *script, int in_shm) #endif void *mem, *buf; + /* FIXME: dump jited codes out to file cache? */ + if (ZCG(accel_directives).jit) { + return FAILURE; + } + filename = zend_file_cache_get_bin_file_path(script->script.filename); if (zend_file_cache_mkdir(filename, strlen(ZCG(accel_directives).file_cache)) != SUCCESS) { From 9c1f2ba40fa1c7f12c2d7260f3c22f98798e0008 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Tue, 16 Jan 2018 15:13:00 +0800 Subject: [PATCH 504/569] Use zend_cpu_supports --- ext/opcache/jit/zend_cpuid.h | 60 ------------------------------- ext/opcache/jit/zend_jit_x86.dasc | 14 ++------ 2 files changed, 3 insertions(+), 71 deletions(-) delete mode 100644 ext/opcache/jit/zend_cpuid.h diff --git a/ext/opcache/jit/zend_cpuid.h b/ext/opcache/jit/zend_cpuid.h deleted file mode 100644 index f095afb028663..0000000000000 --- a/ext/opcache/jit/zend_cpuid.h +++ /dev/null @@ -1,60 +0,0 @@ -/* - +----------------------------------------------------------------------+ - | Zend JIT | - +----------------------------------------------------------------------+ - | Copyright (c) 1998-2016 The PHP Group | - +----------------------------------------------------------------------+ - | This source file is subject to version 3.01 of the PHP license, | - | that is bundled with this package in the file LICENSE, and is | - | available through the world-wide-web at the following url: | - | http://www.php.net/license/3_01.txt | - | If you did not receive a copy of the PHP license and are unable to | - | obtain it through the world-wide-web, please send a note to | - | license@php.net so we can mail you a copy immediately. | - +----------------------------------------------------------------------+ - | Authors: Dmitry Stogov | - +----------------------------------------------------------------------+ -*/ - -#ifndef ZEND_CPUID -#define ZEND_CPUID - -#define ZEND_CPUID_EDX_FPU (1<<0) -#define ZEND_CPUID_EDX_MMX (1<<23) -#define ZEND_CPUID_EDX_SSE (1<<25) -#define ZEND_CPUID_EDX_SSE2 (1<<26) - -#define ZEND_CPUID_ECX_SSE3 (1<<0) -#define ZEND_CPUID_ECX_SSSE3 (1<<9) -#define ZEND_CPUID_ECX_FMA3 (1<<12) -#define ZEND_CPUID_ECX_SSE41 (1<<19) -#define ZEND_CPUID_ECX_SSE42 (1<<20) -#define ZEND_CPUID_ECX_AES (1<<25) -#define ZEND_CPUID_ECX_AVX (1<<28) -#define ZEND_CPUID_ECX_RDRAND (1<<30) - -typedef struct _zend_cpuid_info { - uint32_t eax; - uint32_t ebx; - uint32_t ecx; - uint32_t edx; -} zend_cpuid_info; - -static inline void zend_cpuid(uint32_t func, uint32_t subfunc, zend_cpuid_info *info) -{ - __asm__ __volatile__ ( - "cpuid" - : "=a"(info->eax), "=b"(info->ebx), "=c"(info->ecx), "=d"(info->edx) - : "a"(func), "c"(subfunc) - ); -} - -#endif - -/* - * Local variables: - * tab-width: 4 - * c-basic-offset: 4 - * indent-tabs-mode: t - * End: - */ diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index b4a4a6bce14d0..0d279c6b25314 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -63,8 +63,8 @@ |.define HYBRID_SPAD, 16 +#include "Zend/zend_cpuinfo.h" #include "jit/zend_jit_x86.h" -#include "jit/zend_cpuid.h" const char* zend_reg_name[] = { #ifdef __x86_64__ @@ -2161,21 +2161,13 @@ static const zend_jit_stub zend_jit_stubs[] = { static int zend_jit_setup(void) { - zend_cpuid_info info; - - zend_cpuid(0, 0, &info); - if (info.eax < 1) { - return FAILURE; - } - zend_cpuid(1, 0, &info); - |.if SSE - if (!(info.edx & ZEND_CPUID_EDX_SSE2)) { + if (!zend_cpu_supports(ZEND_CPU_FEATURE_SSE2)) { zend_error(E_CORE_ERROR, "CPU doesn't support SSE2"); return FAILURE; } if (zend_jit_cpu_flags & ZEND_JIT_CPU_AVX) { - if (info.ecx & ZEND_CPUID_ECX_AVX) { + if (zend_cpu_supports(ZEND_CPU_FEATURE_AVX)) { zend_jit_x86_flags |= ZEND_JIT_CPU_AVX; } } From ed6f8d60abf4542c8a66db395f97cd793f516258 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 24 Jan 2018 10:46:09 +0300 Subject: [PATCH 505/569] Fixed JIT support for recent master changes --- ext/opcache/jit/zend_jit_x86.dasc | 42 +++---------------------------- 1 file changed, 3 insertions(+), 39 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 0d279c6b25314..d8369d5877a0e 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -1189,23 +1189,6 @@ static void* dasm_labels[zend_lb_MAX]; | EXT_CALL _zval_copy_ctor_func, r0 |.endmacro -// zval should be in FCARG1a -|.macro ZVAL_COPY_CTOR, val_info, type_flags_reg, value_ptr_reg, filename, lineno -|| if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { -|| if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY)) { -| IF_NOT_FLAGS type_flags_reg, IS_TYPE_REFCOUNTED + IS_TYPE_COPYABLE, >2 -| IF_NOT_FLAGS type_flags_reg, IS_TYPE_COPYABLE, >1 -| GC_ADDREF value_ptr_reg -| jmp >2 -|1: -| ZVAL_COPY_CTOR_FUNC filename, lineno -|2: -|| } else { -| TRY_ADDREF val_info, type_flags_reg, value_ptr_reg -|| } -|| } -|.endmacro - |.macro ZVAL_DEREF, reg, info || if (info & MAY_BE_REF) { | IF_NOT_Z_TYPE, reg, IS_REFERENCE, >1 @@ -1337,11 +1320,11 @@ static void* dasm_labels[zend_lb_MAX]; |.macro SEPARATE_ZVAL_NOREF, addr, op_info, cold, filename, lineno || if ((op_info & (MAY_BE_STRING|MAY_BE_ARRAY)) && RC_MAY_BE_N(op_info)) { || if (cold) { -| IF_ZVAL_FLAGS addr, IS_TYPE_COPYABLE, >1 +| IF_ZVAL_TYPE addr, IS_ARRAY, >1 |.cold_code |1: || } else { -| IF_NOT_ZVAL_FLAGS addr, IS_TYPE_COPYABLE, >2 +| IF_NOT_ZVAL_TYPE addr, IS_ARRAY, >2 || } | GET_ZVAL_PTR r0, addr || if (RC_MAY_BE_1(op_info)) { @@ -2653,27 +2636,8 @@ static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, zend_op_arr zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); | ZVAL_COPY_VALUE res_addr, res_use_info, val_addr, op1_info, ZREG_R0, ZREG_R2 - | //ZVAL_COPY_CTOR op1_info, ah, r2, op_array->filename, opline->lineno - if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { - if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY)) { - | IF_NOT_FLAGS ah, IS_TYPE_REFCOUNTED + IS_TYPE_COPYABLE, >2 - | IF_FLAGS ah, IS_TYPE_COPYABLE, >1 - | GC_ADDREF r2 - | jmp >2 - |1: - | mov aword [r4], FCARG1a // save - | ZVAL_COPY_CTOR_FUNC op_array->filename, opline->lineno - | mov FCARG1a, aword [r4] // restore - |2: - } else { - | TRY_ADDREF op1_info, ah, r2 - } - } + | TRY_ADDREF op1_info, ah, r2 } - } else { - zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); - - | SEPARATE_ZVAL_NOREF var_addr, op1_info, 0, op_array->filename, opline->lineno } if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { | EXT_CALL increment_function, r0 From 7726583c73e261a3b479afcda5972e934860a992 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 29 Jan 2018 13:28:36 +0300 Subject: [PATCH 506/569] Strings are not COPYABLE anymore. --- ext/opcache/jit/zend_jit_x86.dasc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index d8369d5877a0e..fb0d27f7bc968 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -1318,7 +1318,7 @@ static void* dasm_labels[zend_lb_MAX]; |.endmacro |.macro SEPARATE_ZVAL_NOREF, addr, op_info, cold, filename, lineno -|| if ((op_info & (MAY_BE_STRING|MAY_BE_ARRAY)) && RC_MAY_BE_N(op_info)) { +|| if ((op_info & MAY_BE_ARRAY) && RC_MAY_BE_N(op_info)) { || if (cold) { | IF_ZVAL_TYPE addr, IS_ARRAY, >1 |.cold_code From 4e8b398621b353e302490c1851f57837066bf9b5 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 5 Feb 2018 22:50:45 +0300 Subject: [PATCH 507/569] Fixed JIT support for recent master changes. --- ext/opcache/jit/zend_jit_helpers.c | 31 ++++++++++----------- ext/opcache/jit/zend_jit_vm_helpers.c | 9 +++++-- ext/opcache/jit/zend_jit_x86.dasc | 39 +++++++++++++++++---------- 3 files changed, 48 insertions(+), 31 deletions(-) diff --git a/ext/opcache/jit/zend_jit_helpers.c b/ext/opcache/jit/zend_jit_helpers.c index 73ca3de80f577..c214c14322e0e 100644 --- a/ext/opcache/jit/zend_jit_helpers.c +++ b/ext/opcache/jit/zend_jit_helpers.c @@ -841,6 +841,7 @@ static zend_never_inline ZEND_COLD void zend_wrong_string_offset(void) break; case ZEND_SEND_REF: case ZEND_SEND_VAR_EX: + case ZEND_SEND_FUNC_ARG: msg = "Only variables can be passed by reference"; break; EMPTY_SWITCH_DEFAULT_CASE(); @@ -1672,7 +1673,7 @@ static zval* ZEND_FASTCALL zend_jit_new_ref_helper(zval *value) return value; } -static zval* ZEND_FASTCALL zend_jit_fetch_global_helper(zend_execute_data *execute_data, zval *varname) +static zval* ZEND_FASTCALL zend_jit_fetch_global_helper(zend_execute_data *execute_data, zval *varname, uint32_t cache_slot) { uint32_t idx; zval *value = zend_hash_find(&EG(symbol_table), Z_STR_P(varname)); @@ -1681,11 +1682,11 @@ static zval* ZEND_FASTCALL zend_jit_fetch_global_helper(zend_execute_data *execu value = zend_hash_add_new(&EG(symbol_table), Z_STR_P(varname), &EG(uninitialized_zval)); idx = ((char*)value - (char*)EG(symbol_table).arData) / sizeof(Bucket); /* Store "hash slot index" + 1 (NULL is a mark of uninitialized cache slot) */ - CACHE_PTR(Z_CACHE_SLOT_P(varname), (void*)(uintptr_t)(idx + 1)); + CACHE_PTR(cache_slot, (void*)(uintptr_t)(idx + 1)); } else { idx = ((char*)value - (char*)EG(symbol_table).arData) / sizeof(Bucket); /* Store "hash slot index" + 1 (NULL is a mark of uninitialized cache slot) */ - CACHE_PTR(Z_CACHE_SLOT_P(varname), (void*)(uintptr_t)(idx + 1)); + CACHE_PTR(cache_slot, (void*)(uintptr_t)(idx + 1)); /* GLOBAL variable may be an INDIRECT pointer to CV */ if (UNEXPECTED(Z_TYPE_P(value) == IS_INDIRECT)) { value = Z_INDIRECT_P(value); @@ -1949,7 +1950,7 @@ static void ZEND_FASTCALL zend_jit_zval_copy_unref_helper(zval *dst, zval *src) ZVAL_COPY(dst, src); } -static void ZEND_FASTCALL zend_jit_fetch_obj_r_slow(zend_object *zobj, zval *offset, zval *result) +static void ZEND_FASTCALL zend_jit_fetch_obj_r_slow(zend_object *zobj, zval *offset, zval *result, uint32_t cache_slot) { zval *retval; zend_execute_data *execute_data = EG(current_execute_data); @@ -1962,14 +1963,14 @@ static void ZEND_FASTCALL zend_jit_fetch_obj_r_slow(zend_object *zobj, zval *off ZVAL_NULL(result); } else { ZVAL_OBJ(&tmp, zobj); - retval = zobj->handlers->read_property(&tmp, offset, BP_VAR_R, CACHE_ADDR(Z_CACHE_SLOT_P(offset)), result); + retval = zobj->handlers->read_property(&tmp, offset, BP_VAR_R, CACHE_ADDR(cache_slot), result); if (retval != result) { ZVAL_COPY_UNREF(result, retval); } } } -static void ZEND_FASTCALL zend_jit_fetch_obj_r_dynamic(zend_object *zobj, intptr_t prop_offset, zval *offset, zval *result) +static void ZEND_FASTCALL zend_jit_fetch_obj_r_dynamic(zend_object *zobj, intptr_t prop_offset, zval *offset, zval *result, uint32_t cache_slot) { if (zobj->properties) { zval *retval; @@ -1990,22 +1991,22 @@ static void ZEND_FASTCALL zend_jit_fetch_obj_r_dynamic(zend_object *zobj, intptr return; } } - CACHE_PTR_EX((void**)((char*)EG(current_execute_data)->run_time_cache + Z_CACHE_SLOT_P(offset)) + 1, (void*)ZEND_DYNAMIC_PROPERTY_OFFSET); + CACHE_PTR_EX((void**)((char*)EG(current_execute_data)->run_time_cache + cache_slot) + 1, (void*)ZEND_DYNAMIC_PROPERTY_OFFSET); } retval = zend_hash_find(zobj->properties, Z_STR_P(offset)); if (EXPECTED(retval)) { intptr_t idx = (char*)retval - (char*)zobj->properties->arData; - CACHE_PTR_EX((void**)((char*)EG(current_execute_data)->run_time_cache + Z_CACHE_SLOT_P(offset)) + 1, (void*)ZEND_ENCODE_DYN_PROP_OFFSET(idx)); + CACHE_PTR_EX((void**)((char*)EG(current_execute_data)->run_time_cache + cache_slot) + 1, (void*)ZEND_ENCODE_DYN_PROP_OFFSET(idx)); ZVAL_COPY_UNREF(result, retval); return; } } - zend_jit_fetch_obj_r_slow(zobj, offset, result); + zend_jit_fetch_obj_r_slow(zobj, offset, result, cache_slot); } -static void ZEND_FASTCALL zend_jit_fetch_obj_is_slow(zend_object *zobj, zval *offset, zval *result) +static void ZEND_FASTCALL zend_jit_fetch_obj_is_slow(zend_object *zobj, zval *offset, zval *result, uint32_t cache_slot) { zval *retval; zend_execute_data *execute_data = EG(current_execute_data); @@ -2015,14 +2016,14 @@ static void ZEND_FASTCALL zend_jit_fetch_obj_is_slow(zend_object *zobj, zval *of ZVAL_NULL(result); } else { ZVAL_OBJ(&tmp, zobj); - retval = zobj->handlers->read_property(&tmp, offset, BP_VAR_IS, CACHE_ADDR(Z_CACHE_SLOT_P(offset)), result); + retval = zobj->handlers->read_property(&tmp, offset, BP_VAR_IS, CACHE_ADDR(cache_slot), result); if (retval != result) { ZVAL_COPY(result, retval); } } } -static void ZEND_FASTCALL zend_jit_fetch_obj_is_dynamic(zend_object *zobj, intptr_t prop_offset, zval *offset, zval *result) +static void ZEND_FASTCALL zend_jit_fetch_obj_is_dynamic(zend_object *zobj, intptr_t prop_offset, zval *offset, zval *result, uint32_t cache_slot) { if (zobj->properties) { zval *retval; @@ -2043,19 +2044,19 @@ static void ZEND_FASTCALL zend_jit_fetch_obj_is_dynamic(zend_object *zobj, intpt return; } } - CACHE_PTR_EX((void**)((char*)EG(current_execute_data)->run_time_cache + Z_CACHE_SLOT_P(offset)) + 1, (void*)ZEND_DYNAMIC_PROPERTY_OFFSET); + CACHE_PTR_EX((void**)((char*)EG(current_execute_data)->run_time_cache + cache_slot) + 1, (void*)ZEND_DYNAMIC_PROPERTY_OFFSET); } retval = zend_hash_find(zobj->properties, Z_STR_P(offset)); if (EXPECTED(retval)) { intptr_t idx = (char*)retval - (char*)zobj->properties->arData; - CACHE_PTR_EX((void**)((char*)EG(current_execute_data)->run_time_cache + Z_CACHE_SLOT_P(offset)) + 1, (void*)ZEND_ENCODE_DYN_PROP_OFFSET(idx)); + CACHE_PTR_EX((void**)((char*)EG(current_execute_data)->run_time_cache + cache_slot) + 1, (void*)ZEND_ENCODE_DYN_PROP_OFFSET(idx)); ZVAL_COPY(result, retval); return; } } - zend_jit_fetch_obj_is_slow(zobj, offset, result); + zend_jit_fetch_obj_is_slow(zobj, offset, result, cache_slot); } static void ZEND_FASTCALL zend_jit_vm_stack_free_args_helper(zend_execute_data *call) diff --git a/ext/opcache/jit/zend_jit_vm_helpers.c b/ext/opcache/jit/zend_jit_vm_helpers.c index 3e37f04c3b3a6..7e1a93b03df45 100644 --- a/ext/opcache/jit/zend_jit_vm_helpers.c +++ b/ext/opcache/jit/zend_jit_vm_helpers.c @@ -116,13 +116,18 @@ void ZEND_FASTCALL zend_jit_copy_extra_args_helper(void) src--; dst--; } while (src != end); + if (type_flags & (IS_TYPE_REFCOUNTED << Z_TYPE_FLAGS_SHIFT)) { + ZEND_ADD_CALL_FLAG(execute_data, ZEND_CALL_FREE_EXTRA_ARGS); + } } else { do { - type_flags |= Z_TYPE_INFO_P(src); + if (Z_REFCOUNTED_P(src)) { + ZEND_ADD_CALL_FLAG(execute_data, ZEND_CALL_FREE_EXTRA_ARGS); + break; + } src--; } while (src != end); } - ZEND_ADD_CALL_FLAG(execute_data, ((type_flags >> Z_TYPE_FLAGS_SHIFT) & IS_TYPE_REFCOUNTED)); } } diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index fb0d27f7bc968..6b164f302581d 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -6696,6 +6696,7 @@ static int zend_jit_needs_call_chain(zend_call_info *call_info, uint32_t b, zend case ZEND_SEND_VAR: case ZEND_SEND_VAL_EX: case ZEND_SEND_VAR_EX: + case ZEND_SEND_FUNC_ARG: case ZEND_SEND_REF: case ZEND_SEND_VAR_NO_REF: case ZEND_SEND_VAR_NO_REF_EX: @@ -6762,6 +6763,7 @@ static int zend_jit_needs_call_chain(zend_call_info *call_info, uint32_t b, zend case ZEND_SEND_VAR: case ZEND_SEND_VAL_EX: case ZEND_SEND_VAR_EX: + case ZEND_SEND_FUNC_ARG: case ZEND_SEND_REF: case ZEND_SEND_VAR_NO_REF: case ZEND_SEND_VAR_NO_REF_EX: @@ -6814,9 +6816,9 @@ static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, uint32_t } else { zval *zv = RT_CONSTANT(opline, opline->op2); - | // if (CACHED_PTR(Z_CACHE_SLOT_P(fname))) + | // if (CACHED_PTR(opline->result.num)) | mov r0, EX->run_time_cache - | mov r0, aword [r0 + Z_CACHE_SLOT_P(zv)] + | mov r0, aword [r0 + opline->result.num] | test r0, r0 | jz >1 |.cold_code @@ -6838,9 +6840,9 @@ static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, uint32_t | test r0, r0 | jz ->exception_handler } - | // CACHE_PTR(Z_CACHE_SLOT_P(fname), fbc); + | // CACHE_PTR(opline->result.num, fbc); | mov r1, EX->run_time_cache - | mov aword [r1 + Z_CACHE_SLOT_P(zv)], r0 + | mov aword [r1 + opline->result.num], r0 | jmp >3 |.code |3: @@ -7815,9 +7817,9 @@ static int zend_jit_defined(dasm_State **Dst, const zend_op *opline, int b, int } } - | // if (CACHED_PTR(Z_CACHE_SLOT_P(EX_CONSTANT(opline->op1)))) { + | // if (CACHED_PTR(opline->extended_value)) { | mov r0, EX->run_time_cache - | mov r0, aword [r0 + Z_CACHE_SLOT_P(zv)] + | mov r0, aword [r0 + opline->extended_value] | test r0, r0 | jz >1 |.cold_code @@ -7833,7 +7835,7 @@ static int zend_jit_defined(dasm_State **Dst, const zend_op *opline, int b, int | jz >3 } | mov r1, EX->run_time_cache - | mov aword [r1 + Z_CACHE_SLOT_P(zv)], r0 + | mov aword [r1 + opline->extended_value], r0 if (defined_label != (uint32_t)-1) { | jmp =>defined_label } else { @@ -7843,7 +7845,7 @@ static int zend_jit_defined(dasm_State **Dst, const zend_op *opline, int b, int res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1); | jz >2 | mov r1, EX->run_time_cache - | mov aword [r1 + Z_CACHE_SLOT_P(zv)], r0 + | mov aword [r1 + opline->extended_value], r0 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE | jmp >3 |2: @@ -8709,9 +8711,9 @@ static int zend_jit_bind_global(dasm_State **Dst, const zend_op *opline, zend_op op1_info = OP1_INFO(); } - //idx = (uint32_t)(uintptr_t)CACHED_PTR(Z_CACHE_SLOT_P(varname)) - 1; + //idx = (uint32_t)(uintptr_t)CACHED_PTR(opline->extended_value) - 1; | mov r0, EX->run_time_cache - | mov r0, aword [r0 + Z_CACHE_SLOT_P(varname)] + | mov r0, aword [r0 + opline->extended_value] | sub r0, 1 //if (EXPECTED(idx < EG(symbol_table).nNumUsed)) | MEM_OP2_2 cmp, eax, dword, &EG(symbol_table).nNumUsed, r1 @@ -8835,6 +8837,11 @@ static int zend_jit_bind_global(dasm_State **Dst, const zend_op *opline, zend_op |9: | mov FCARG1a, FP | LOAD_ADDR FCARG2a, (ptrdiff_t)varname + |.if X64 + | mov CARG3, opline->extended_value + |.else + | push opline->extended_value + |.endif | EXT_CALL zend_jit_fetch_global_helper, r0 | jmp <1 |.code @@ -9019,7 +9026,7 @@ static int zend_jit_recv_init(dasm_State **Dst, const zend_op *opline, zend_op_a | jne >8 | mov FCARG1a, r0 | mov r0, EX->run_time_cache - | lea r0, [r0 + Z_CACHE_SLOT_P(zv)] + | lea r0, [r0 + opline->extended_value] | LOAD_ADDR FCARG2a, (ptrdiff_t)op_array |.if X64 | mov CARG3, arg_num @@ -9062,7 +9069,7 @@ static int zend_jit_recv_init(dasm_State **Dst, const zend_op *opline, zend_op_a |8: | mov FCARG1a, r0 | mov r0, EX->run_time_cache - | lea r0, [r0 + Z_CACHE_SLOT_P(zv)] + | lea r0, [r0 + opline->extended_value] | LOAD_ADDR FCARG2a, (ptrdiff_t)op_array |.if X64 | mov CARG3, arg_num @@ -9232,10 +9239,10 @@ static int zend_jit_fetch_obj_read(dasm_State **Dst, zend_op *opline, zend_op_ar if (offset == ZEND_WRONG_PROPERTY_OFFSET) { | mov r0, EX->run_time_cache - | mov r2, aword [r0 + Z_CACHE_SLOT_P(member)] + | mov r2, aword [r0 + opline->extended_value] | cmp r2, aword [FCARG1a + offsetof(zend_object, ce)] | jne >5 - | mov r0, aword [r0 + Z_CACHE_SLOT_P(member) + sizeof(void*)] + | mov r0, aword [r0 + opline->extended_value + sizeof(void*)] may_be_dynamic = zend_may_be_dynamic_property(ce, Z_STR_P(member), opline->op1_type == IS_UNUSED, op_array->filename); if (may_be_dynamic) { | test r0, r0 @@ -9272,7 +9279,9 @@ static int zend_jit_fetch_obj_read(dasm_State **Dst, zend_op *opline, zend_op_ar | LOAD_ADDR FCARG2a, member |.if X64 | LOAD_ZVAL_ADDR CARG3, res_addr + | mov CARG4, opline->extended_value |.else + | push opline->extended_value | PUSH_ZVAL_ADDR res_addr, r0 |.endif | SAVE_VALID_OPLINE opline @@ -9335,7 +9344,9 @@ static int zend_jit_fetch_obj_read(dasm_State **Dst, zend_op *opline, zend_op_ar |.if X64 | LOAD_ADDR CARG3, member | LOAD_ZVAL_ADDR CARG4, res_addr + | mov CARG5, opline->extended_value |.else + | push opline->extended_value | PUSH_ZVAL_ADDR res_addr, r0 | PUSH_ADDR member, r0 |.endif From d66680fc80cec8800016aeedc3ea7110252497ae Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 2 Mar 2018 12:48:31 +0300 Subject: [PATCH 508/569] Fixed support for recent "master" changes --- ext/opcache/jit/zend_jit.c | 3 ++- ext/opcache/jit/zend_jit_x86.dasc | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index e083225444f36..c1fcd2f311f94 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -718,7 +718,8 @@ static int zend_jit_op_array_analyze2(zend_op_array *op_array, zend_script *scri && !(op_array->fn_flags & ZEND_ACC_GENERATOR) && !(ssa->cfg.flags & ZEND_FUNC_INDIRECT_VAR_ACCESS)) { - if (zend_ssa_inference(&CG(arena), op_array, script, ssa) != SUCCESS) { + /* TODO: passing -1 as optimization_level may break overloaded operators ??? */ + if (zend_ssa_inference(&CG(arena), op_array, script, ssa, -1) != SUCCESS) { return FAILURE; } } diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 6b164f302581d..7109ba6a4e440 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -7270,7 +7270,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar | GC_DELREF r0 | // zend_object_store_ctor_failed(object); | // GC_FLAGS(obj) |= IS_OBJ_DESTRUCTOR_CALLED; - | or byte [r0 + offsetof(zend_object, gc.u.v.flags)], IS_OBJ_DESTRUCTOR_CALLED + | or byte [r0 + offsetof(zend_object, gc.u.type_info)], IS_OBJ_DESTRUCTOR_CALLED |1: } | // OBJ_RELEASE(object); @@ -8201,7 +8201,7 @@ static int zend_jit_leave_func(dasm_State **Dst, const zend_op *opline, zend_op_ | GC_DELREF r0 | // zend_object_store_ctor_failed(object); | // GC_FLAGS(obj) |= IS_OBJ_DESTRUCTOR_CALLED; - | or byte [r0 + offsetof(zend_object, gc.u.v.flags)], IS_OBJ_DESTRUCTOR_CALLED + | or byte [r0 + offsetof(zend_object, gc.u.type_info)], IS_OBJ_DESTRUCTOR_CALLED | jmp >5 |.code |5: From c66267496c3d6851e7495f5d86f8fdd6d4b57424 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 12 Mar 2018 11:37:58 +0300 Subject: [PATCH 509/569] ws --- ext/opcache/zend_persist.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opcache/zend_persist.c b/ext/opcache/zend_persist.c index 1b846389fd6f3..cbee30d9085b9 100644 --- a/ext/opcache/zend_persist.c +++ b/ext/opcache/zend_persist.c @@ -901,7 +901,7 @@ zend_persistent_script *zend_accel_script_persist(zend_persistent_script *script #ifdef HAVE_JIT zend_long orig_jit = 0; #endif - + script->mem = ZCG(mem); ZEND_ASSERT(((zend_uintptr_t)ZCG(mem) & 0x7) == 0); /* should be 8 byte aligned */ From c2a48cc6228d020d6ed756157cb7166b9cc81fa1 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 12 Mar 2018 12:00:08 +0300 Subject: [PATCH 510/569] Cleanup --- ext/opcache/zend_persist.c | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/ext/opcache/zend_persist.c b/ext/opcache/zend_persist.c index cbee30d9085b9..2e5ec8a23237b 100644 --- a/ext/opcache/zend_persist.c +++ b/ext/opcache/zend_persist.c @@ -364,9 +364,6 @@ static void zend_persist_op_array_ex(zend_op_array *op_array, zend_persistent_sc int already_stored = 0; zend_op *persist_ptr; zval *orig_literals = NULL; -#ifdef HAVE_JIT - int do_jit = 1; -#endif if (op_array->refcount && --(*op_array->refcount) == 0) { efree(op_array->refcount); @@ -429,9 +426,6 @@ static void zend_persist_op_array_ex(zend_op_array *op_array, zend_persistent_sc persist_ptr = zend_shared_alloc_get_xlat_entry(op_array->opcodes); ZEND_ASSERT(persist_ptr != NULL); op_array->opcodes = persist_ptr; -#ifdef HAVE_JIT - do_jit = 0; -#endif } else { zend_op *new_opcodes = zend_accel_memdup(op_array->opcodes, sizeof(zend_op) * op_array->last); zend_op *opline = new_opcodes; @@ -619,7 +613,8 @@ static void zend_persist_op_array_ex(zend_op_array *op_array, zend_persistent_sc #ifdef HAVE_JIT if (ZCG(accel_directives).jit && - ZEND_JIT_LEVEL(ZCG(accel_directives).jit) <= ZEND_JIT_LEVEL_OPT_FUNCS) { + ZEND_JIT_LEVEL(ZCG(accel_directives).jit) <= ZEND_JIT_LEVEL_OPT_FUNCS && + !already_stored) { zend_jit_op_array(op_array, ZCG(current_persistent_script) ? &ZCG(current_persistent_script)->script : NULL); } #endif @@ -943,11 +938,9 @@ zend_persistent_script *zend_accel_script_persist(zend_persistent_script *script } #endif - ZCG(current_persistent_script) = script; zend_accel_persist_class_table(&script->script.class_table); zend_hash_persist(&script->script.function_table, zend_persist_op_array); zend_persist_op_array_ex(&script->script.main_op_array, script); - ZCG(current_persistent_script) = NULL; #ifdef HAVE_JIT if (ZCG(accel_directives).jit) { From 9c70524888e06f45bfd82c2888bf954505a5eb2d Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 19 Mar 2018 19:20:34 +0300 Subject: [PATCH 511/569] Fixed register-allocation bug --- ext/opcache/jit/zend_jit_x86.dasc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 7109ba6a4e440..3df2142602f41 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -4215,7 +4215,7 @@ static int zend_jit_assign_to_variable(dasm_State **Dst, zend_jit_addr res_addr) /* Labels: 1,2,3,4,5 */ { - ZEND_ASSERT(Z_MODE(var_addr) == IS_MEM_ZVAL); + //ZEND_ASSERT(Z_MODE(var_addr) == IS_MEM_ZVAL); if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { int in_cold = 0; @@ -6558,7 +6558,7 @@ static int zend_jit_assign(dasm_State **Dst, const zend_op *opline, zend_op_arra op1_info = OP1_INFO(); op2_info = OP2_INFO(); - op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].op1_use : -1); + op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].op1_def : -1); op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].op2_use : -1); if (opline->result_type == IS_UNUSED) { res_addr = 0; From c8386354666a3fedd545229a595eb7d2a6f98950 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 31 May 2018 20:09:50 +0300 Subject: [PATCH 512/569] Fixed compatibility with recent "master" changes --- ext/opcache/jit/zend_jit_x86.dasc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 3df2142602f41..47b3d3eba7b5d 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -8575,7 +8575,7 @@ static int zend_jit_isset_isempty_dim(dasm_State **Dst, const zend_op *opline, i uint32_t op1_info, op2_info; zend_jit_addr op1_addr, op2_addr, res_addr; - if (!ssa->ops || !ssa->var_info || !(opline->extended_value & ZEND_ISSET)) { + if (!ssa->ops || !ssa->var_info || (opline->extended_value & ZEND_ISEMPTY)) { // TODO: support for empty() ??? goto fallback; } @@ -8639,7 +8639,7 @@ static int zend_jit_isset_isempty_dim(dasm_State **Dst, const zend_op *opline, i return 0; } } - if (opline->extended_value & ZEND_ISSET) { + if (!(opline->extended_value & ZEND_ISEMPTY)) { if ((opline+1)->opcode == ZEND_JMPZ) { unsigned int target_label = ssa->cfg.blocks[b].successors[1]; | jmp =>target_label @@ -8666,7 +8666,7 @@ static int zend_jit_isset_isempty_dim(dasm_State **Dst, const zend_op *opline, i return 0; } } - if (opline->extended_value & ZEND_ISSET) { + if (!(opline->extended_value & ZEND_ISEMPTY)) { if ((opline+1)->opcode == ZEND_JMPZ) { unsigned int target_label = ssa->cfg.blocks[b].successors[0]; | jmp =>target_label From 277cedf256a548dc947cb83f48597f0282c82ab9 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 26 Jun 2018 14:47:51 +0300 Subject: [PATCH 513/569] Added support for new master changes (incomplete) --- ext/opcache/jit/zend_jit_disasm_x86.c | 2 +- ext/opcache/jit/zend_jit_helpers.c | 50 +++++++++++++++++++-------- ext/opcache/jit/zend_jit_x86.dasc | 19 ++++------ 3 files changed, 43 insertions(+), 28 deletions(-) diff --git a/ext/opcache/jit/zend_jit_disasm_x86.c b/ext/opcache/jit/zend_jit_disasm_x86.c index 571df581e127b..56838c5003cbf 100644 --- a/ext/opcache/jit/zend_jit_disasm_x86.c +++ b/ext/opcache/jit/zend_jit_disasm_x86.c @@ -419,7 +419,7 @@ static int zend_jit_disasm_init(void) REGISTER_HELPER(zend_jit_fast_concat_helper); REGISTER_HELPER(zend_jit_isset_dim_helper); REGISTER_HELPER(zend_jit_free_call_frame); - REGISTER_HELPER(zend_jit_zval_copy_unref_helper); + REGISTER_HELPER(zend_jit_zval_copy_deref_helper) REGISTER_HELPER(zend_jit_new_ref_helper); REGISTER_HELPER(zend_jit_fetch_global_helper); REGISTER_HELPER(zend_jit_verify_arg_object); diff --git a/ext/opcache/jit/zend_jit_helpers.c b/ext/opcache/jit/zend_jit_helpers.c index c214c14322e0e..460fcac2f7aec 100644 --- a/ext/opcache/jit/zend_jit_helpers.c +++ b/ext/opcache/jit/zend_jit_helpers.c @@ -29,8 +29,19 @@ static zend_function* ZEND_FASTCALL zend_jit_find_func_helper(zend_string *name) } fbc = Z_FUNC_P(func); if (EXPECTED(fbc->type == ZEND_USER_FUNCTION) && UNEXPECTED(!fbc->op_array.run_time_cache)) { - fbc->op_array.run_time_cache = zend_arena_alloc(&CG(arena), fbc->op_array.cache_size); - memset(fbc->op_array.run_time_cache, 0, fbc->op_array.cache_size); + if (fbc->op_array.fn_flags & ZEND_ACC_IMMUTABLE) { + zend_op_array *new_op_array = zend_arena_alloc(&CG(arena), sizeof(zend_op_array) + fbc->op_array.cache_size); + + Z_PTR_P(func) = new_op_array; + memcpy(new_op_array, fbc, sizeof(zend_op_array)); + new_op_array->fn_flags &= ~ZEND_ACC_IMMUTABLE; + new_op_array->run_time_cache = (void**)(new_op_array + 1); + memset(new_op_array->run_time_cache, 0, new_op_array->cache_size); + return (zend_function*)new_op_array; + } else { + fbc->op_array.run_time_cache = zend_arena_alloc(&CG(arena), fbc->op_array.cache_size); + memset(fbc->op_array.run_time_cache, 0, fbc->op_array.cache_size); + } } return fbc; } @@ -46,8 +57,19 @@ static zend_function* ZEND_FASTCALL zend_jit_find_func_by_name_helper(zend_strin } fbc = Z_FUNC_P(func); if (EXPECTED(fbc->type == ZEND_USER_FUNCTION) && UNEXPECTED(!fbc->op_array.run_time_cache)) { - fbc->op_array.run_time_cache = zend_arena_alloc(&CG(arena), fbc->op_array.cache_size); - memset(fbc->op_array.run_time_cache, 0, fbc->op_array.cache_size); + if (fbc->op_array.fn_flags & ZEND_ACC_IMMUTABLE) { + zend_op_array *new_op_array = zend_arena_alloc(&CG(arena), sizeof(zend_op_array) + fbc->op_array.cache_size); + + Z_PTR_P(func) = new_op_array; + memcpy(new_op_array, fbc, sizeof(zend_op_array)); + new_op_array->fn_flags &= ~ZEND_ACC_IMMUTABLE; + new_op_array->run_time_cache = (void**)(new_op_array + 1); + memset(new_op_array->run_time_cache, 0, new_op_array->cache_size); + return (zend_function*)new_op_array; + } else { + fbc->op_array.run_time_cache = zend_arena_alloc(&CG(arena), fbc->op_array.cache_size); + memset(fbc->op_array.run_time_cache, 0, fbc->op_array.cache_size); + } } return fbc; } @@ -277,12 +299,12 @@ static void ZEND_FASTCALL zend_jit_fetch_dim_r_helper(zend_array *ht, zval *dim, ZVAL_NULL(result); return; } - ZVAL_COPY_UNREF(result, retval); + ZVAL_COPY_DEREF(result, retval); return; num_index: ZEND_HASH_INDEX_FIND(ht, hval, retval, num_undef); - ZVAL_COPY_UNREF(result, retval); + ZVAL_COPY_DEREF(result, retval); return; num_undef: @@ -347,12 +369,12 @@ static void ZEND_FASTCALL zend_jit_fetch_dim_is_helper(zend_array *ht, zval *dim ZVAL_NULL(result); return; } - ZVAL_COPY_UNREF(result, retval); + ZVAL_COPY_DEREF(result, retval); return; num_index: ZEND_HASH_INDEX_FIND(ht, hval, retval, num_undef); - ZVAL_COPY_UNREF(result, retval); + ZVAL_COPY_DEREF(result, retval); return; num_undef: @@ -1944,9 +1966,9 @@ static void ZEND_FASTCALL zend_jit_verify_arg_slow(zval *arg, zend_op_array *op_ zend_verify_arg_error((zend_function*)op_array, arg_info, arg_num, ce, arg); } -static void ZEND_FASTCALL zend_jit_zval_copy_unref_helper(zval *dst, zval *src) +static void ZEND_FASTCALL zend_jit_zval_copy_deref_helper(zval *dst, zval *src) { - ZVAL_UNREF(src); + ZVAL_DEREF(src); ZVAL_COPY(dst, src); } @@ -1965,7 +1987,7 @@ static void ZEND_FASTCALL zend_jit_fetch_obj_r_slow(zend_object *zobj, zval *off ZVAL_OBJ(&tmp, zobj); retval = zobj->handlers->read_property(&tmp, offset, BP_VAR_R, CACHE_ADDR(cache_slot), result); if (retval != result) { - ZVAL_COPY_UNREF(result, retval); + ZVAL_COPY_DEREF(result, retval); } } } @@ -1987,7 +2009,7 @@ static void ZEND_FASTCALL zend_jit_fetch_obj_r_dynamic(zend_object *zobj, intptr EXPECTED(p->key != NULL) && EXPECTED(ZSTR_LEN(p->key) == Z_STRLEN_P(offset)) && EXPECTED(memcmp(ZSTR_VAL(p->key), Z_STRVAL_P(offset), Z_STRLEN_P(offset)) == 0)))) { - ZVAL_COPY_UNREF(result, &p->val); + ZVAL_COPY_DEREF(result, &p->val); return; } } @@ -1999,7 +2021,7 @@ static void ZEND_FASTCALL zend_jit_fetch_obj_r_dynamic(zend_object *zobj, intptr if (EXPECTED(retval)) { intptr_t idx = (char*)retval - (char*)zobj->properties->arData; CACHE_PTR_EX((void**)((char*)EG(current_execute_data)->run_time_cache + cache_slot) + 1, (void*)ZEND_ENCODE_DYN_PROP_OFFSET(idx)); - ZVAL_COPY_UNREF(result, retval); + ZVAL_COPY_DEREF(result, retval); return; } } @@ -2040,7 +2062,7 @@ static void ZEND_FASTCALL zend_jit_fetch_obj_is_dynamic(zend_object *zobj, intpt EXPECTED(p->key != NULL) && EXPECTED(ZSTR_LEN(p->key) == Z_STRLEN_P(offset)) && EXPECTED(memcmp(ZSTR_VAL(p->key), Z_STRVAL_P(offset), Z_STRLEN_P(offset)) == 0)))) { - ZVAL_COPY_UNREF(result, &p->val); + ZVAL_COPY_DEREF(result, &p->val); return; } } diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 47b3d3eba7b5d..33fc911945bc3 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -8520,21 +8520,14 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, const zend_op *opline, zend |8: if (res_info & MAY_BE_REF) { - | // ZVAL_COPY_UNREF + | // ZVAL_COPY_DEREF if (res_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { | IF_NOT_ZVAL_REFCOUNTED val_addr, >2 } - | GET_ZVAL_PTR r1, val_addr | IF_NOT_ZVAL_TYPE val_addr, IS_REFERENCE, >1 - | cmp dword [r1], 1 - | je >3 - |.cold_code - |3: - | LOAD_ZVAL_ADDR FCARG1a, res_addr - | LOAD_ZVAL_ADDR FCARG2a, val_addr - | EXT_CALL zend_jit_zval_copy_unref_helper, r0 - | jmp >9 - |.code + | GET_Z_PTR r0, r0 + | add r0, offsetof(zend_reference, val) + | IF_NOT_ZVAL_REFCOUNTED val_addr, >2 |1: | GC_ADDREF r1 |2: @@ -9259,7 +9252,7 @@ static int zend_jit_fetch_obj_read(dasm_State **Dst, zend_op *opline, zend_op_ar } | GET_ZVAL_PTR r0, prop_addr | IF_NOT_REFCOUNTED dh, >2 - if (opline->opcode == ZEND_FETCH_OBJ_R) { + if (opline->opcode == ZEND_FETCH_OBJ_R || opline->opcode == ZEND_FETCH_OBJ_IS) { | IF_TYPE dl, IS_REFERENCE, >6 } |1: @@ -9304,7 +9297,7 @@ static int zend_jit_fetch_obj_read(dasm_State **Dst, zend_op *opline, zend_op_ar | lea FCARG2a, [FCARG1a + offset] } | LOAD_ZVAL_ADDR FCARG1a, res_addr - | EXT_CALL zend_jit_zval_copy_unref_helper, r0 + | EXT_CALL zend_jit_zval_copy_deref_helper, r0 | jmp >9 } From 56382e2ed7e5374c73c9f5d5ba3d222fcaf17059 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 26 Jun 2018 19:33:26 +0300 Subject: [PATCH 514/569] Added support for new master changes (incomplete) --- ext/opcache/jit/zend_jit_helpers.c | 10 ++++++++-- ext/opcache/jit/zend_jit_x86.dasc | 18 ++++++++++++------ 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/ext/opcache/jit/zend_jit_helpers.c b/ext/opcache/jit/zend_jit_helpers.c index 460fcac2f7aec..daef33343e5ea 100644 --- a/ext/opcache/jit/zend_jit_helpers.c +++ b/ext/opcache/jit/zend_jit_helpers.c @@ -691,7 +691,9 @@ static void ZEND_FASTCALL zend_jit_fetch_dim_obj_r_helper(zval *container, zval if (retval) { if (result != retval) { - ZVAL_COPY(result, retval); + ZVAL_COPY_DEREF(result, retval); + } else if (UNEXPECTED(Z_ISREF_P(retval))) { + zend_unwrap_reference(retval); } } else { ZVAL_NULL(result); @@ -713,7 +715,9 @@ static void ZEND_FASTCALL zend_jit_fetch_dim_obj_is_helper(zval *container, zval if (retval) { if (result != retval) { - ZVAL_COPY(result, retval); + ZVAL_COPY_DEREF(result, retval); + } else if (UNEXPECTED(Z_ISREF_P(retval))) { + zend_unwrap_reference(result); } } else { ZVAL_NULL(result); @@ -1988,6 +1992,8 @@ static void ZEND_FASTCALL zend_jit_fetch_obj_r_slow(zend_object *zobj, zval *off retval = zobj->handlers->read_property(&tmp, offset, BP_VAR_R, CACHE_ADDR(cache_slot), result); if (retval != result) { ZVAL_COPY_DEREF(result, retval); + } else if (UNEXPECTED(Z_ISREF_P(retval))) { + zend_unwrap_reference(result); } } } diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 33fc911945bc3..baec945e39d9d 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -4395,6 +4395,9 @@ static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, zend_op_ uint32_t var_info = zend_array_element_type(op1_info, 0, 0); zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); + if (op1_info & (MAY_BE_ARRAY_OF_REF|MAY_BE_OBJECT)) { + var_info |= MAY_BE_REF; + } if (!zend_jit_simple_assign(Dst, opline, op_array, ssa, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, op3_addr, val_info, res_addr, 0)) { return 0; } @@ -4402,6 +4405,9 @@ static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, zend_op_ uint32_t var_info = zend_array_element_type(op1_info, 0, 0); zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); + if (op1_info & (MAY_BE_ARRAY_OF_REF|MAY_BE_OBJECT)) { + var_info |= MAY_BE_REF; + } | // value = zend_assign_to_variable(variable_ptr, value, OP_DATA_TYPE); | ZVAL_DEREF FCARG1a, var_info if (!zend_jit_assign_to_variable(Dst, opline, op_array, ssa, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, op3_addr, val_info, res_addr)) { @@ -4607,6 +4613,9 @@ static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, zend_ uint32_t var_info = zend_array_element_type(op1_info, 0, 0); uint32_t var_def_info = zend_array_element_type(OP1_DEF_INFO(), 1, 0); + if (op1_info & (MAY_BE_ARRAY_OF_REF|MAY_BE_OBJECT)) { + var_info |= MAY_BE_REF; + } |6: if (opline->op2_type == IS_UNUSED) { | // var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval)); @@ -8519,16 +8528,15 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, const zend_op *opline, zend zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); |8: - if (res_info & MAY_BE_REF) { + if (op1_info & MAY_BE_ARRAY_OF_REF) { | // ZVAL_COPY_DEREF - if (res_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { - | IF_NOT_ZVAL_REFCOUNTED val_addr, >2 - } + | IF_NOT_ZVAL_REFCOUNTED val_addr, >2 | IF_NOT_ZVAL_TYPE val_addr, IS_REFERENCE, >1 | GET_Z_PTR r0, r0 | add r0, offsetof(zend_reference, val) | IF_NOT_ZVAL_REFCOUNTED val_addr, >2 |1: + | GET_Z_PTR r1, r0 | GC_ADDREF r1 |2: | ZVAL_COPY_VALUE res_addr, -1, val_addr, MAY_BE_ANY, ZREG_R1, ZREG_R2 @@ -9289,8 +9297,6 @@ static int zend_jit_fetch_obj_read(dasm_State **Dst, zend_op *opline, zend_op_ar if (opline->opcode == ZEND_FETCH_OBJ_R) { |6: - | cmp dword [r0], 1 - | jne <1 if (offset == ZEND_WRONG_PROPERTY_OFFSET) { | mov FCARG2a, FCARG1a } else { From e43b6cf7cbd36d5c35c4a406391d56e587754393 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 26 Jun 2018 21:02:38 +0300 Subject: [PATCH 515/569] Added support for new master changes (incomplete) --- ext/opcache/jit/zend_jit_helpers.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ext/opcache/jit/zend_jit_helpers.c b/ext/opcache/jit/zend_jit_helpers.c index daef33343e5ea..97dce19b34d66 100644 --- a/ext/opcache/jit/zend_jit_helpers.c +++ b/ext/opcache/jit/zend_jit_helpers.c @@ -1993,7 +1993,7 @@ static void ZEND_FASTCALL zend_jit_fetch_obj_r_slow(zend_object *zobj, zval *off if (retval != result) { ZVAL_COPY_DEREF(result, retval); } else if (UNEXPECTED(Z_ISREF_P(retval))) { - zend_unwrap_reference(result); + zend_unwrap_reference(retval); } } } @@ -2046,7 +2046,9 @@ static void ZEND_FASTCALL zend_jit_fetch_obj_is_slow(zend_object *zobj, zval *of ZVAL_OBJ(&tmp, zobj); retval = zobj->handlers->read_property(&tmp, offset, BP_VAR_IS, CACHE_ADDR(cache_slot), result); if (retval != result) { - ZVAL_COPY(result, retval); + ZVAL_COPY_DEREF(result, retval); + } else if (UNEXPECTED(Z_ISREF_P(retval))) { + zend_unwrap_reference(retval); } } } From 60c7b9e1b36e97ee615b48f4abb228718d49943e Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 26 Jun 2018 22:14:22 +0300 Subject: [PATCH 516/569] Fixed JIT support for new master changes --- ext/opcache/jit/zend_jit_x86.dasc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index baec945e39d9d..0408ea84cd939 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -9295,7 +9295,7 @@ static int zend_jit_fetch_obj_read(dasm_State **Dst, zend_op *opline, zend_op_ar } | jmp >9 - if (opline->opcode == ZEND_FETCH_OBJ_R) { + if (opline->opcode == ZEND_FETCH_OBJ_R || opline->opcode == ZEND_FETCH_OBJ_IS) { |6: if (offset == ZEND_WRONG_PROPERTY_OFFSET) { | mov FCARG2a, FCARG1a From fb92f584a1988f9fad036a60ca927adc2efaf8ac Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 27 Jun 2018 00:56:02 +0300 Subject: [PATCH 517/569] Fixed CPU stack alignment --- ext/opcache/jit/zend_jit_x86.dasc | 108 ++++++++++++++++++++++++++++++ 1 file changed, 108 insertions(+) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 0408ea84cd939..a74d2cd24c13a 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -1183,10 +1183,16 @@ static void* dasm_labels[zend_lb_MAX]; | .if X64 | mov CARG3d, lineno | .else +| sub r4, 12 | push lineno | .endif || } | EXT_CALL _zval_copy_ctor_func, r0 +|| if (ZEND_DEBUG) { +| .if not(X64) +| add r4, 12 +| .endif +|| } |.endmacro |.macro ZVAL_DEREF, reg, info @@ -1247,10 +1253,16 @@ static void* dasm_labels[zend_lb_MAX]; | .if X64 | mov CARG3d, lineno | .else +| sub r4, 12 | push lineno | .endif || } | EXT_CALL _zval_dtor_func, r0 +|| if (ZEND_DEBUG) { +| .if not(X64) +| add r4, 12 +| .endif +|| } || } while(0); |.endmacro @@ -1399,11 +1411,15 @@ static void* dasm_labels[zend_lb_MAX]; | xor CARG4, CARG4 | xor CARG5, CARG5 | .else +| sub r4, 4 | push 0 | push 0 | push opline->lineno | .endif | EXT_CALL _efree, r0 +| .if not(X64) +| add r4, 4 +| .endif ||#else | EXT_CALL _efree_24, r0 ||#endif @@ -1424,11 +1440,15 @@ static void* dasm_labels[zend_lb_MAX]; | xor CARG4, CARG4 | xor CARG5, CARG5 | .else +| sub r4, 4 | push 0 | push 0 | push opline->lineno | .endif | EXT_CALL _emalloc, r0 +| .if not(X64) +| add r4, 4 +| .endif ||#else || if (size == 24) { | EXT_CALL _emalloc_24, r0 @@ -3183,6 +3203,7 @@ static int zend_jit_math_helper(dasm_State **Dst, |.if X64 | LOAD_ZVAL_ADDR CARG3, op2_addr |.else + | sub r4, 12 | PUSH_ZVAL_ADDR op2_addr, r0 |.endif if (opline->opcode == ZEND_ADD || opline->opcode == ZEND_ASSIGN_ADD) { @@ -3196,6 +3217,9 @@ static int zend_jit_math_helper(dasm_State **Dst, } else { ZEND_ASSERT(0); } + |.if not(X64) + | add r4, 12 + |.endif | FREE_OP op1_type, op1, op1_info, 0, op_array, opline | FREE_OP op2_type, op2, op2_info, 0, op_array, opline if (zend_may_throw(opline, op_array, ssa)) { @@ -3516,6 +3540,7 @@ static int zend_jit_long_math_helper(dasm_State **Dst, |.if X64 | LOAD_ZVAL_ADDR CARG3, op2_addr |.else + | sub r4, 12 | PUSH_ZVAL_ADDR op2_addr, r0 |.endif if (opline->opcode == ZEND_BW_OR || opline->opcode == ZEND_ASSIGN_BW_OR) { @@ -3533,6 +3558,9 @@ static int zend_jit_long_math_helper(dasm_State **Dst, } else { ZEND_ASSERT(0); } + |.if not(X64) + | add r4, 12 + |.endif | FREE_OP op1_type, op1, op1_info, 0, op_array, opline | FREE_OP op2_type, op2, op2_info, 0, op_array, opline if (zend_may_throw(opline, op_array, ssa)) { @@ -3649,9 +3677,13 @@ static int zend_jit_concat_helper(dasm_State **Dst, |.if X64 | LOAD_ZVAL_ADDR CARG3, op2_addr |.else + | sub r4, 12 | PUSH_ZVAL_ADDR op2_addr, r0 |.endif | EXT_CALL zend_jit_fast_concat_helper, r0 + |.if not(X64) + | add r4, 12 + |.endif } | FREE_OP op1_type, op1, op1_info, 0, op_array, opline | FREE_OP op2_type, op2, op2_info, 0, op_array, opline @@ -3672,9 +3704,13 @@ static int zend_jit_concat_helper(dasm_State **Dst, |.if X64 | LOAD_ZVAL_ADDR CARG3, op2_addr |.else + | sub r4, 12 | PUSH_ZVAL_ADDR op2_addr, r0 |.endif | EXT_CALL concat_function, r0 + |.if not(X64) + | add r4, 12 + |.endif | FREE_OP op1_type, op1, op1_info, 0, op_array, opline | FREE_OP op2_type, op2, op2_info, 0, op_array, opline if (zend_may_throw(opline, op_array, ssa)) { @@ -3886,9 +3922,14 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o | LOAD_ADDR CARG4, filename | mov CARG5d, opline->lineno |.else + | sub r4, 4 | push opline->lineno | push filename |.endif + } else { + |.if not(X64) + | sub r4, 12 + |.endif } |.if X64 | LOAD_ADDR CARG3, &EG(uninitialized_zval) @@ -3896,6 +3937,15 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o | PUSH_ADDR &EG(uninitialized_zval), r0 |.endif | EXT_CALL _zend_hash_index_add_new, r0 + if (ZEND_DEBUG) { + |.if not(X64) + | add r4, 4 + |.endif + } else { + |.if not(X64) + | add r4, 12 + |.endif + } if (op1_info & MAY_BE_ARRAY_KEY_LONG) { | jmp >8 |4: @@ -4026,9 +4076,13 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o |.if X64 | LOAD_ZVAL_ADDR CARG3, res_addr |.else + | sub r4, 12 | PUSH_ZVAL_ADDR res_addr, r0 |.endif | EXT_CALL zend_jit_fetch_dim_r_helper, r0 + |.if not(X64) + | add r4, 12 + |.endif | jmp >9 break; case BP_JIT_IS: @@ -4042,9 +4096,13 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o |.if X64 | LOAD_ZVAL_ADDR CARG3, res_addr |.else + | sub r4, 12 | PUSH_ZVAL_ADDR res_addr, r0 |.endif | EXT_CALL zend_jit_fetch_dim_is_helper, r0 + |.if not(X64) + | add r4, 12 + |.endif | jmp >9 break; case BP_VAR_RW: @@ -4472,6 +4530,9 @@ static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, zend_op_ } else { | LOAD_ZVAL_ADDR FCARG2a, op2_addr } + |.if not(X64) + | sub r4, 8 + |.endif if (opline->result_type == IS_UNUSED) { |.if X64 | xor CARG4, CARG4 @@ -4494,6 +4555,9 @@ static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, zend_op_ | PUSH_ZVAL_ADDR op3_addr, r0 |.endif | EXT_CALL zend_jit_assign_dim_helper, r0 + |.if not(X64) + | add r4, 8 + |.endif #ifdef ZEND_JIT_USE_RC_INFERENCE if (((opline+1)->op1_type & (IS_TMP_VAR|IS_VAR)) && (val_info & MAY_BE_RC1)) { @@ -4626,11 +4690,17 @@ static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, zend_ | LOAD_ADDR CARG3, filename | mov CARG4d, opline->lineno |.else + | add r4, 8 | push opline->lineno | push filename |.endif } | EXT_CALL _zend_hash_next_index_insert, r0 + || if (ZEND_DEBUG) { + | .if not(X64) + | sub r4, 8 + | .endif + || } | // if (UNEXPECTED(!var_ptr)) { | test r0, r0 | jz >1 @@ -4752,6 +4822,7 @@ static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, zend_ |.if X64 | LOAD_ZVAL_ADDR CARG3, op3_addr |.else + | sub r4, 12 | PUSH_ZVAL_ADDR op3_addr, r0 |.endif switch (opline->opcode) { @@ -4791,6 +4862,9 @@ static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, zend_ default: ZEND_ASSERT(0); } + |.if not(X64) + | add r4, 12 + |.endif } if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { @@ -5766,6 +5840,7 @@ static int zend_jit_cmp(dasm_State **Dst, const zend_op *opline, int b, int *opn |.if X64 | LOAD_ADDR CARG3, &EG(uninitialized_zval) |.else + | sub r4, 12 | PUSH_ADDR &EG(uninitialized_zval), r0 |.endif | jmp >2 @@ -5773,6 +5848,7 @@ static int zend_jit_cmp(dasm_State **Dst, const zend_op *opline, int b, int *opn |.if X64 | LOAD_ZVAL_ADDR CARG3, op2_addr |.else + | sub r4, 12 | PUSH_ZVAL_ADDR op2_addr, r0 |.endif |2: @@ -5780,11 +5856,15 @@ static int zend_jit_cmp(dasm_State **Dst, const zend_op *opline, int b, int *opn |.if X64 | LOAD_ZVAL_ADDR CARG3, op2_addr |.else + | sub r4, 12 | PUSH_ZVAL_ADDR op2_addr, r0 |.endif } | LOAD_ZVAL_ADDR FCARG1a, res_addr | EXT_CALL compare_function, r0 + |.if not(X64) + | add r4, 12 + |.endif if (opline->opcode != ZEND_CASE) { | FREE_OP opline->op1_type, opline->op1, op1_info, 0, op_array, opline } @@ -8450,6 +8530,7 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, const zend_op *opline, zend |.if X64 | LOAD_ZVAL_ADDR CARG3, res_addr |.else + | sub r4, 12 | PUSH_ZVAL_ADDR res_addr, r0 |.endif if (opline->opcode == ZEND_FETCH_DIM_R) { @@ -8459,6 +8540,9 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, const zend_op *opline, zend } else { ZEND_ASSERT(0); } + |.if not(X64) + | add r4, 12 + |.endif if ((op1_info & MAY_BE_ARRAY) || (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_STRING)))) { | jmp >9 // END @@ -8478,6 +8562,7 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, const zend_op *opline, zend |.if X64 | LOAD_ZVAL_ADDR CARG3, res_addr |.else + | sub r4, 12 | PUSH_ZVAL_ADDR res_addr, r0 |.endif if (opline->opcode == ZEND_FETCH_DIM_R) { @@ -8487,6 +8572,9 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, const zend_op *opline, zend } else { ZEND_ASSERT(0); } + |.if not(X64) + | add r4, 12 + |.endif if ((op1_info & MAY_BE_ARRAY) || (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_STRING|MAY_BE_OBJECT)))) { | jmp >9 // END @@ -8841,9 +8929,13 @@ static int zend_jit_bind_global(dasm_State **Dst, const zend_op *opline, zend_op |.if X64 | mov CARG3, opline->extended_value |.else + | sub r4, 12 | push opline->extended_value |.endif | EXT_CALL zend_jit_fetch_global_helper, r0 + |.if not(X64) + | add r4, 12 + |.endif | jmp <1 |.code @@ -8900,11 +8992,15 @@ static int zend_jit_recv(dasm_State **Dst, const zend_op *opline, zend_op_array | LOAD_ADDR CARG4, (ptrdiff_t)arg_info | mov CARG5, r0 |.else + | sub r4, 4 | push r0 | push (ptrdiff_t)arg_info | push arg_num |.endif | EXT_CALL zend_jit_verify_arg_object, r0 + |.if not(X64) + | add r4, 4 + |.endif if (!zend_jit_check_exception(Dst)) { return 0; } @@ -9034,11 +9130,15 @@ static int zend_jit_recv_init(dasm_State **Dst, const zend_op *opline, zend_op_a | LOAD_ADDR CARG4, (ptrdiff_t)arg_info | mov CARG5, r0 |.else + | sub r4, 4 | push r0 | push (ptrdiff_t)arg_info | push arg_num |.endif | EXT_CALL zend_jit_verify_arg_object, r0 + |.if not(X64) + | add r4, 4 + |.endif } } while (0); } @@ -9282,6 +9382,7 @@ static int zend_jit_fetch_obj_read(dasm_State **Dst, zend_op *opline, zend_op_ar | LOAD_ZVAL_ADDR CARG3, res_addr | mov CARG4, opline->extended_value |.else + | sub r4, 8 | push opline->extended_value | PUSH_ZVAL_ADDR res_addr, r0 |.endif @@ -9293,6 +9394,9 @@ static int zend_jit_fetch_obj_read(dasm_State **Dst, zend_op *opline, zend_op_ar } else { ZEND_ASSERT(0); } + |.if not(X64) + | add r4, 8 + |.endif | jmp >9 if (opline->opcode == ZEND_FETCH_OBJ_R || opline->opcode == ZEND_FETCH_OBJ_IS) { @@ -9345,6 +9449,7 @@ static int zend_jit_fetch_obj_read(dasm_State **Dst, zend_op *opline, zend_op_ar | LOAD_ZVAL_ADDR CARG4, res_addr | mov CARG5, opline->extended_value |.else + | sub r4, 4 | push opline->extended_value | PUSH_ZVAL_ADDR res_addr, r0 | PUSH_ADDR member, r0 @@ -9355,6 +9460,9 @@ static int zend_jit_fetch_obj_read(dasm_State **Dst, zend_op *opline, zend_op_ar } else if (opline->opcode == ZEND_FETCH_OBJ_IS) { | EXT_CALL zend_jit_fetch_obj_is_dynamic, r0 } + |.if not(X64) + | add r4, 4 + |.endif | jmp >9 } From fd69f1e9d9c1354bccfd965548ad358d8c83cd09 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 23 Jul 2018 11:50:12 +0300 Subject: [PATCH 518/569] Fixed JIT support for recent master changes --- ext/opcache/jit/zend_jit_internal.h | 2 + ext/opcache/jit/zend_jit_vm_helpers.c | 112 ++++++++++++++++++++++++++ ext/opcache/jit/zend_jit_x86.dasc | 35 +++++--- 3 files changed, 137 insertions(+), 12 deletions(-) diff --git a/ext/opcache/jit/zend_jit_internal.h b/ext/opcache/jit/zend_jit_internal.h index dda0135bb6f58..f6b51d5183d82 100644 --- a/ext/opcache/jit/zend_jit_internal.h +++ b/ext/opcache/jit/zend_jit_internal.h @@ -46,6 +46,8 @@ void ZEND_FASTCALL zend_jit_deprecated_or_abstract_helper(void); void ZEND_FASTCALL zend_jit_profile_helper(void); void ZEND_FASTCALL zend_jit_func_counter_helper(void); void ZEND_FASTCALL zend_jit_loop_counter_helper(void); +void ZEND_FASTCALL zend_jit_get_constant(const zval *key, uint32_t flags); +int ZEND_FASTCALL zend_jit_check_constant(const zval *key); #endif /* ZEND_JIT_INTERNAL_H */ diff --git a/ext/opcache/jit/zend_jit_vm_helpers.c b/ext/opcache/jit/zend_jit_vm_helpers.c index 7e1a93b03df45..15d07df6b25ab 100644 --- a/ext/opcache/jit/zend_jit_vm_helpers.c +++ b/ext/opcache/jit/zend_jit_vm_helpers.c @@ -21,6 +21,8 @@ #include "Zend/zend_exceptions.h" #include "Zend/zend_vm.h" #include "Zend/zend_closures.h" +#include "Zend/zend_constants.h" +#include "Zend/zend_API.h" #include #include "Optimizer/zend_func_info.h" @@ -209,6 +211,116 @@ void ZEND_FASTCALL zend_jit_loop_counter_helper(void) } } +static zend_always_inline int _zend_quick_get_constant( + const zval *key, uint32_t flags, int check_defined_only) +{ + zval *zv; + const zval *orig_key = key; + zend_constant *c = NULL; + + zv = zend_hash_find_ex(EG(zend_constants), Z_STR_P(key), 1); + if (zv) { + c = (zend_constant*)Z_PTR_P(zv); + } else { + key++; + zv = zend_hash_find_ex(EG(zend_constants), Z_STR_P(key), 1); + if (zv && (((zend_constant*)Z_PTR_P(zv))->flags & CONST_CS) == 0) { + c = (zend_constant*)Z_PTR_P(zv); + } else { + if ((flags & (IS_CONSTANT_IN_NAMESPACE|IS_CONSTANT_UNQUALIFIED)) == (IS_CONSTANT_IN_NAMESPACE|IS_CONSTANT_UNQUALIFIED)) { + key++; + zv = zend_hash_find_ex(EG(zend_constants), Z_STR_P(key), 1); + if (zv) { + c = (zend_constant*)Z_PTR_P(zv); + } else { + key++; + zv = zend_hash_find_ex(EG(zend_constants), Z_STR_P(key), 1); + if (zv && (((zend_constant*)Z_PTR_P(zv))->flags & CONST_CS) == 0) { + c = (zend_constant*)Z_PTR_P(zv); + } + } + } + } + } + + if (!c) { + if (!check_defined_only) { + if ((opline->op1.num & IS_CONSTANT_UNQUALIFIED) != 0) { + char *actual = (char *)zend_memrchr(Z_STRVAL_P(RT_CONSTANT(opline, opline->op2)), '\\', Z_STRLEN_P(RT_CONSTANT(opline, opline->op2))); + if (!actual) { + ZVAL_STR_COPY(EX_VAR(opline->result.var), Z_STR_P(RT_CONSTANT(opline, opline->op2))); + } else { + actual++; + ZVAL_STRINGL(EX_VAR(opline->result.var), + actual, Z_STRLEN_P(RT_CONSTANT(opline, opline->op2)) - (actual - Z_STRVAL_P(RT_CONSTANT(opline, opline->op2)))); + } + /* non-qualified constant - allow text substitution */ + zend_error(E_WARNING, "Use of undefined constant %s - assumed '%s' (this will throw an Error in a future version of PHP)", + Z_STRVAL_P(EX_VAR(opline->result.var)), Z_STRVAL_P(EX_VAR(opline->result.var))); + } else { + zend_throw_error(NULL, "Undefined constant '%s'", Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))); + ZVAL_UNDEF(EX_VAR(opline->result.var)); + } + } + return FAILURE; + } + + if (!check_defined_only) { + ZVAL_COPY_OR_DUP(EX_VAR(opline->result.var), &c->value); + if (!(c->flags & (CONST_CS|CONST_CT_SUBST))) { + const char *ns_sep; + size_t shortname_offset; + size_t shortname_len; + zend_bool is_deprecated; + + if (flags & IS_CONSTANT_UNQUALIFIED) { + const zval *access_key; + + if (!(flags & IS_CONSTANT_IN_NAMESPACE)) { + access_key = orig_key - 1; + } else { + if (key < orig_key + 2) { + goto check_short_name; + } else { + access_key = orig_key + 2; + } + } + is_deprecated = !zend_string_equals(c->name, Z_STR_P(access_key)); + } else { +check_short_name: + ns_sep = zend_memrchr(ZSTR_VAL(c->name), '\\', ZSTR_LEN(c->name)); + ZEND_ASSERT(ns_sep); + /* Namespaces are always case-insensitive. Only compare shortname. */ + shortname_offset = ns_sep - ZSTR_VAL(c->name) + 1; + shortname_len = ZSTR_LEN(c->name) - shortname_offset; + + is_deprecated = memcmp(ZSTR_VAL(c->name) + shortname_offset, Z_STRVAL_P(orig_key - 1) + shortname_offset, shortname_len) != 0; + } + + if (is_deprecated) { + zend_error(E_DEPRECATED, + "Case-insensitive constants are deprecated. " + "The correct casing for this constant is \"%s\"", + ZSTR_VAL(c->name)); + return SUCCESS; + } + } + } + + CACHE_PTR(opline->extended_value, c); + return SUCCESS; +} + +void ZEND_FASTCALL zend_jit_get_constant(const zval *key, uint32_t flags) +{ + _zend_quick_get_constant(key, flags, 0); +} + +int ZEND_FASTCALL zend_jit_check_constant(const zval *key) +{ + return _zend_quick_get_constant(key, 0, 1); +} + /* * Local variables: * tab-width: 4 diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index a74d2cd24c13a..0db60bed4bfbf 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -1257,7 +1257,7 @@ static void* dasm_labels[zend_lb_MAX]; | push lineno | .endif || } -| EXT_CALL _zval_dtor_func, r0 +| EXT_CALL _rc_dtor_func, r0 || if (ZEND_DEBUG) { | .if not(X64) | add r4, 12 @@ -4527,6 +4527,9 @@ static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, zend_op_ } if (opline->op2_type == IS_UNUSED) { | xor FCARG2a, FCARG2a + } else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { + ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); + | LOAD_ADDR FCARG2a, (Z_ZV(op2_addr) + 1) } else { | LOAD_ZVAL_ADDR FCARG2a, op2_addr } @@ -4816,6 +4819,9 @@ static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, zend_ } if (opline->op2_type == IS_UNUSED) { | xor FCARG2a, FCARG2a + } else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { + ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); + | LOAD_ADDR FCARG2a, (Z_ZV(op2_addr) + 1) } else { | LOAD_ZVAL_ADDR FCARG2a, op2_addr } @@ -7914,17 +7920,14 @@ static int zend_jit_defined(dasm_State **Dst, const zend_op *opline, int b, int |.cold_code |1: | LOAD_ADDR FCARG1a, zv - | xor FCARG2d, FCARG2d - | EXT_CALL zend_quick_get_constant, r0 + | EXT_CALL zend_jit_check_constant, r0 | test r0, r0 if (smart_branch) { if (undefined_label != (uint32_t)-1) { - | jz =>undefined_label + | jnz =>undefined_label } else { - | jz >3 + | jnz >3 } - | mov r1, EX->run_time_cache - | mov aword [r1 + opline->extended_value], r0 if (defined_label != (uint32_t)-1) { | jmp =>defined_label } else { @@ -7932,9 +7935,7 @@ static int zend_jit_defined(dasm_State **Dst, const zend_op *opline, int b, int } } else { res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1); - | jz >2 - | mov r1, EX->run_time_cache - | mov aword [r1 + opline->extended_value], r0 + | jnz >2 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE | jmp >3 |2: @@ -8558,7 +8559,12 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, const zend_op *opline, zend if (Z_REG(op1_addr) != ZREG_FCARG1a) { | LOAD_ZVAL_ADDR FCARG1a, op1_addr } - | LOAD_ZVAL_ADDR FCARG2a, op2_addr + if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { + ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); + | LOAD_ADDR FCARG2a, (Z_ZV(op2_addr) + 1) + } else { + | LOAD_ZVAL_ADDR FCARG2a, op2_addr + } |.if X64 | LOAD_ZVAL_ADDR CARG3, res_addr |.else @@ -8702,7 +8708,12 @@ static int zend_jit_isset_isempty_dim(dasm_State **Dst, const zend_op *opline, i if (Z_REG(op1_addr) != ZREG_FCARG1a) { | LOAD_ZVAL_ADDR FCARG1a, op1_addr } - | LOAD_ZVAL_ADDR FCARG2a, op2_addr + if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { + ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); + | LOAD_ADDR FCARG2a, (Z_ZV(op2_addr) + 1) + } else { + | LOAD_ZVAL_ADDR FCARG2a, op2_addr + } | EXT_CALL zend_jit_isset_dim_helper, r0 | test r0, r0 | jz >9 From 00f9d6dffcb66217bb3a63611caef2bc144cd57f Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 23 Jul 2018 15:26:22 +0300 Subject: [PATCH 519/569] Removed useless filename and lineno arguments, used in DEBUG build. --- ext/opcache/jit/zend_jit_x86.dasc | 212 +++++------------------------- 1 file changed, 32 insertions(+), 180 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 0db60bed4bfbf..0ad4fbc8bdbf1 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -1172,29 +1172,6 @@ static void* dasm_labels[zend_lb_MAX]; || } |.endmacro -// zval should be in FCARG1a -|.macro ZVAL_COPY_CTOR_FUNC, filename, lineno // arg1 must be in FCARG1a -|| if (ZEND_DEBUG) { -|| if (filename) { -| LOAD_ADDR FCARG2a, ZSTR_VAL((zend_string*)filename) -|| } else { -| xor FCARG2a, FCARG2a -|| } -| .if X64 -| mov CARG3d, lineno -| .else -| sub r4, 12 -| push lineno -| .endif -|| } -| EXT_CALL _zval_copy_ctor_func, r0 -|| if (ZEND_DEBUG) { -| .if not(X64) -| add r4, 12 -| .endif -|| } -|.endmacro - |.macro ZVAL_DEREF, reg, info || if (info & MAY_BE_REF) { | IF_NOT_Z_TYPE, reg, IS_REFERENCE, >1 @@ -1213,7 +1190,7 @@ static void* dasm_labels[zend_lb_MAX]; |.endmacro // zval should be in FCARG1a -|.macro ZVAL_DTOR_FUNC, var_info, filename, opline // arg1 must be in FCARG1a +|.macro ZVAL_DTOR_FUNC, var_info, opline // arg1 must be in FCARG1a || do { || if (has_concrete_type((var_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { || zend_uchar type = concrete_type((var_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)); @@ -1243,30 +1220,11 @@ static void* dasm_labels[zend_lb_MAX]; || if (opline) { | SAVE_VALID_OPLINE opline || } -|| if (ZEND_DEBUG) { -|| uint32_t lineno = opline ? ((zend_op*)opline)->lineno : 0; -|| if (filename) { -| LOAD_ADDR FCARG2a, ZSTR_VAL((zend_string*)filename) -|| } else { -| xor FCARG2a, FCARG2a -|| } -| .if X64 -| mov CARG3d, lineno -| .else -| sub r4, 12 -| push lineno -| .endif -|| } -| EXT_CALL _rc_dtor_func, r0 -|| if (ZEND_DEBUG) { -| .if not(X64) -| add r4, 12 -| .endif -|| } +| EXT_CALL rc_dtor_func, r0 || } while(0); |.endmacro -|.macro ZVAL_PTR_DTOR, addr, op_info, gc, cold, safe, filename, opline +|.macro ZVAL_PTR_DTOR, addr, op_info, gc, cold, safe, opline || if ((op_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { || if ((op_info) & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { | // if (Z_REFCOUNTED_P(cv)) { @@ -1294,7 +1252,7 @@ static void* dasm_labels[zend_lb_MAX]; | SET_ZVAL_TYPE_INFO addr, IS_NULL || } | // zval_dtor_func(r); -| ZVAL_DTOR_FUNC op_info, filename, opline +| ZVAL_DTOR_FUNC op_info, opline || if (gc && RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT))) { | jmp >4 || } @@ -1325,11 +1283,11 @@ static void* dasm_labels[zend_lb_MAX]; |.macro FREE_OP, op_type, op, op_info, cold, op_array, opline || if (op_type & (IS_VAR|IS_TMP_VAR)) { -| ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_FP, op.var), op_info, 0, cold, 0, op_array->filename, opline +| ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_FP, op.var), op_info, 0, cold, 0, opline || } |.endmacro -|.macro SEPARATE_ZVAL_NOREF, addr, op_info, cold, filename, lineno +|.macro SEPARATE_ZVAL_NOREF, addr, op_info, cold || if ((op_info & MAY_BE_ARRAY) && RC_MAY_BE_N(op_info)) { || if (cold) { | IF_ZVAL_TYPE addr, IS_ARRAY, >1 @@ -1351,7 +1309,7 @@ static void* dasm_labels[zend_lb_MAX]; || } else { | LOAD_ZVAL_ADDR FCARG1a, addr || } -| ZVAL_COPY_CTOR_FUNC filename, lineno +| EXT_CALL zval_copy_ctor_func, r0 || if (Z_REG(addr) == ZREG_FCARG1a) { | mov FCARG1a, aword [r4] // restore || } @@ -1363,7 +1321,7 @@ static void* dasm_labels[zend_lb_MAX]; || } |.endmacro -|.macro SEPARATE_ARRAY, addr, op_info, cold, filename, lineno +|.macro SEPARATE_ARRAY, addr, op_info, cold || if (RC_MAY_BE_N(op_info)) { || zend_reg tmp_reg; || @@ -1387,7 +1345,7 @@ static void* dasm_labels[zend_lb_MAX]; || } else { | LOAD_ZVAL_ADDR FCARG1a, addr || } -| ZVAL_COPY_CTOR_FUNC filename, lineno +| EXT_CALL zval_copy_ctor_func, r0 || if (Z_REG(addr) == ZREG_FCARG1a) { | mov FCARG1a, aword [r4] // restore || } @@ -1724,7 +1682,7 @@ static int zend_jit_throw_cannot_pass_by_ref_stub(dasm_State **Dst) | mov r0, OP:RX->op1.var |.endif | add r0, FP - | ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0), MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, 0, 0, NULL, NULL + | ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0), MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, 0, 0, NULL |9: | jmp ->exception_handler @@ -3177,7 +3135,7 @@ static int zend_jit_math_helper(dasm_State **Dst, |6: | SAVE_VALID_OPLINE opline if (separate_op1) { - | SEPARATE_ZVAL_NOREF op1_addr, op1_info, 0, op_array->filename, opline->lineno + | SEPARATE_ZVAL_NOREF op1_addr, op1_info, 0 } if (Z_MODE(res_addr) == IS_REG) { zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); @@ -3514,7 +3472,7 @@ static int zend_jit_long_math_helper(dasm_State **Dst, |6: | SAVE_VALID_OPLINE opline if (separate_op1) { - | SEPARATE_ZVAL_NOREF op1_addr, op1_info, 0, op_array->filename, opline->lineno + | SEPARATE_ZVAL_NOREF op1_addr, op1_info, 0 } if (Z_MODE(res_addr) == IS_REG) { zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); @@ -3916,36 +3874,16 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o case BP_VAR_W: |2: | //retval = zend_hash_index_add_new(ht, hval, &EG(uninitialized_zval)); - if (ZEND_DEBUG) { - const char *filename = op_array->filename ? op_array->filename->val : NULL; - |.if X64 - | LOAD_ADDR CARG4, filename - | mov CARG5d, opline->lineno - |.else - | sub r4, 4 - | push opline->lineno - | push filename - |.endif - } else { - |.if not(X64) - | sub r4, 12 - |.endif - } |.if X64 | LOAD_ADDR CARG3, &EG(uninitialized_zval) |.else + | sub r4, 12 | PUSH_ADDR &EG(uninitialized_zval), r0 |.endif - | EXT_CALL _zend_hash_index_add_new, r0 - if (ZEND_DEBUG) { - |.if not(X64) - | add r4, 4 - |.endif - } else { - |.if not(X64) - | add r4, 12 - |.endif - } + | EXT_CALL zend_hash_index_add_new, r0 + |.if not(X64) + | add r4, 12 + |.endif if (op1_info & MAY_BE_ARRAY_KEY_LONG) { | jmp >8 |4: @@ -4301,7 +4239,7 @@ static int zend_jit_assign_to_variable(dasm_State **Dst, | cmp dword [FCARG1a], 0 | jnz >3 } - | ZVAL_DTOR_FUNC var_info, op_array->filename, opline + | ZVAL_DTOR_FUNC var_info, opline | jmp >3 |4: } @@ -4378,7 +4316,7 @@ static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, zend_op_ if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7 } - | SEPARATE_ARRAY op1_addr, op1_info, 1, op_array->filename, opline->lineno + | SEPARATE_ARRAY op1_addr, op1_info, 1 } else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE)) { if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { | CMP_ZVAL_TYPE op1_addr, IS_FALSE @@ -4388,23 +4326,8 @@ static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, zend_op_ if (Z_REG(op1_addr) != ZREG_FP) { | mov [r4], Ra(Z_REG(op1_addr)) // save } - if (ZEND_DEBUG) { - const char *filename = op_array->filename ? op_array->filename->val : NULL; - |.if X64 - | mov CARG3d, opline->lineno - |.else - | sub r4, 12 - | push opline->lineno - |.endif - | LOAD_ADDR FCARG2a, filename - } | mov FCARG1d, 8 | EXT_CALL _zend_new_array, r0 - if (ZEND_DEBUG) { - |.if not X64 - | add r4, 12 - |.endif - } if (Z_REG(op1_addr) != ZREG_FP) { | mov Ra(Z_REG(op1_addr)), [r4] // restore } @@ -4418,17 +4341,7 @@ static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, zend_op_ if (opline->op2_type == IS_UNUSED) { | // var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval)); | LOAD_ADDR FCARG2a, &EG(uninitialized_zval) - if (ZEND_DEBUG) { - const char *filename = op_array->filename ? op_array->filename->val : NULL; - |.if X64 - | LOAD_ADDR CARG3, filename - | mov CARG4d, opline->lineno - |.else - | push opline->lineno - | push filename - |.endif - } - | EXT_CALL _zend_hash_next_index_insert, r0 + | EXT_CALL zend_hash_next_index_insert, r0 | // if (UNEXPECTED(!var_ptr)) { | test r0, r0 | jz >1 @@ -4492,23 +4405,8 @@ static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, zend_op_ if (Z_REG(op1_addr) != ZREG_FP) { | mov [r4], Ra(Z_REG(op1_addr)) // save } - if (ZEND_DEBUG) { - const char *filename = op_array->filename ? op_array->filename->val : NULL; - |.if X64 - | mov CARG3d, opline->lineno - |.else - | sub r4, 12 - | push opline->lineno - |.endif - | LOAD_ADDR FCARG2a, filename - } | mov FCARG1d, 8 | EXT_CALL _zend_new_array, r0 - if (ZEND_DEBUG) { - |.if not X64 - | add r4, 12 - |.endif - } if (Z_REG(op1_addr) != ZREG_FP) { | mov Ra(Z_REG(op1_addr)), [r4] // restore } @@ -4632,7 +4530,7 @@ static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, zend_ if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7 } - | SEPARATE_ARRAY op1_addr, op1_info, 1, op_array->filename, opline->lineno + | SEPARATE_ARRAY op1_addr, op1_info, 1 } else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE)) { if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { | CMP_ZVAL_TYPE op1_addr, IS_FALSE @@ -4651,23 +4549,8 @@ static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, zend_ if (Z_REG(op1_addr) != ZREG_FP) { | mov [r4], Ra(Z_REG(op1_addr)) // save } - if (ZEND_DEBUG) { - const char *filename = op_array->filename ? op_array->filename->val : NULL; - |.if X64 - | mov CARG3d, opline->lineno - |.else - | sub r4, 12 - | push opline->lineno - |.endif - | LOAD_ADDR FCARG2a, filename - } | mov FCARG1d, 8 | EXT_CALL _zend_new_array, r0 - if (ZEND_DEBUG) { - |.if not X64 - | add r4, 12 - |.endif - } if (Z_REG(op1_addr) != ZREG_FP) { | mov Ra(Z_REG(op1_addr)), [r4] // restore } @@ -4687,23 +4570,7 @@ static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, zend_ if (opline->op2_type == IS_UNUSED) { | // var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval)); | LOAD_ADDR FCARG2a, &EG(uninitialized_zval) - if (ZEND_DEBUG) { - const char *filename = op_array->filename ? op_array->filename->val : NULL; - |.if X64 - | LOAD_ADDR CARG3, filename - | mov CARG4d, opline->lineno - |.else - | add r4, 8 - | push opline->lineno - | push filename - |.endif - } - | EXT_CALL _zend_hash_next_index_insert, r0 - || if (ZEND_DEBUG) { - | .if not(X64) - | sub r4, 8 - | .endif - || } + | EXT_CALL zend_hash_next_index_insert, r0 | // if (UNEXPECTED(!var_ptr)) { | test r0, r0 | jz >1 @@ -4784,23 +4651,8 @@ static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, zend_ if (Z_REG(op1_addr) != ZREG_FP) { | mov [r4], Ra(Z_REG(op1_addr)) // save } - if (ZEND_DEBUG) { - const char *filename = op_array->filename ? op_array->filename->val : NULL; - |.if X64 - | mov CARG3d, opline->lineno - |.else - | sub r4, 12 - | push opline->lineno - |.endif - | LOAD_ADDR FCARG2a, filename - } | mov FCARG1d, 8 | EXT_CALL _zend_new_array, r0 - if (ZEND_DEBUG) { - |.if not X64 - | add r4, 12 - |.endif - } if (Z_REG(op1_addr) != ZREG_FP) { | mov Ra(Z_REG(op1_addr)), [r4] // restore } @@ -6579,7 +6431,7 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, int b, | GC_DELREF FCARG1a | jnz >3 | mov aword [r4], r0 // save - | ZVAL_DTOR_FUNC op1_info, op_array->filename, opline + | ZVAL_DTOR_FUNC op1_info, opline | mov r0, aword [r4] // restore |3: } @@ -7341,7 +7193,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar if (func) { for (i = 0; i < call_info->num_args; i++ ) { uint32_t offset = (uint32_t)(uintptr_t)ZEND_CALL_VAR_NUM(NULL, i); - | ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_RX, offset), MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN, 0, 1, 1, op_array->filename, opline + | ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_RX, offset), MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN, 0, 1, 1, opline } } else { | mov FCARG1a, RX @@ -7393,7 +7245,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar (MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN); if (func_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { - | ZVAL_PTR_DTOR res_addr, func_info, 1, 1, 0, op_array->filename, opline + | ZVAL_PTR_DTOR res_addr, func_info, 1, 1, 0, opline } | add r4, 16 /* revert alloca() */ } @@ -8055,7 +7907,7 @@ static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, int b, i } | mov byte [r4], al // save | // zval_dtor_func(r); - | ZVAL_DTOR_FUNC op1_info, op_array->filename, opline + | ZVAL_DTOR_FUNC op1_info, opline | mov cl, byte [r4] // restore |jmp >2 } @@ -8132,7 +7984,7 @@ static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, int b, i } | mov byte [r4], al // save | // zval_dtor_func(r); - | ZVAL_DTOR_FUNC op1_info, op_array->filename, opline + | ZVAL_DTOR_FUNC op1_info, opline | mov cl, byte [r4] // restore |jmp >2 } @@ -8235,7 +8087,7 @@ static int zend_jit_free_compiled_variables(dasm_State **Dst, const zend_op *opl if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { uint32_t offset = (uint32_t)(uintptr_t)ZEND_CALL_VAR_NUM(NULL, i); - | ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_FP, offset), info, 1, 1, 1, op_array->filename, opline + | ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_FP, offset), info, 1, 1, 1, opline } } return 1; @@ -8402,7 +8254,7 @@ static int zend_jit_return(dasm_State **Dst, const zend_op *opline, zend_op_arra } } | //SAVE_OPLINE() - | ZVAL_DTOR_FUNC op1_info, op_array->filename, opline + | ZVAL_DTOR_FUNC op1_info, opline | //????mov r1, EX->return_value // reload ??? } if (jit_return_label >= 0) { @@ -8905,7 +8757,7 @@ static int zend_jit_bind_global(dasm_State **Dst, const zend_op *opline, zend_op | jnz >3 } | mov aword [r4], r0 // save - | ZVAL_DTOR_FUNC op1_info, op_array->filename, opline + | ZVAL_DTOR_FUNC op1_info, opline | mov r0, aword [r4] // restore | jmp >5 if (op1_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) { @@ -9168,7 +9020,7 @@ static int zend_jit_recv_init(dasm_State **Dst, const zend_op *opline, zend_op_a |.cold_code if (has_slow & 1) { |7: - | ZVAL_PTR_DTOR res_addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN, 1, 0, 0, op_array->filename, opline + | ZVAL_PTR_DTOR res_addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN, 1, 0, 0, opline | SET_ZVAL_TYPE_INFO res_addr, IS_UNDEF if (zend_may_throw(opline, op_array, ssa)) { if (!zend_jit_check_exception(Dst)) { @@ -9513,7 +9365,7 @@ static int zend_jit_free(dasm_State **Dst, const zend_op *opline, zend_op_array | EXT_CALL zend_hash_iterator_del, r0 |7: } - | ZVAL_PTR_DTOR op1_addr, op1_info, 0, 0, 0, op_array->filename, opline + | ZVAL_PTR_DTOR op1_addr, op1_info, 0, 0, 0, opline if (zend_may_throw(opline, op_array, ssa)) { if (!zend_jit_check_exception(Dst)) { return 0; From 2e6eb220ba31c6b513ed4e9a94929ceae1c77b8d Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 23 Jul 2018 15:34:41 +0300 Subject: [PATCH 520/569] Use better function --- ext/opcache/jit/zend_jit_x86.dasc | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 0ad4fbc8bdbf1..84cda6ffd19cd 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -4326,8 +4326,7 @@ static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, zend_op_ if (Z_REG(op1_addr) != ZREG_FP) { | mov [r4], Ra(Z_REG(op1_addr)) // save } - | mov FCARG1d, 8 - | EXT_CALL _zend_new_array, r0 + | EXT_CALL _zend_new_array_0, r0 if (Z_REG(op1_addr) != ZREG_FP) { | mov Ra(Z_REG(op1_addr)), [r4] // restore } @@ -4405,8 +4404,7 @@ static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, zend_op_ if (Z_REG(op1_addr) != ZREG_FP) { | mov [r4], Ra(Z_REG(op1_addr)) // save } - | mov FCARG1d, 8 - | EXT_CALL _zend_new_array, r0 + | EXT_CALL _zend_new_array_0, r0 if (Z_REG(op1_addr) != ZREG_FP) { | mov Ra(Z_REG(op1_addr)), [r4] // restore } @@ -4549,8 +4547,7 @@ static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, zend_ if (Z_REG(op1_addr) != ZREG_FP) { | mov [r4], Ra(Z_REG(op1_addr)) // save } - | mov FCARG1d, 8 - | EXT_CALL _zend_new_array, r0 + | EXT_CALL _zend_new_array_0, r0 if (Z_REG(op1_addr) != ZREG_FP) { | mov Ra(Z_REG(op1_addr)), [r4] // restore } @@ -4651,8 +4648,7 @@ static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, zend_ if (Z_REG(op1_addr) != ZREG_FP) { | mov [r4], Ra(Z_REG(op1_addr)) // save } - | mov FCARG1d, 8 - | EXT_CALL _zend_new_array, r0 + | EXT_CALL _zend_new_array_0, r0 if (Z_REG(op1_addr) != ZREG_FP) { | mov Ra(Z_REG(op1_addr)), [r4] // restore } From b413c52a72952c76b5ad4722641b383d55b38c9a Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 31 Jul 2018 15:18:20 +0300 Subject: [PATCH 521/569] fixed support for recent master changes --- ext/opcache/jit/zend_jit_vm_helpers.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ext/opcache/jit/zend_jit_vm_helpers.c b/ext/opcache/jit/zend_jit_vm_helpers.c index 15d07df6b25ab..35835cfc6a53e 100644 --- a/ext/opcache/jit/zend_jit_vm_helpers.c +++ b/ext/opcache/jit/zend_jit_vm_helpers.c @@ -224,7 +224,7 @@ static zend_always_inline int _zend_quick_get_constant( } else { key++; zv = zend_hash_find_ex(EG(zend_constants), Z_STR_P(key), 1); - if (zv && (((zend_constant*)Z_PTR_P(zv))->flags & CONST_CS) == 0) { + if (zv && (ZEND_CONSTANT_FLAGS((zend_constant*)Z_PTR_P(zv)) & CONST_CS) == 0) { c = (zend_constant*)Z_PTR_P(zv); } else { if ((flags & (IS_CONSTANT_IN_NAMESPACE|IS_CONSTANT_UNQUALIFIED)) == (IS_CONSTANT_IN_NAMESPACE|IS_CONSTANT_UNQUALIFIED)) { @@ -235,7 +235,7 @@ static zend_always_inline int _zend_quick_get_constant( } else { key++; zv = zend_hash_find_ex(EG(zend_constants), Z_STR_P(key), 1); - if (zv && (((zend_constant*)Z_PTR_P(zv))->flags & CONST_CS) == 0) { + if (zv && (ZEND_CONSTANT_FLAGS((zend_constant*)Z_PTR_P(zv)) & CONST_CS) == 0) { c = (zend_constant*)Z_PTR_P(zv); } } @@ -267,7 +267,7 @@ static zend_always_inline int _zend_quick_get_constant( if (!check_defined_only) { ZVAL_COPY_OR_DUP(EX_VAR(opline->result.var), &c->value); - if (!(c->flags & (CONST_CS|CONST_CT_SUBST))) { + if (!(ZEND_CONSTANT_FLAGS(c) & (CONST_CS|CONST_CT_SUBST))) { const char *ns_sep; size_t shortname_offset; size_t shortname_len; From b49c3398e7186d69117dfa404c28aa64c3a0a075 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 17 Oct 2018 18:29:45 +0300 Subject: [PATCH 522/569] JIT support for immutable op_arrays --- ext/opcache/jit/zend_jit_helpers.c | 36 +++++++----------------------- ext/opcache/jit/zend_jit_x86.dasc | 15 ++++++++++++- 2 files changed, 22 insertions(+), 29 deletions(-) diff --git a/ext/opcache/jit/zend_jit_helpers.c b/ext/opcache/jit/zend_jit_helpers.c index 97dce19b34d66..1b218e7080c50 100644 --- a/ext/opcache/jit/zend_jit_helpers.c +++ b/ext/opcache/jit/zend_jit_helpers.c @@ -28,20 +28,10 @@ static zend_function* ZEND_FASTCALL zend_jit_find_func_helper(zend_string *name) return NULL; } fbc = Z_FUNC_P(func); - if (EXPECTED(fbc->type == ZEND_USER_FUNCTION) && UNEXPECTED(!fbc->op_array.run_time_cache)) { - if (fbc->op_array.fn_flags & ZEND_ACC_IMMUTABLE) { - zend_op_array *new_op_array = zend_arena_alloc(&CG(arena), sizeof(zend_op_array) + fbc->op_array.cache_size); - - Z_PTR_P(func) = new_op_array; - memcpy(new_op_array, fbc, sizeof(zend_op_array)); - new_op_array->fn_flags &= ~ZEND_ACC_IMMUTABLE; - new_op_array->run_time_cache = (void**)(new_op_array + 1); - memset(new_op_array->run_time_cache, 0, new_op_array->cache_size); - return (zend_function*)new_op_array; - } else { - fbc->op_array.run_time_cache = zend_arena_alloc(&CG(arena), fbc->op_array.cache_size); - memset(fbc->op_array.run_time_cache, 0, fbc->op_array.cache_size); - } + if (EXPECTED(fbc->type == ZEND_USER_FUNCTION) && UNEXPECTED(!RUN_TIME_CACHE(&fbc->op_array))) { + void **run_time_cache = zend_arena_alloc(&CG(arena), fbc->op_array.cache_size); + memset(run_time_cache, 0, fbc->op_array.cache_size); + ZEND_MAP_PTR_SET(fbc->op_array.run_time_cache, run_time_cache); } return fbc; } @@ -56,20 +46,10 @@ static zend_function* ZEND_FASTCALL zend_jit_find_func_by_name_helper(zend_strin return NULL; } fbc = Z_FUNC_P(func); - if (EXPECTED(fbc->type == ZEND_USER_FUNCTION) && UNEXPECTED(!fbc->op_array.run_time_cache)) { - if (fbc->op_array.fn_flags & ZEND_ACC_IMMUTABLE) { - zend_op_array *new_op_array = zend_arena_alloc(&CG(arena), sizeof(zend_op_array) + fbc->op_array.cache_size); - - Z_PTR_P(func) = new_op_array; - memcpy(new_op_array, fbc, sizeof(zend_op_array)); - new_op_array->fn_flags &= ~ZEND_ACC_IMMUTABLE; - new_op_array->run_time_cache = (void**)(new_op_array + 1); - memset(new_op_array->run_time_cache, 0, new_op_array->cache_size); - return (zend_function*)new_op_array; - } else { - fbc->op_array.run_time_cache = zend_arena_alloc(&CG(arena), fbc->op_array.cache_size); - memset(fbc->op_array.run_time_cache, 0, fbc->op_array.cache_size); - } + if (EXPECTED(fbc->type == ZEND_USER_FUNCTION) && UNEXPECTED(!RUN_TIME_CACHE(&fbc->op_array))) { + void **run_time_cache = zend_arena_alloc(&CG(arena), fbc->op_array.cache_size); + memset(run_time_cache, 0, fbc->op_array.cache_size); + ZEND_MAP_PTR_SET(fbc->op_array.run_time_cache, run_time_cache); } return fbc; } diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 84cda6ffd19cd..e147da0cbc266 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -6978,7 +6978,20 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar if (func) { | mov r0, EX:RX->func } - | mov r2, aword [r0 + offsetof(zend_op_array, run_time_cache)] + | mov r2, aword [r0 + offsetof(zend_op_array, run_time_cache__ptr)] +#if ZEND_MAP_PTR_KIND == ZEND_MAP_PTR_KIND_PTR + | mov r2, aword [r2] +#elif ZEND_MAP_PTR_KIND == ZEND_MAP_PTR_KIND_PTR_OR_OFFSET + | xor r1, r1 + | test r2, 1 + | jz >1 + | MEM_OP2_2 mov, r1, aword, &CG(map_ptr_base), r1 + | sub r1, 1 + |1: + | mov r2, aword [r1 + r2] +#else +# error "Unknown ZEND_MAP_PTR_KIND" +#endif | mov EX:RX->run_time_cache, r2 } } From 40690c3e945f2e3d7a4fd56cb99b374f263c8c0d Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 27 Dec 2018 11:04:09 +0300 Subject: [PATCH 523/569] Peform run-time constructor check for non-linked classes --- ext/opcache/jit/zend_jit_x86.dasc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index e147da0cbc266..5adc96683aeb3 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -7323,7 +7323,7 @@ static int zend_jit_new(dasm_State **Dst, const zend_op *opline, int *opnum, zen } (*opnum)++; - if (!ce || ce->constructor) { + if (!ce || !(ce->ce_flags & ZEND_ACC_LINKED) || ce->constructor) { const zend_op *next_opline = opline + 1; | cmp IPl, next_opline From 03f08788df433d66a14f664a599b5052be0eeed8 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 27 Dec 2018 11:53:34 +0300 Subject: [PATCH 524/569] Peform run-time property pffset resolution for non-linked classes --- ext/opcache/jit/zend_jit_x86.dasc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 5adc96683aeb3..8bbc754a6d007 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -9070,7 +9070,7 @@ static uint32_t zend_get_known_property_offset(zend_class_entry *ce, zend_string { zend_property_info *info; - if (!ce || (ce->ce_flags & ZEND_ACC_TRAIT)) { + if (!ce || !(ce->ce_flags & ZEND_ACC_LINKED) || (ce->ce_flags & ZEND_ACC_TRAIT)) { return ZEND_WRONG_PROPERTY_OFFSET; } From f21c977e33e5d60dec876b739a5a4d0de072e294 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 27 Dec 2018 12:19:22 +0300 Subject: [PATCH 525/569] Fixed ext/spl/tests/bug73686.phpt failure --- ext/opcache/jit/zend_jit_helpers.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_helpers.c b/ext/opcache/jit/zend_jit_helpers.c index 1b218e7080c50..289cbdd275230 100644 --- a/ext/opcache/jit/zend_jit_helpers.c +++ b/ext/opcache/jit/zend_jit_helpers.c @@ -937,7 +937,7 @@ static zend_never_inline void zend_assign_to_string_offset(zval *str, zval *dim, static void ZEND_FASTCALL zend_jit_assign_dim_helper(zval *object_ptr, zval *dim, zval *value, zval *result) { if (EXPECTED(Z_TYPE_P(object_ptr) == IS_OBJECT)) { - + ZVAL_DEREF(value); if (UNEXPECTED(!Z_OBJ_HT_P(object_ptr)->write_dimension)) { zend_throw_error(NULL, "Cannot use object as array"); } else { From 57ad25b563da3dd20b8210c184df92b1d8753eaf Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 15 Jan 2019 11:16:01 +0300 Subject: [PATCH 526/569] Partial typed properties support (incomplete) --- ext/opcache/jit/zend_jit.c | 1 + ext/opcache/jit/zend_jit_helpers.c | 1 + ext/opcache/jit/zend_jit_x86.dasc | 14 +++++++++----- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index c1fcd2f311f94..fbde9aeeac7ea 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -1999,6 +1999,7 @@ static void zend_calc_checked_this_r(zend_bitset checked_this, zend_op_array *op case ZEND_FETCH_OBJ_FUNC_ARG: case ZEND_FETCH_OBJ_UNSET: case ZEND_ASSIGN_OBJ: + case ZEND_ASSIGN_OBJ_REF: case ZEND_INIT_METHOD_CALL: case ZEND_CLONE: case ZEND_UNSET_OBJ: diff --git a/ext/opcache/jit/zend_jit_helpers.c b/ext/opcache/jit/zend_jit_helpers.c index 289cbdd275230..b1269fa15bd14 100644 --- a/ext/opcache/jit/zend_jit_helpers.c +++ b/ext/opcache/jit/zend_jit_helpers.c @@ -1672,6 +1672,7 @@ static zval* ZEND_FASTCALL zend_jit_new_ref_helper(zval *value) zend_reference *ref = (zend_reference*)emalloc(sizeof(zend_reference)); GC_SET_REFCOUNT(ref, 1); GC_TYPE_INFO(ref) = IS_REFERENCE; + ref->sources.ptr = NULL; ZVAL_COPY_VALUE(&ref->val, value); Z_REF_P(value) = ref; Z_TYPE_INFO_P(value) = IS_REFERENCE_EX; diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 8bbc754a6d007..68bd611edac06 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -2261,6 +2261,8 @@ static int zend_jit_handler(dasm_State **Dst, const zend_op *opline, int may_thr switch (opline->opcode) { case ZEND_ASSIGN_DIM: case ZEND_ASSIGN_OBJ: + case ZEND_ASSIGN_STATIC_PROP: + case ZEND_ASSIGN_OBJ_REF: last_valid_opline++; break; case ZEND_ASSIGN_ADD: @@ -4745,7 +4747,7 @@ static int zend_jit_assign_op(dasm_State **Dst, const zend_op *opline, zend_op_a if (opline->extended_value == ZEND_ASSIGN_DIM) { return zend_jit_assign_dim_op(Dst, opline, op_array, ssa); - } else if (opline->extended_value == ZEND_ASSIGN_OBJ) { + } else if (opline->extended_value == ZEND_ASSIGN_OBJ || opline->extended_value == ZEND_ASSIGN_STATIC_PROP) { goto fallback; } @@ -7433,7 +7435,8 @@ static int zend_jit_send_ref(dasm_State **Dst, const zend_op *opline, zend_op_ar | SET_ZVAL_PTR arg_addr, r0 | SET_ZVAL_TYPE_INFO arg_addr, IS_REFERENCE_EX | mov dword [r0], 1 - | mov dword [r0 + 4], IS_REFERENCE; + | mov dword [r0 + offsetof(zend_reference, gc.u.type_info)], IS_REFERENCE + | mov aword [r0 + offsetof(zend_reference, sources.ptr)], 0 ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 8); | // ZVAL_NULL(Z_REFVAL_P(arg)); | SET_ZVAL_TYPE_INFO ref_addr, IS_NULL @@ -7483,7 +7486,8 @@ static int zend_jit_send_ref(dasm_State **Dst, const zend_op *opline, zend_op_ar } | EMALLOC sizeof(zend_reference), op_array, opline | mov dword [r0], 2 - | mov dword [r0 + 4], IS_REFERENCE; + | mov dword [r0 + offsetof(zend_reference, gc.u.type_info)], IS_REFERENCE + | mov aword [r0 + offsetof(zend_reference, sources.ptr)], 0 ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 8); if (opline->op1_type == IS_VAR) { zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R1, 0); @@ -8841,14 +8845,14 @@ static int zend_jit_recv(dasm_State **Dst, const zend_op *opline, zend_op_array if (!ZEND_TYPE_IS_CLASS(type)) { unsigned char code = ZEND_TYPE_CODE(type); - if (type == _IS_BOOL) { + if (code == _IS_BOOL) { | cmp byte [r0 + 8], IS_FALSE | je >1 | cmp byte [r0 + 8], IS_TRUE | jne >8 |1: } else { - | cmp byte [r0 + 8], code + | cmp byte [r0 + 8], code | jne >8 } } else { From 2d2cff1a88bff0c01de37d0173faf79f45245e22 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 15 Jan 2019 14:33:35 +0300 Subject: [PATCH 527/569] JIT support for typed properties (incomplete) --- ext/opcache/jit/zend_jit_disasm_x86.c | 4 +++ ext/opcache/jit/zend_jit_helpers.c | 28 ++++++++++++++++ ext/opcache/jit/zend_jit_x86.dasc | 46 ++++++++++++++++++++------- 3 files changed, 67 insertions(+), 11 deletions(-) diff --git a/ext/opcache/jit/zend_jit_disasm_x86.c b/ext/opcache/jit/zend_jit_disasm_x86.c index 56838c5003cbf..121295f403bf5 100644 --- a/ext/opcache/jit/zend_jit_disasm_x86.c +++ b/ext/opcache/jit/zend_jit_disasm_x86.c @@ -432,6 +432,10 @@ static int zend_jit_disasm_init(void) REGISTER_HELPER(zend_jit_copy_extra_args_helper); REGISTER_HELPER(zend_jit_deprecated_or_abstract_helper); REGISTER_HELPER(zend_jit_verify_internal_arg_types_helper); + REGISTER_HELPER(zend_jit_assign_const_to_typed_ref); + REGISTER_HELPER(zend_jit_assign_tmp_to_typed_ref); + REGISTER_HELPER(zend_jit_assign_var_to_typed_ref); + REGISTER_HELPER(zend_jit_assign_cv_to_typed_ref); REGISTER_HELPER(zend_runtime_jit); REGISTER_HELPER(zend_jit_hot_func); #undef REGISTER_HELPER diff --git a/ext/opcache/jit/zend_jit_helpers.c b/ext/opcache/jit/zend_jit_helpers.c index b1269fa15bd14..aee12b969f2c3 100644 --- a/ext/opcache/jit/zend_jit_helpers.c +++ b/ext/opcache/jit/zend_jit_helpers.c @@ -2094,6 +2094,34 @@ static int ZEND_FASTCALL zend_jit_verify_internal_arg_types_helper(zend_execute_ return 1; } +static zend_always_inline void zend_jit_assign_to_typed_ref(zend_reference *ref, zval *value, zend_uchar value_type) +{ + zval variable; + + ZVAL_REF(&variable, ref); + zend_assign_to_variable(&variable, value, value_type, ZEND_CALL_USES_STRICT_TYPES(EG(current_execute_data))); +} + +static void ZEND_FASTCALL zend_jit_assign_const_to_typed_ref(zend_reference *ref, zval *value) +{ + zend_jit_assign_to_typed_ref(ref, value, IS_CONST); +} + +static void ZEND_FASTCALL zend_jit_assign_tmp_to_typed_ref(zend_reference *ref, zval *value) +{ + zend_jit_assign_to_typed_ref(ref, value, IS_TMP_VAR); +} + +static void ZEND_FASTCALL zend_jit_assign_var_to_typed_ref(zend_reference *ref, zval *value) +{ + zend_jit_assign_to_typed_ref(ref, value, IS_VAR); +} + +static void ZEND_FASTCALL zend_jit_assign_cv_to_typed_ref(zend_reference *ref, zval *value) +{ + zend_jit_assign_to_typed_ref(ref, value, IS_CV); +} + /* * Local variables: * tab-width: 4 diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 68bd611edac06..7a7a13192f441 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -2262,6 +2262,7 @@ static int zend_jit_handler(dasm_State **Dst, const zend_op *opline, int may_thr case ZEND_ASSIGN_DIM: case ZEND_ASSIGN_OBJ: case ZEND_ASSIGN_STATIC_PROP: + case ZEND_ASSIGN_STATIC_PROP_REF: case ZEND_ASSIGN_OBJ_REF: last_valid_opline++; break; @@ -4211,9 +4212,39 @@ static int zend_jit_assign_to_variable(dasm_State **Dst, zend_jit_addr val_addr, uint32_t val_info, zend_jit_addr res_addr) -/* Labels: 1,2,3,4,5 */ +/* Labels: 1,2,3,4,5,8 */ { //ZEND_ASSERT(Z_MODE(var_addr) == IS_MEM_ZVAL); + if (var_info & MAY_BE_REF) { + if (Z_MODE(var_addr) != IS_REG || Z_REG(var_addr) != ZREG_FCARG1a) { + | LOAD_ZVAL_ADDR FCARG1a, var_addr + var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); + } + | // if (Z_ISREF_P(variable_ptr)) { + | IF_NOT_Z_TYPE, FCARG1a, IS_REFERENCE, >1 + | // if (UNEXPECTED(ZEND_REF_HAS_TYPE_SOURCES(Z_REF_P(variable_ptr)))) { + | GET_Z_PTR FCARG1a, FCARG1a + | cmp aword [FCARG1a + offsetof(zend_reference, sources.ptr)], 0 + | jnz >2 + | add FCARG1a, offsetof(zend_reference, val) + |.cold_code + |2: + | LOAD_ZVAL_ADDR FCARG2a, val_addr + if (val_type == IS_CONST) { + | EXT_CALL zend_jit_assign_const_to_typed_ref, r0 + } else if (val_type == IS_TMP_VAR) { + | EXT_CALL zend_jit_assign_tmp_to_typed_ref, r0 + } else if (val_type == IS_VAR) { + | EXT_CALL zend_jit_assign_var_to_typed_ref, r0 + } else if (val_type == IS_CV) { + | EXT_CALL zend_jit_assign_cv_to_typed_ref, r0 + } else { + ZEND_ASSERT(0); + } + | jmp >8 + |.code + |1: + } if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { int in_cold = 0; @@ -4239,10 +4270,10 @@ static int zend_jit_assign_to_variable(dasm_State **Dst, | mov FCARG1a, aword [r4] // restore if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { | cmp dword [FCARG1a], 0 - | jnz >3 + | jnz >8 } | ZVAL_DTOR_FUNC var_info, opline - | jmp >3 + | jmp >8 |4: } if (RC_MAY_BE_N(var_info)) { @@ -4277,7 +4308,7 @@ static int zend_jit_assign_to_variable(dasm_State **Dst, if (!zend_jit_simple_assign(Dst, opline, op_array, ssa, var_addr, var_info, val_type, val, val_addr, val_info, res_addr, 0)) { return 0; } - |3: + |8: return 1; } @@ -4381,7 +4412,6 @@ static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, zend_op_ var_info |= MAY_BE_REF; } | // value = zend_assign_to_variable(variable_ptr, value, OP_DATA_TYPE); - | ZVAL_DEREF FCARG1a, var_info if (!zend_jit_assign_to_variable(Dst, opline, op_array, ssa, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, op3_addr, val_info, res_addr)) { return 0; } @@ -6521,12 +6551,6 @@ static int zend_jit_assign(dasm_State **Dst, const zend_op *opline, zend_op_arra } } - if (op1_info & MAY_BE_REF) { - | LOAD_ZVAL_ADDR FCARG1a, op1_addr - | ZVAL_DEREF FCARG1a, op1_info - op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); - } - if (!zend_jit_assign_to_variable(Dst, opline, op_array, ssa, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, res_addr)) { return 0; } From 78a30aeb12ba0918a3724562827e50c38e6502fd Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 17 Jan 2019 17:11:16 +0300 Subject: [PATCH 528/569] JIT support for typed properties. --- ext/opcache/jit/zend_jit_disasm_x86.c | 4 ++ ext/opcache/jit/zend_jit_helpers.c | 100 ++++++++++++++++++++++++++ ext/opcache/jit/zend_jit_x86.dasc | 41 ++++++++++- 3 files changed, 144 insertions(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_disasm_x86.c b/ext/opcache/jit/zend_jit_disasm_x86.c index 121295f403bf5..0d02613754300 100644 --- a/ext/opcache/jit/zend_jit_disasm_x86.c +++ b/ext/opcache/jit/zend_jit_disasm_x86.c @@ -436,6 +436,10 @@ static int zend_jit_disasm_init(void) REGISTER_HELPER(zend_jit_assign_tmp_to_typed_ref); REGISTER_HELPER(zend_jit_assign_var_to_typed_ref); REGISTER_HELPER(zend_jit_assign_cv_to_typed_ref); + REGISTER_HELPER(zend_jit_pre_inc_typed_ref); + REGISTER_HELPER(zend_jit_pre_dec_typed_ref); + REGISTER_HELPER(zend_jit_post_inc_typed_ref); + REGISTER_HELPER(zend_jit_post_dec_typed_ref); REGISTER_HELPER(zend_runtime_jit); REGISTER_HELPER(zend_jit_hot_func); #undef REGISTER_HELPER diff --git a/ext/opcache/jit/zend_jit_helpers.c b/ext/opcache/jit/zend_jit_helpers.c index aee12b969f2c3..8814c6996b1d1 100644 --- a/ext/opcache/jit/zend_jit_helpers.c +++ b/ext/opcache/jit/zend_jit_helpers.c @@ -2122,6 +2122,106 @@ static void ZEND_FASTCALL zend_jit_assign_cv_to_typed_ref(zend_reference *ref, z zend_jit_assign_to_typed_ref(ref, value, IS_CV); } +static zend_property_info *zend_jit_get_prop_not_accepting_double(zend_reference *ref) +{ + zend_property_info *prop; + ZEND_REF_FOREACH_TYPE_SOURCES(ref, prop) { + if (ZEND_TYPE_CODE(prop->type) != IS_DOUBLE) { + return prop; + } + } ZEND_REF_FOREACH_TYPE_SOURCES_END(); + return NULL; +} + +static ZEND_COLD void zend_jit_throw_incdec_ref_error(zend_reference *ref, zend_bool inc) +{ + zend_property_info *error_prop = zend_jit_get_prop_not_accepting_double(ref); + /* Currently there should be no way for a typed reference to accept both int and double. + * Generalize this and the related property code once this becomes possible. */ + ZEND_ASSERT(error_prop); + zend_type_error( + "Cannot %s a reference held by property %s::$%s of type %sint past its %simal value", + inc ? "increment" : "decrement", + ZSTR_VAL(error_prop->ce->name), + zend_get_unmangled_property_name(error_prop->name), + ZEND_TYPE_ALLOW_NULL(error_prop->type) ? "?" : "", + inc ? "max" : "min"); +} + +static void ZEND_FASTCALL zend_jit_pre_inc_typed_ref(zval *var_ptr, zend_reference *ref, zval *ret) +{ + zval tmp; + + ZVAL_COPY(&tmp, var_ptr); + + increment_function(var_ptr); + + if (UNEXPECTED(Z_TYPE_P(var_ptr) == IS_DOUBLE) && Z_TYPE(tmp) == IS_LONG) { + zend_jit_throw_incdec_ref_error(ref, 1); + ZVAL_COPY_VALUE(var_ptr, &tmp); + } else if (UNEXPECTED(!zend_verify_ref_assignable_zval(ref, var_ptr, ZEND_CALL_USES_STRICT_TYPES(EG(current_execute_data))))) { + zval_ptr_dtor(var_ptr); + ZVAL_COPY_VALUE(var_ptr, &tmp); + } else { + zval_ptr_dtor(&tmp); + } + if (ret) { + ZVAL_COPY(ret, var_ptr); + } +} + +static void ZEND_FASTCALL zend_jit_pre_dec_typed_ref(zval *var_ptr, zend_reference *ref, zval *ret) +{ + zval tmp; + + ZVAL_COPY(&tmp, var_ptr); + + decrement_function(var_ptr); + + if (UNEXPECTED(Z_TYPE_P(var_ptr) == IS_DOUBLE) && Z_TYPE(tmp) == IS_LONG) { + zend_jit_throw_incdec_ref_error(ref, 0); + ZVAL_COPY_VALUE(var_ptr, &tmp); + } else if (UNEXPECTED(!zend_verify_ref_assignable_zval(ref, var_ptr, ZEND_CALL_USES_STRICT_TYPES(EG(current_execute_data))))) { + zval_ptr_dtor(var_ptr); + ZVAL_COPY_VALUE(var_ptr, &tmp); + } else { + zval_ptr_dtor(&tmp); + } + if (ret) { + ZVAL_COPY(ret, var_ptr); + } +} + +static void ZEND_FASTCALL zend_jit_post_inc_typed_ref(zval *var_ptr, zend_reference *ref, zval *ret) +{ + ZVAL_COPY(ret, var_ptr); + + increment_function(var_ptr); + + if (UNEXPECTED(Z_TYPE_P(var_ptr) == IS_DOUBLE) && Z_TYPE_P(ret) == IS_LONG) { + zend_jit_throw_incdec_ref_error(ref, 1); + ZVAL_COPY_VALUE(var_ptr, ret); + } else if (UNEXPECTED(!zend_verify_ref_assignable_zval(ref, var_ptr, ZEND_CALL_USES_STRICT_TYPES(EG(current_execute_data))))) { + zval_ptr_dtor(var_ptr); + ZVAL_COPY_VALUE(var_ptr, ret); + } +} + +static void ZEND_FASTCALL zend_jit_post_dec_typed_ref(zval *var_ptr, zend_reference *ref, zval *ret) +{ + ZVAL_COPY(ret, var_ptr); + + decrement_function(var_ptr); + + if (UNEXPECTED(Z_TYPE_P(var_ptr) == IS_DOUBLE) && Z_TYPE_P(ret) == IS_LONG) { + zend_jit_throw_incdec_ref_error(ref, 0); + ZVAL_COPY_VALUE(var_ptr, ret); + } else if (UNEXPECTED(!zend_verify_ref_assignable_zval(ref, var_ptr, ZEND_CALL_USES_STRICT_TYPES(EG(current_execute_data))))) { + zval_ptr_dtor(var_ptr); + ZVAL_COPY_VALUE(var_ptr, ret); + } +} + /* * Local variables: * tab-width: 4 diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 7a7a13192f441..c48aabc3a79c4 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -2610,8 +2610,47 @@ static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, zend_op_arr } |2: | LOAD_ZVAL_ADDR FCARG1a, op1_addr + | // ZVAL_DEREF(var_ptr); - | ZVAL_DEREF FCARG1a, op1_info + if (op1_info & MAY_BE_REF) { + | IF_NOT_Z_TYPE, FCARG1a, IS_REFERENCE, >2 + | GET_Z_PTR FCARG2a, FCARG1a + | cmp aword [FCARG2a + offsetof(zend_reference, sources.ptr)], 0 + | lea FCARG1a, [FCARG2a + offsetof(zend_reference, val)] + | jz >2 + |.if X64 + if (RETURN_VALUE_USED(opline)) { + | LOAD_ZVAL_ADDR CARG3, res_addr + } else { + | mov CARG3, 0 + } + |.else + | sub r4, 12 + if (RETURN_VALUE_USED(opline)) { + | PUSH_ZVAL_ADDR res_addr, r0 + } else { + | push 0 + } + |.endif + if (opline->opcode == ZEND_PRE_INC) { + | EXT_CALL zend_jit_pre_inc_typed_ref, r0 + } else if (opline->opcode == ZEND_PRE_DEC) { + | EXT_CALL zend_jit_pre_dec_typed_ref, r0 + } else if (opline->opcode == ZEND_POST_INC) { + | EXT_CALL zend_jit_post_inc_typed_ref, r0 + } else if (opline->opcode == ZEND_POST_DEC) { + | EXT_CALL zend_jit_post_dec_typed_ref, r0 + } else { + ZEND_ASSERT(0); + } + |.if not(X64) + | add r4, 12 + |.endif + zend_jit_check_exception(Dst); + | jmp >3 + |2: + } + if ((opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC)) { if (opline->result_type != IS_UNUSED) { zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); From 00bc2f11a1143f35d95a93302b0054c7e86a830b Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 31 Jan 2019 20:13:58 +0300 Subject: [PATCH 529/569] Fixed compatibility with master --- ext/opcache/jit/zend_jit_vm_helpers.c | 76 ++------------------------- 1 file changed, 5 insertions(+), 71 deletions(-) diff --git a/ext/opcache/jit/zend_jit_vm_helpers.c b/ext/opcache/jit/zend_jit_vm_helpers.c index 35835cfc6a53e..c7f7afea43e3e 100644 --- a/ext/opcache/jit/zend_jit_vm_helpers.c +++ b/ext/opcache/jit/zend_jit_vm_helpers.c @@ -215,96 +215,30 @@ static zend_always_inline int _zend_quick_get_constant( const zval *key, uint32_t flags, int check_defined_only) { zval *zv; - const zval *orig_key = key; zend_constant *c = NULL; + /* null/true/false are resolved during compilation, so don't check for them here. */ zv = zend_hash_find_ex(EG(zend_constants), Z_STR_P(key), 1); if (zv) { c = (zend_constant*)Z_PTR_P(zv); - } else { + } else if (flags & IS_CONSTANT_UNQUALIFIED_IN_NAMESPACE) { key++; zv = zend_hash_find_ex(EG(zend_constants), Z_STR_P(key), 1); - if (zv && (ZEND_CONSTANT_FLAGS((zend_constant*)Z_PTR_P(zv)) & CONST_CS) == 0) { + if (zv) { c = (zend_constant*)Z_PTR_P(zv); - } else { - if ((flags & (IS_CONSTANT_IN_NAMESPACE|IS_CONSTANT_UNQUALIFIED)) == (IS_CONSTANT_IN_NAMESPACE|IS_CONSTANT_UNQUALIFIED)) { - key++; - zv = zend_hash_find_ex(EG(zend_constants), Z_STR_P(key), 1); - if (zv) { - c = (zend_constant*)Z_PTR_P(zv); - } else { - key++; - zv = zend_hash_find_ex(EG(zend_constants), Z_STR_P(key), 1); - if (zv && (ZEND_CONSTANT_FLAGS((zend_constant*)Z_PTR_P(zv)) & CONST_CS) == 0) { - c = (zend_constant*)Z_PTR_P(zv); - } - } - } } } if (!c) { if (!check_defined_only) { - if ((opline->op1.num & IS_CONSTANT_UNQUALIFIED) != 0) { - char *actual = (char *)zend_memrchr(Z_STRVAL_P(RT_CONSTANT(opline, opline->op2)), '\\', Z_STRLEN_P(RT_CONSTANT(opline, opline->op2))); - if (!actual) { - ZVAL_STR_COPY(EX_VAR(opline->result.var), Z_STR_P(RT_CONSTANT(opline, opline->op2))); - } else { - actual++; - ZVAL_STRINGL(EX_VAR(opline->result.var), - actual, Z_STRLEN_P(RT_CONSTANT(opline, opline->op2)) - (actual - Z_STRVAL_P(RT_CONSTANT(opline, opline->op2)))); - } - /* non-qualified constant - allow text substitution */ - zend_error(E_WARNING, "Use of undefined constant %s - assumed '%s' (this will throw an Error in a future version of PHP)", - Z_STRVAL_P(EX_VAR(opline->result.var)), Z_STRVAL_P(EX_VAR(opline->result.var))); - } else { - zend_throw_error(NULL, "Undefined constant '%s'", Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))); - ZVAL_UNDEF(EX_VAR(opline->result.var)); - } + zend_throw_error(NULL, "Undefined constant '%s'", Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))); + ZVAL_UNDEF(EX_VAR(opline->result.var)); } return FAILURE; } if (!check_defined_only) { ZVAL_COPY_OR_DUP(EX_VAR(opline->result.var), &c->value); - if (!(ZEND_CONSTANT_FLAGS(c) & (CONST_CS|CONST_CT_SUBST))) { - const char *ns_sep; - size_t shortname_offset; - size_t shortname_len; - zend_bool is_deprecated; - - if (flags & IS_CONSTANT_UNQUALIFIED) { - const zval *access_key; - - if (!(flags & IS_CONSTANT_IN_NAMESPACE)) { - access_key = orig_key - 1; - } else { - if (key < orig_key + 2) { - goto check_short_name; - } else { - access_key = orig_key + 2; - } - } - is_deprecated = !zend_string_equals(c->name, Z_STR_P(access_key)); - } else { -check_short_name: - ns_sep = zend_memrchr(ZSTR_VAL(c->name), '\\', ZSTR_LEN(c->name)); - ZEND_ASSERT(ns_sep); - /* Namespaces are always case-insensitive. Only compare shortname. */ - shortname_offset = ns_sep - ZSTR_VAL(c->name) + 1; - shortname_len = ZSTR_LEN(c->name) - shortname_offset; - - is_deprecated = memcmp(ZSTR_VAL(c->name) + shortname_offset, Z_STRVAL_P(orig_key - 1) + shortname_offset, shortname_len) != 0; - } - - if (is_deprecated) { - zend_error(E_DEPRECATED, - "Case-insensitive constants are deprecated. " - "The correct casing for this constant is \"%s\"", - ZSTR_VAL(c->name)); - return SUCCESS; - } - } } CACHE_PTR(opline->extended_value, c); From f939bd05ef34f4282847a77f6635f1125c3660d9 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 4 Feb 2019 13:41:00 +0300 Subject: [PATCH 530/569] Fixed support for changes in object handlers API. --- ext/opcache/jit/zend_jit_helpers.c | 90 ++++++++++++++++-------------- 1 file changed, 47 insertions(+), 43 deletions(-) diff --git a/ext/opcache/jit/zend_jit_helpers.c b/ext/opcache/jit/zend_jit_helpers.c index 8814c6996b1d1..315064beb84d6 100644 --- a/ext/opcache/jit/zend_jit_helpers.c +++ b/ext/opcache/jit/zend_jit_helpers.c @@ -667,7 +667,7 @@ static void ZEND_FASTCALL zend_jit_fetch_dim_obj_r_helper(zval *container, zval zend_throw_error(NULL, "Cannot use object as array"); ZVAL_NULL(result); } else { - zval *retval = Z_OBJ_HT_P(container)->read_dimension(container, dim, BP_VAR_R, result); + zval *retval = Z_OBJ_HT_P(container)->read_dimension(Z_OBJ_P(container), dim, BP_VAR_R, result); if (retval) { if (result != retval) { @@ -691,7 +691,7 @@ static void ZEND_FASTCALL zend_jit_fetch_dim_obj_is_helper(zval *container, zval zend_throw_error(NULL, "Cannot use object as array"); ZVAL_NULL(result); } else { - zval *retval = Z_OBJ_HT_P(container)->read_dimension(container, dim, BP_VAR_IS, result); + zval *retval = Z_OBJ_HT_P(container)->read_dimension(Z_OBJ_P(container), dim, BP_VAR_IS, result); if (retval) { if (result != retval) { @@ -941,7 +941,7 @@ static void ZEND_FASTCALL zend_jit_assign_dim_helper(zval *object_ptr, zval *dim if (UNEXPECTED(!Z_OBJ_HT_P(object_ptr)->write_dimension)) { zend_throw_error(NULL, "Cannot use object as array"); } else { - Z_OBJ_HT_P(object_ptr)->write_dimension(object_ptr, dim, value); + Z_OBJ_HT_P(object_ptr)->write_dimension(Z_OBJ_P(object_ptr), dim, value); if (result && EXPECTED(!EG(exception))) { ZVAL_COPY(result, value); } @@ -971,11 +971,11 @@ static void ZEND_FASTCALL zend_jit_assign_dim_add_helper(zval *container, zval * zval rv, res; if (Z_OBJ_HT_P(object)->read_dimension && - (z = Z_OBJ_HT_P(object)->read_dimension(object, property, BP_VAR_R, &rv)) != NULL) { + (z = Z_OBJ_HT_P(object)->read_dimension(Z_OBJ_P(object), property, BP_VAR_R, &rv)) != NULL) { if (Z_TYPE_P(z) == IS_OBJECT && Z_OBJ_HT_P(z)->get) { zval rv2; - zval *value = Z_OBJ_HT_P(z)->get(z, &rv2); + zval *value = Z_OBJ_HT_P(z)->get(Z_OBJ_P(z), &rv2); if (z == &rv) { zval_ptr_dtor(&rv); @@ -983,7 +983,7 @@ static void ZEND_FASTCALL zend_jit_assign_dim_add_helper(zval *container, zval * ZVAL_COPY_VALUE(z, value); } add_function(&res, Z_ISREF_P(z) ? Z_REFVAL_P(z) : z, value); - Z_OBJ_HT_P(object)->write_dimension(object, property, &res); + Z_OBJ_HT_P(object)->write_dimension(Z_OBJ_P(object), property, &res); if (z == &rv) { zval_ptr_dtor(&rv); } @@ -1027,11 +1027,11 @@ static void ZEND_FASTCALL zend_jit_assign_dim_sub_helper(zval *container, zval * zval rv, res; if (Z_OBJ_HT_P(object)->read_dimension && - (z = Z_OBJ_HT_P(object)->read_dimension(object, property, BP_VAR_R, &rv)) != NULL) { + (z = Z_OBJ_HT_P(object)->read_dimension(Z_OBJ_P(object), property, BP_VAR_R, &rv)) != NULL) { if (Z_TYPE_P(z) == IS_OBJECT && Z_OBJ_HT_P(z)->get) { zval rv2; - zval *value = Z_OBJ_HT_P(z)->get(z, &rv2); + zval *value = Z_OBJ_HT_P(z)->get(Z_OBJ_P(z), &rv2); if (z == &rv) { zval_ptr_dtor(&rv); @@ -1039,7 +1039,7 @@ static void ZEND_FASTCALL zend_jit_assign_dim_sub_helper(zval *container, zval * ZVAL_COPY_VALUE(z, value); } sub_function(&res, Z_ISREF_P(z) ? Z_REFVAL_P(z) : z, value); - Z_OBJ_HT_P(object)->write_dimension(object, property, &res); + Z_OBJ_HT_P(object)->write_dimension(Z_OBJ_P(object), property, &res); if (z == &rv) { zval_ptr_dtor(&rv); } @@ -1083,11 +1083,11 @@ static void ZEND_FASTCALL zend_jit_assign_dim_mul_helper(zval *container, zval * zval rv, res; if (Z_OBJ_HT_P(object)->read_dimension && - (z = Z_OBJ_HT_P(object)->read_dimension(object, property, BP_VAR_R, &rv)) != NULL) { + (z = Z_OBJ_HT_P(object)->read_dimension(Z_OBJ_P(object), property, BP_VAR_R, &rv)) != NULL) { if (Z_TYPE_P(z) == IS_OBJECT && Z_OBJ_HT_P(z)->get) { zval rv2; - zval *value = Z_OBJ_HT_P(z)->get(z, &rv2); + zval *value = Z_OBJ_HT_P(z)->get(Z_OBJ_P(z), &rv2); if (z == &rv) { zval_ptr_dtor(&rv); @@ -1095,7 +1095,7 @@ static void ZEND_FASTCALL zend_jit_assign_dim_mul_helper(zval *container, zval * ZVAL_COPY_VALUE(z, value); } mul_function(&res, Z_ISREF_P(z) ? Z_REFVAL_P(z) : z, value); - Z_OBJ_HT_P(object)->write_dimension(object, property, &res); + Z_OBJ_HT_P(object)->write_dimension(Z_OBJ_P(object), property, &res); if (z == &rv) { zval_ptr_dtor(&rv); } @@ -1139,11 +1139,11 @@ static void ZEND_FASTCALL zend_jit_assign_dim_div_helper(zval *container, zval * zval rv, res; if (Z_OBJ_HT_P(object)->read_dimension && - (z = Z_OBJ_HT_P(object)->read_dimension(object, property, BP_VAR_R, &rv)) != NULL) { + (z = Z_OBJ_HT_P(object)->read_dimension(Z_OBJ_P(object), property, BP_VAR_R, &rv)) != NULL) { if (Z_TYPE_P(z) == IS_OBJECT && Z_OBJ_HT_P(z)->get) { zval rv2; - zval *value = Z_OBJ_HT_P(z)->get(z, &rv2); + zval *value = Z_OBJ_HT_P(z)->get(Z_OBJ_P(z), &rv2); if (z == &rv) { zval_ptr_dtor(&rv); @@ -1151,7 +1151,7 @@ static void ZEND_FASTCALL zend_jit_assign_dim_div_helper(zval *container, zval * ZVAL_COPY_VALUE(z, value); } div_function(&res, Z_ISREF_P(z) ? Z_REFVAL_P(z) : z, value); - Z_OBJ_HT_P(object)->write_dimension(object, property, &res); + Z_OBJ_HT_P(object)->write_dimension(Z_OBJ_P(object), property, &res); if (z == &rv) { zval_ptr_dtor(&rv); } @@ -1195,11 +1195,11 @@ static void ZEND_FASTCALL zend_jit_assign_dim_bw_or_helper(zval *container, zval zval rv, res; if (Z_OBJ_HT_P(object)->read_dimension && - (z = Z_OBJ_HT_P(object)->read_dimension(object, property, BP_VAR_R, &rv)) != NULL) { + (z = Z_OBJ_HT_P(object)->read_dimension(Z_OBJ_P(object), property, BP_VAR_R, &rv)) != NULL) { if (Z_TYPE_P(z) == IS_OBJECT && Z_OBJ_HT_P(z)->get) { zval rv2; - zval *value = Z_OBJ_HT_P(z)->get(z, &rv2); + zval *value = Z_OBJ_HT_P(z)->get(Z_OBJ_P(z), &rv2); if (z == &rv) { zval_ptr_dtor(&rv); @@ -1207,7 +1207,7 @@ static void ZEND_FASTCALL zend_jit_assign_dim_bw_or_helper(zval *container, zval ZVAL_COPY_VALUE(z, value); } bitwise_or_function(&res, Z_ISREF_P(z) ? Z_REFVAL_P(z) : z, value); - Z_OBJ_HT_P(object)->write_dimension(object, property, &res); + Z_OBJ_HT_P(object)->write_dimension(Z_OBJ_P(object), property, &res); if (z == &rv) { zval_ptr_dtor(&rv); } @@ -1251,11 +1251,11 @@ static void ZEND_FASTCALL zend_jit_assign_dim_bw_and_helper(zval *container, zva zval rv, res; if (Z_OBJ_HT_P(object)->read_dimension && - (z = Z_OBJ_HT_P(object)->read_dimension(object, property, BP_VAR_R, &rv)) != NULL) { + (z = Z_OBJ_HT_P(object)->read_dimension(Z_OBJ_P(object), property, BP_VAR_R, &rv)) != NULL) { if (Z_TYPE_P(z) == IS_OBJECT && Z_OBJ_HT_P(z)->get) { zval rv2; - zval *value = Z_OBJ_HT_P(z)->get(z, &rv2); + zval *value = Z_OBJ_HT_P(z)->get(Z_OBJ_P(z), &rv2); if (z == &rv) { zval_ptr_dtor(&rv); @@ -1263,7 +1263,7 @@ static void ZEND_FASTCALL zend_jit_assign_dim_bw_and_helper(zval *container, zva ZVAL_COPY_VALUE(z, value); } bitwise_and_function(&res, Z_ISREF_P(z) ? Z_REFVAL_P(z) : z, value); - Z_OBJ_HT_P(object)->write_dimension(object, property, &res); + Z_OBJ_HT_P(object)->write_dimension(Z_OBJ_P(object), property, &res); if (z == &rv) { zval_ptr_dtor(&rv); } @@ -1307,11 +1307,11 @@ static void ZEND_FASTCALL zend_jit_assign_dim_bw_xor_helper(zval *container, zva zval rv, res; if (Z_OBJ_HT_P(object)->read_dimension && - (z = Z_OBJ_HT_P(object)->read_dimension(object, property, BP_VAR_R, &rv)) != NULL) { + (z = Z_OBJ_HT_P(object)->read_dimension(Z_OBJ_P(object), property, BP_VAR_R, &rv)) != NULL) { if (Z_TYPE_P(z) == IS_OBJECT && Z_OBJ_HT_P(z)->get) { zval rv2; - zval *value = Z_OBJ_HT_P(z)->get(z, &rv2); + zval *value = Z_OBJ_HT_P(z)->get(Z_OBJ_P(z), &rv2); if (z == &rv) { zval_ptr_dtor(&rv); @@ -1319,7 +1319,7 @@ static void ZEND_FASTCALL zend_jit_assign_dim_bw_xor_helper(zval *container, zva ZVAL_COPY_VALUE(z, value); } bitwise_xor_function(&res, Z_ISREF_P(z) ? Z_REFVAL_P(z) : z, value); - Z_OBJ_HT_P(object)->write_dimension(object, property, &res); + Z_OBJ_HT_P(object)->write_dimension(Z_OBJ_P(object), property, &res); if (z == &rv) { zval_ptr_dtor(&rv); } @@ -1363,11 +1363,11 @@ static void ZEND_FASTCALL zend_jit_assign_dim_sl_helper(zval *container, zval *d zval rv, res; if (Z_OBJ_HT_P(object)->read_dimension && - (z = Z_OBJ_HT_P(object)->read_dimension(object, property, BP_VAR_R, &rv)) != NULL) { + (z = Z_OBJ_HT_P(object)->read_dimension(Z_OBJ_P(object), property, BP_VAR_R, &rv)) != NULL) { if (Z_TYPE_P(z) == IS_OBJECT && Z_OBJ_HT_P(z)->get) { zval rv2; - zval *value = Z_OBJ_HT_P(z)->get(z, &rv2); + zval *value = Z_OBJ_HT_P(z)->get(Z_OBJ_P(z), &rv2); if (z == &rv) { zval_ptr_dtor(&rv); @@ -1375,7 +1375,7 @@ static void ZEND_FASTCALL zend_jit_assign_dim_sl_helper(zval *container, zval *d ZVAL_COPY_VALUE(z, value); } shift_left_function(&res, Z_ISREF_P(z) ? Z_REFVAL_P(z) : z, value); - Z_OBJ_HT_P(object)->write_dimension(object, property, &res); + Z_OBJ_HT_P(object)->write_dimension(Z_OBJ_P(object), property, &res); if (z == &rv) { zval_ptr_dtor(&rv); } @@ -1419,11 +1419,11 @@ static void ZEND_FASTCALL zend_jit_assign_dim_sr_helper(zval *container, zval *d zval rv, res; if (Z_OBJ_HT_P(object)->read_dimension && - (z = Z_OBJ_HT_P(object)->read_dimension(object, property, BP_VAR_R, &rv)) != NULL) { + (z = Z_OBJ_HT_P(object)->read_dimension(Z_OBJ_P(object), property, BP_VAR_R, &rv)) != NULL) { if (Z_TYPE_P(z) == IS_OBJECT && Z_OBJ_HT_P(z)->get) { zval rv2; - zval *value = Z_OBJ_HT_P(z)->get(z, &rv2); + zval *value = Z_OBJ_HT_P(z)->get(Z_OBJ_P(z), &rv2); if (z == &rv) { zval_ptr_dtor(&rv); @@ -1431,7 +1431,7 @@ static void ZEND_FASTCALL zend_jit_assign_dim_sr_helper(zval *container, zval *d ZVAL_COPY_VALUE(z, value); } shift_right_function(&res, Z_ISREF_P(z) ? Z_REFVAL_P(z) : z, value); - Z_OBJ_HT_P(object)->write_dimension(object, property, &res); + Z_OBJ_HT_P(object)->write_dimension(Z_OBJ_P(object), property, &res); if (z == &rv) { zval_ptr_dtor(&rv); } @@ -1475,11 +1475,11 @@ static void ZEND_FASTCALL zend_jit_assign_dim_mod_helper(zval *container, zval * zval rv, res; if (Z_OBJ_HT_P(object)->read_dimension && - (z = Z_OBJ_HT_P(object)->read_dimension(object, property, BP_VAR_R, &rv)) != NULL) { + (z = Z_OBJ_HT_P(object)->read_dimension(Z_OBJ_P(object), property, BP_VAR_R, &rv)) != NULL) { if (Z_TYPE_P(z) == IS_OBJECT && Z_OBJ_HT_P(z)->get) { zval rv2; - zval *value = Z_OBJ_HT_P(z)->get(z, &rv2); + zval *value = Z_OBJ_HT_P(z)->get(Z_OBJ_P(z), &rv2); if (z == &rv) { zval_ptr_dtor(&rv); @@ -1487,7 +1487,7 @@ static void ZEND_FASTCALL zend_jit_assign_dim_mod_helper(zval *container, zval * ZVAL_COPY_VALUE(z, value); } mod_function(&res, Z_ISREF_P(z) ? Z_REFVAL_P(z) : z, value); - Z_OBJ_HT_P(object)->write_dimension(object, property, &res); + Z_OBJ_HT_P(object)->write_dimension(Z_OBJ_P(object), property, &res); if (z == &rv) { zval_ptr_dtor(&rv); } @@ -1531,11 +1531,11 @@ static void ZEND_FASTCALL zend_jit_assign_dim_concat_helper(zval *container, zva zval rv, res; if (Z_OBJ_HT_P(object)->read_dimension && - (z = Z_OBJ_HT_P(object)->read_dimension(object, property, BP_VAR_R, &rv)) != NULL) { + (z = Z_OBJ_HT_P(object)->read_dimension(Z_OBJ_P(object), property, BP_VAR_R, &rv)) != NULL) { if (Z_TYPE_P(z) == IS_OBJECT && Z_OBJ_HT_P(z)->get) { zval rv2; - zval *value = Z_OBJ_HT_P(z)->get(z, &rv2); + zval *value = Z_OBJ_HT_P(z)->get(Z_OBJ_P(z), &rv2); if (z == &rv) { zval_ptr_dtor(&rv); @@ -1543,7 +1543,7 @@ static void ZEND_FASTCALL zend_jit_assign_dim_concat_helper(zval *container, zva ZVAL_COPY_VALUE(z, value); } concat_function(&res, Z_ISREF_P(z) ? Z_REFVAL_P(z) : z, value); - Z_OBJ_HT_P(object)->write_dimension(object, property, &res); + Z_OBJ_HT_P(object)->write_dimension(Z_OBJ_P(object), property, &res); if (z == &rv) { zval_ptr_dtor(&rv); } @@ -1633,7 +1633,7 @@ static int ZEND_FASTCALL zend_jit_isset_dim_helper(zval *container, zval *offset if (EXPECTED(Z_TYPE_P(container) == IS_OBJECT)) { if (EXPECTED(Z_OBJ_HT_P(container)->has_dimension)) { - return Z_OBJ_HT_P(container)->has_dimension(container, offset, 0); + return Z_OBJ_HT_P(container)->has_dimension(Z_OBJ_P(container), offset, 0); } else { zend_error(E_NOTICE, "Trying to check element of non-array"); } @@ -1961,7 +1961,6 @@ static void ZEND_FASTCALL zend_jit_fetch_obj_r_slow(zend_object *zobj, zval *off { zval *retval; zend_execute_data *execute_data = EG(current_execute_data); - zval tmp; if (UNEXPECTED(zobj->handlers->read_property == NULL)) { zend_string *property_name = zval_get_string(offset); @@ -1969,8 +1968,11 @@ static void ZEND_FASTCALL zend_jit_fetch_obj_r_slow(zend_object *zobj, zval *off zend_string_release(property_name); ZVAL_NULL(result); } else { - ZVAL_OBJ(&tmp, zobj); - retval = zobj->handlers->read_property(&tmp, offset, BP_VAR_R, CACHE_ADDR(cache_slot), result); + zend_string *name, *tmp_name; + + name = zval_get_tmp_string(offset, &tmp_name); + retval = zobj->handlers->read_property(zobj, name, BP_VAR_R, CACHE_ADDR(cache_slot), result); + zend_tmp_string_release(tmp_name); if (retval != result) { ZVAL_COPY_DEREF(result, retval); } else if (UNEXPECTED(Z_ISREF_P(retval))) { @@ -2019,13 +2021,15 @@ static void ZEND_FASTCALL zend_jit_fetch_obj_is_slow(zend_object *zobj, zval *of { zval *retval; zend_execute_data *execute_data = EG(current_execute_data); - zval tmp; if (UNEXPECTED(zobj->handlers->read_property == NULL)) { ZVAL_NULL(result); } else { - ZVAL_OBJ(&tmp, zobj); - retval = zobj->handlers->read_property(&tmp, offset, BP_VAR_IS, CACHE_ADDR(cache_slot), result); + zend_string *name, *tmp_name; + + name = zval_get_tmp_string(offset, &tmp_name); + retval = zobj->handlers->read_property(zobj, name, BP_VAR_IS, CACHE_ADDR(cache_slot), result); + zend_tmp_string_release(tmp_name); if (retval != result) { ZVAL_COPY_DEREF(result, retval); } else if (UNEXPECTED(Z_ISREF_P(retval))) { From 5ebee5ceb3e074e42835df7f6ffb673f1e69cee7 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 4 Feb 2019 16:19:20 +0300 Subject: [PATCH 531/569] Fixed compatibility with changes intriduced by typed properties patch --- ext/opcache/jit/zend_jit_x86.dasc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index c48aabc3a79c4..8f28ede0986fa 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -9279,10 +9279,10 @@ static int zend_jit_fetch_obj_read(dasm_State **Dst, zend_op *opline, zend_op_ar if (offset == ZEND_WRONG_PROPERTY_OFFSET) { | mov r0, EX->run_time_cache - | mov r2, aword [r0 + opline->extended_value] + | mov r2, aword [r0 + (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS)] | cmp r2, aword [FCARG1a + offsetof(zend_object, ce)] | jne >5 - | mov r0, aword [r0 + opline->extended_value + sizeof(void*)] + | mov r0, aword [r0 + (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*)] may_be_dynamic = zend_may_be_dynamic_property(ce, Z_STR_P(member), opline->op1_type == IS_UNUSED, op_array->filename); if (may_be_dynamic) { | test r0, r0 @@ -9319,10 +9319,10 @@ static int zend_jit_fetch_obj_read(dasm_State **Dst, zend_op *opline, zend_op_ar | LOAD_ADDR FCARG2a, member |.if X64 | LOAD_ZVAL_ADDR CARG3, res_addr - | mov CARG4, opline->extended_value + | mov CARG4, (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) |.else | sub r4, 8 - | push opline->extended_value + | push (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) | PUSH_ZVAL_ADDR res_addr, r0 |.endif | SAVE_VALID_OPLINE opline @@ -9386,10 +9386,10 @@ static int zend_jit_fetch_obj_read(dasm_State **Dst, zend_op *opline, zend_op_ar |.if X64 | LOAD_ADDR CARG3, member | LOAD_ZVAL_ADDR CARG4, res_addr - | mov CARG5, opline->extended_value + | mov CARG5, (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) |.else | sub r4, 4 - | push opline->extended_value + | push (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) | PUSH_ZVAL_ADDR res_addr, r0 | PUSH_ADDR member, r0 |.endif From 597aa207c9a04756168cd97aeff58563b274b598 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 4 Feb 2019 17:15:37 +0300 Subject: [PATCH 532/569] - renumber ZEND_JIT_DEBUG_* codes - use different options to disassemble staubs and the JIT-ed PHP - disable implicit automatic JIT code registration in DEBUG build --- ext/opcache/jit/zend_jit.c | 38 ++++++++++++++++++---------- ext/opcache/jit/zend_jit.h | 8 +++--- ext/opcache/jit/zend_jit_gdb.c | 4 ++- ext/opcache/jit/zend_jit_perf_dump.c | 4 +-- 4 files changed, 34 insertions(+), 20 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index fbde9aeeac7ea..d6f93216c31ab 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -297,19 +297,29 @@ static void *dasm_link_and_encode(dasm_State **dasm_state, name = ZSTR_VAL(str); } } - } -#endif - #ifdef HAVE_DISASM - if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_ASM) { - zend_jit_disasm_add_symbol(name, (uintptr_t)entry, size); - zend_jit_disasm( - name, - (op_array && op_array->filename) ? ZSTR_VAL(op_array->filename) : NULL, - op_array, - &ssa->cfg, - entry, - size); + if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_ASM) { + zend_jit_disasm_add_symbol(name, (uintptr_t)entry, size); + zend_jit_disasm( + name, + (op_array && op_array->filename) ? ZSTR_VAL(op_array->filename) : NULL, + op_array, + &ssa->cfg, + entry, + size); + } + } else { + if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_ASM_STUBS) { + zend_jit_disasm_add_symbol(name, (uintptr_t)entry, size); + zend_jit_disasm( + name, + (op_array && op_array->filename) ? ZSTR_VAL(op_array->filename) : NULL, + op_array, + &ssa->cfg, + entry, + size); + } +# endif } #endif @@ -3128,7 +3138,7 @@ ZEND_API int zend_jit_startup(zend_long jit, size_t size) zend_jit_protect(); #ifdef HAVE_DISASM - if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_ASM) { + if (ZCG(accel_directives).jit_debug & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_ASM_STUBS)) { if (!zend_jit_disasm_init()) { // TODO: error reporting and cleanup ??? return FAILURE; @@ -3169,7 +3179,7 @@ ZEND_API void zend_jit_shutdown(void) #endif #ifdef HAVE_DISASM - if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_ASM) { + if (ZCG(accel_directives).jit_debug & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_ASM_STUBS)) { zend_jit_disasm_shutdown(); } #endif diff --git a/ext/opcache/jit/zend_jit.h b/ext/opcache/jit/zend_jit.h index d8a072e6cbf2d..825b8a51d157a 100644 --- a/ext/opcache/jit/zend_jit.h +++ b/ext/opcache/jit/zend_jit.h @@ -68,12 +68,14 @@ #define ZEND_JIT_DEBUG_ASM (1<<0) #define ZEND_JIT_DEBUG_SSA (1<<1) #define ZEND_JIT_DEBUG_REG_ALLOC (1<<2) +#define ZEND_JIT_DEBUG_ASM_STUBS (1<<3) -#define ZEND_JIT_DEBUG_GDB (1<<4) -#define ZEND_JIT_DEBUG_PERF (1<<5) +#define ZEND_JIT_DEBUG_PERF (1<<4) +#define ZEND_JIT_DEBUG_PERF_DUMP (1<<5) #define ZEND_JIT_DEBUG_OPROFILE (1<<6) #define ZEND_JIT_DEBUG_VTUNE (1<<7) -#define ZEND_JIT_DEBUG_PERF_DUMP (1<<8) + +#define ZEND_JIT_DEBUG_GDB (1<<8) ZEND_API int zend_jit_op_array(zend_op_array *op_array, zend_script *script); ZEND_API int zend_jit_script(zend_script *script); diff --git a/ext/opcache/jit/zend_jit_gdb.c b/ext/opcache/jit/zend_jit_gdb.c index 5062037da849d..e66c1b5d4b6d4 100644 --- a/ext/opcache/jit/zend_jit_gdb.c +++ b/ext/opcache/jit/zend_jit_gdb.c @@ -481,7 +481,9 @@ static int zend_jit_gdb_unregister(void) static void zend_jit_gdb_init(void) { -#if 1 +#if 0 + /* This might enable registration of all JIT-ed code, but unfortunately, + * in case of many functions, this takes enormous time. */ if (zend_gdb_present()) { ZCG(accel_directives).jit_debug |= ZEND_JIT_DEBUG_GDB; } diff --git a/ext/opcache/jit/zend_jit_perf_dump.c b/ext/opcache/jit/zend_jit_perf_dump.c index 097a1bee67faa..89f7e16bfd9a3 100644 --- a/ext/opcache/jit/zend_jit_perf_dump.c +++ b/ext/opcache/jit/zend_jit_perf_dump.c @@ -29,12 +29,12 @@ /* * 1) Profile using perf-.map * - * perf record php -d opcache.huge_code_pages=0 -d opcache.jit_debug=0x20 bench.php + * perf record php -d opcache.huge_code_pages=0 -d opcache.jit_debug=0x10 bench.php * perf report * * 2) Profile using jit-.dump * - * perf record php -d opcache.huge_code_pages=0 -d opcache.jit_debug=0x100 bench.php + * perf record php -d opcache.huge_code_pages=0 -d opcache.jit_debug=0x20 bench.php * perf inject -j -i perf.data -o perf.data.jitted * perf report -i perf.data.jitted * From 2436437b77d1e348d4f5d43cf3e1551ce84edca9 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 4 Feb 2019 20:30:39 +0300 Subject: [PATCH 533/569] Fixed possible incorrect code generation for ZEND_SL, ZEND_SR and ZEND_MOD. --- ext/opcache/jit/zend_jit_x86.dasc | 59 ++++++++++++++-------- ext/opcache/tests/jit/shift_right_003.phpt | 26 ++++++++++ 2 files changed, 64 insertions(+), 21 deletions(-) create mode 100644 ext/opcache/tests/jit/shift_right_003.phpt diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 8f28ede0986fa..88ea13834203b 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -2180,6 +2180,16 @@ static int zend_jit_save_call_chain(dasm_State **Dst, uint32_t call_level) return 1; } +static int zend_jit_set_ip(dasm_State **Dst, const zend_op *opline) +{ + if (!last_valid_opline) { + | LOAD_ADDR IP, opline + } else if (last_valid_opline != opline) { + | add IP, (opline - last_valid_opline) * sizeof(zend_op); + } + return 1; +} + static int zend_jit_set_valid_ip(dasm_State **Dst, const zend_op *opline) { if (delayed_call_chain) { @@ -2187,10 +2197,8 @@ static int zend_jit_set_valid_ip(dasm_State **Dst, const zend_op *opline) return 0; } } - if (!last_valid_opline) { - | LOAD_ADDR IP, opline - } else if (last_valid_opline != opline) { - | add IP, (opline - last_valid_opline) * sizeof(zend_op); + if (!zend_jit_set_ip(Dst, opline)) { + return 0; } last_valid_opline = opline; reuse_ip = 0; @@ -3361,7 +3369,7 @@ static int zend_jit_long_math_helper(dasm_State **Dst, if (EXPECTED(op2_lval > 0)) { | xor Ra(result_reg), Ra(result_reg) } else { - if (!zend_jit_set_valid_ip(Dst, opline)) { + if (!zend_jit_set_ip(Dst, opline)) { return 0; } | jmp ->negative_shift @@ -3378,16 +3386,17 @@ static int zend_jit_long_math_helper(dasm_State **Dst, !ssa->var_info[op2_ssa_var].has_range || ssa->var_info[op2_ssa_var].range.min < 0 || ssa->var_info[op2_ssa_var].range.max >= SIZEOF_ZEND_LONG * 8) { - if (!zend_jit_set_valid_ip(Dst, opline)) { - return 0; - } | cmp r1, (SIZEOF_ZEND_LONG*8) | jae >1 |.cold_code |1: - | jl ->negative_shift - | xor Ra(result_reg), Ra(result_reg) - | jmp >1 + | cmp r1, 0 + | mov Ra(result_reg), 0 + | jg >1 + if (!zend_jit_set_ip(Dst, opline)) { + return 0; + } + | jmp ->negative_shift |.code } | GET_ZVAL_LVAL result_reg, op1_addr @@ -3403,7 +3412,7 @@ static int zend_jit_long_math_helper(dasm_State **Dst, if (EXPECTED(op2_lval > 0)) { | sar Ra(result_reg), (SIZEOF_ZEND_LONG * 8) - 1 } else { - if (!zend_jit_set_valid_ip(Dst, opline)) { + if (!zend_jit_set_ip(Dst, opline)) { return 0; } | jmp ->negative_shift @@ -3419,27 +3428,28 @@ static int zend_jit_long_math_helper(dasm_State **Dst, !ssa->var_info[op2_ssa_var].has_range || ssa->var_info[op2_ssa_var].range.min < 0 || ssa->var_info[op2_ssa_var].range.max >= SIZEOF_ZEND_LONG * 8) { - if (!zend_jit_set_valid_ip(Dst, opline)) { - return 0; - } | cmp r1, (SIZEOF_ZEND_LONG*8) | jae >1 |.cold_code |1: - | jl ->negative_shift - | sar Ra(result_reg), (SIZEOF_ZEND_LONG * 8) - 1 - | jmp >1 + | cmp r1, 0 + | mov r1, (SIZEOF_ZEND_LONG * 8) - 1 + | jg >1 + if (!zend_jit_set_ip(Dst, opline)) { + return 0; + } + | jmp ->negative_shift |.code } - | sar Ra(result_reg), cl |1: + | sar Ra(result_reg), cl } } else if (opcode == ZEND_MOD || opcode == ZEND_ASSIGN_MOD) { if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr)); if (op2_lval == 0) { - if (!zend_jit_set_valid_ip(Dst, opline)) { + if (!zend_jit_set_ip(Dst, opline)) { return 0; } | jmp ->mod_by_zero @@ -3469,7 +3479,14 @@ static int zend_jit_long_math_helper(dasm_State **Dst, } else if (Z_MODE(op2_addr) == IS_REG) { | test Ra(Z_REG(op2_addr)), Ra(Z_REG(op2_addr)) } - | jz ->mod_by_zero + | jz >1 + |.cold_code + |1: + if (!zend_jit_set_ip(Dst, opline)) { + return 0; + } + | jmp ->mod_by_zero + |.code } //TODO: add check for -1 ??? diff --git a/ext/opcache/tests/jit/shift_right_003.phpt b/ext/opcache/tests/jit/shift_right_003.phpt new file mode 100644 index 0000000000000..5f5bf3c40d416 --- /dev/null +++ b/ext/opcache/tests/jit/shift_right_003.phpt @@ -0,0 +1,26 @@ +--TEST-- +JIT Shift Right: 003 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +opcache.protect_memory=1 +--SKIPIF-- + +--FILE-- +> $i) > 0x80; $i += 7) { + $out .= \chr(0x80 | (($int >> $i) & 0x7f)); + } + return $out . \chr($int >> $i); +} +$s = encodeDynamicInteger(235); +var_dump(strlen($s), ord($s[0]), ord($s[1])); +?> +--EXPECT-- +int(2) +int(235) +int(1) From 08c891dfd6a03e6e454bbef0967290266fbcc5eb Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 4 Feb 2019 20:50:23 +0300 Subject: [PATCH 534/569] --disable-opcache-jit just an option (not an extension) --- ext/opcache/config.m4 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opcache/config.m4 b/ext/opcache/config.m4 index b389f35a62b15..0261fbaa7cca7 100644 --- a/ext/opcache/config.m4 +++ b/ext/opcache/config.m4 @@ -11,7 +11,7 @@ PHP_ARG_ENABLE(huge-code-pages, whether to enable copying PHP CODE pages into HU Disable copying PHP CODE pages into HUGE PAGES], yes, no) PHP_ARG_ENABLE(opcache-jit, whether to enable JIT, -[ --disable-opcache-jit Disable JIT], yes) +[ --disable-opcache-jit Disable JIT], yes, no) if test "$PHP_OPCACHE" != "no"; then From baf3db21de5b3e18f3a432d02ec2a89abcadb351 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 5 Feb 2019 10:20:19 +0300 Subject: [PATCH 535/569] Remove editors "Local Variales" and "$Id:$" attibute. --- Zend/zend_gdb.c | 10 +--------- Zend/zend_gdb.h | 8 -------- ext/opcache/jit/zend_elf.c | 8 -------- ext/opcache/jit/zend_elf.h | 8 -------- ext/opcache/jit/zend_jit.c | 9 +-------- ext/opcache/jit/zend_jit.h | 10 ---------- ext/opcache/jit/zend_jit_disasm_x86.c | 8 -------- ext/opcache/jit/zend_jit_gdb.c | 7 ------- ext/opcache/jit/zend_jit_helpers.c | 8 -------- ext/opcache/jit/zend_jit_internal.h | 8 -------- ext/opcache/jit/zend_jit_oprofile.c | 8 -------- ext/opcache/jit/zend_jit_perf_dump.c | 8 -------- ext/opcache/jit/zend_jit_vm_helpers.c | 8 -------- ext/opcache/jit/zend_jit_vtune.c | 7 ------- ext/opcache/jit/zend_jit_x86.h | 7 ------- 15 files changed, 2 insertions(+), 120 deletions(-) diff --git a/Zend/zend_gdb.c b/Zend/zend_gdb.c index 91c887a1e9ad1..199589f5b8fc9 100644 --- a/Zend/zend_gdb.c +++ b/Zend/zend_gdb.c @@ -2,7 +2,7 @@ +----------------------------------------------------------------------+ | Zend Engine | +----------------------------------------------------------------------+ - | Copyright (c) 1998-2017 Zend Technologies Ltd. (http://www.zend.com) | + | Copyright (c) Zend Technologies Ltd. (http://www.zend.com) | +----------------------------------------------------------------------+ | This source file is subject to version 2.00 of the Zend license, | | that is bundled with this package in the file LICENSE, and is | @@ -138,11 +138,3 @@ ZEND_API int zend_gdb_present(void) return ret; } - -/* - * Local variables: - * tab-width: 4 - * c-basic-offset: 4 - * indent-tabs-mode: t - * End: - */ diff --git a/Zend/zend_gdb.h b/Zend/zend_gdb.h index 203bc9556e6df..7869960eda294 100644 --- a/Zend/zend_gdb.h +++ b/Zend/zend_gdb.h @@ -25,11 +25,3 @@ ZEND_API void zend_gdb_unregister_all(void); ZEND_API int zend_gdb_present(void); #endif - -/* - * Local variables: - * tab-width: 4 - * c-basic-offset: 4 - * indent-tabs-mode: t - * End: - */ diff --git a/ext/opcache/jit/zend_elf.c b/ext/opcache/jit/zend_elf.c index ac0287f341c35..aac3ab9181746 100644 --- a/ext/opcache/jit/zend_elf.c +++ b/ext/opcache/jit/zend_elf.c @@ -86,11 +86,3 @@ void zend_elf_load_symbols(void) close(fd); } } - -/* - * Local variables: - * tab-width: 4 - * c-basic-offset: 4 - * indent-tabs-mode: t - * End: - */ diff --git a/ext/opcache/jit/zend_elf.h b/ext/opcache/jit/zend_elf.h index 87d9cc9fe5148..6c75bd003b584 100644 --- a/ext/opcache/jit/zend_elf.h +++ b/ext/opcache/jit/zend_elf.h @@ -104,11 +104,3 @@ enum { void zend_elf_load_symbols(void); #endif - -/* - * Local variables: - * tab-width: 4 - * c-basic-offset: 4 - * indent-tabs-mode: t - * End: - */ diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index d6f93216c31ab..d9f4a88e256e4 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -477,6 +477,7 @@ static int zend_may_overflow(const zend_op *opline, zend_op_array *op_array, zen uint32_t num; int res; + return 1; if (!ssa->ops || !ssa->var_info) { return 1; } @@ -3273,11 +3274,3 @@ ZEND_API void zend_jit_deactivate(void) } #endif /* HAVE_JIT */ - -/* - * Local variables: - * tab-width: 4 - * c-basic-offset: 4 - * indent-tabs-mode: t - * End: - */ diff --git a/ext/opcache/jit/zend_jit.h b/ext/opcache/jit/zend_jit.h index 825b8a51d157a..289ccc5afcf50 100644 --- a/ext/opcache/jit/zend_jit.h +++ b/ext/opcache/jit/zend_jit.h @@ -16,8 +16,6 @@ +----------------------------------------------------------------------+ */ -/* $Id:$ */ - #ifndef HAVE_JIT_H #define HAVE_JIT_H @@ -109,11 +107,3 @@ struct _zend_lifetime_interval { }; #endif /* HAVE_JIT_H */ - -/* - * Local variables: - * tab-width: 4 - * c-basic-offset: 4 - * indent-tabs-mode: t - * End: - */ diff --git a/ext/opcache/jit/zend_jit_disasm_x86.c b/ext/opcache/jit/zend_jit_disasm_x86.c index 0d02613754300..66e6adcba8bcf 100644 --- a/ext/opcache/jit/zend_jit_disasm_x86.c +++ b/ext/opcache/jit/zend_jit_disasm_x86.c @@ -453,11 +453,3 @@ static void zend_jit_disasm_shutdown(void) { zend_jit_disasm_destroy_symbols(symbols); } - -/* - * Local variables: - * tab-width: 4 - * c-basic-offset: 4 - * indent-tabs-mode: t - * End: - */ diff --git a/ext/opcache/jit/zend_jit_gdb.c b/ext/opcache/jit/zend_jit_gdb.c index e66c1b5d4b6d4..1bbd5303be882 100644 --- a/ext/opcache/jit/zend_jit_gdb.c +++ b/ext/opcache/jit/zend_jit_gdb.c @@ -491,10 +491,3 @@ static void zend_jit_gdb_init(void) } #endif -/* - * Local variables: - * tab-width: 4 - * c-basic-offset: 4 - * indent-tabs-mode: t - * End: - */ diff --git a/ext/opcache/jit/zend_jit_helpers.c b/ext/opcache/jit/zend_jit_helpers.c index 315064beb84d6..6b6687b69d4a8 100644 --- a/ext/opcache/jit/zend_jit_helpers.c +++ b/ext/opcache/jit/zend_jit_helpers.c @@ -2225,11 +2225,3 @@ static void ZEND_FASTCALL zend_jit_post_dec_typed_ref(zval *var_ptr, zend_refere ZVAL_COPY_VALUE(var_ptr, ret); } } - -/* - * Local variables: - * tab-width: 4 - * c-basic-offset: 4 - * indent-tabs-mode: t - * End: - */ diff --git a/ext/opcache/jit/zend_jit_internal.h b/ext/opcache/jit/zend_jit_internal.h index f6b51d5183d82..e6c4f0c78d923 100644 --- a/ext/opcache/jit/zend_jit_internal.h +++ b/ext/opcache/jit/zend_jit_internal.h @@ -50,11 +50,3 @@ void ZEND_FASTCALL zend_jit_get_constant(const zval *key, uint32_t flags); int ZEND_FASTCALL zend_jit_check_constant(const zval *key); #endif /* ZEND_JIT_INTERNAL_H */ - -/* - * Local variables: - * tab-width: 4 - * c-basic-offset: 4 - * indent-tabs-mode: t - * End: - */ diff --git a/ext/opcache/jit/zend_jit_oprofile.c b/ext/opcache/jit/zend_jit_oprofile.c index a6221515f3854..7f9f849dfc5dd 100644 --- a/ext/opcache/jit/zend_jit_oprofile.c +++ b/ext/opcache/jit/zend_jit_oprofile.c @@ -48,11 +48,3 @@ static void zend_jit_oprofile_shutdown(void) op_close_agent(op_agent); } } - -/* - * Local variables: - * tab-width: 4 - * c-basic-offset: 4 - * indent-tabs-mode: t - * End: - */ diff --git a/ext/opcache/jit/zend_jit_perf_dump.c b/ext/opcache/jit/zend_jit_perf_dump.c index 89f7e16bfd9a3..b4d1f0435e781 100644 --- a/ext/opcache/jit/zend_jit_perf_dump.c +++ b/ext/opcache/jit/zend_jit_perf_dump.c @@ -205,11 +205,3 @@ static void zend_jit_perf_map_register(const char *name, void *start, size_t siz } fprintf(fp, "%zx %zx %s\n", (size_t)(uintptr_t)start, size, name); } - -/* - * Local variables: - * tab-width: 4 - * c-basic-offset: 4 - * indent-tabs-mode: t - * End: - */ diff --git a/ext/opcache/jit/zend_jit_vm_helpers.c b/ext/opcache/jit/zend_jit_vm_helpers.c index c7f7afea43e3e..9b28a31cd33bb 100644 --- a/ext/opcache/jit/zend_jit_vm_helpers.c +++ b/ext/opcache/jit/zend_jit_vm_helpers.c @@ -254,11 +254,3 @@ int ZEND_FASTCALL zend_jit_check_constant(const zval *key) { return _zend_quick_get_constant(key, 0, 1); } - -/* - * Local variables: - * tab-width: 4 - * c-basic-offset: 4 - * indent-tabs-mode: t - * End: - */ diff --git a/ext/opcache/jit/zend_jit_vtune.c b/ext/opcache/jit/zend_jit_vtune.c index 0fa2ce5a25fee..853a00fb8cd58 100644 --- a/ext/opcache/jit/zend_jit_vtune.c +++ b/ext/opcache/jit/zend_jit_vtune.c @@ -40,10 +40,3 @@ static void zend_jit_vtune_register(const char *name, iJIT_NotifyEvent(iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED, (void*)&jmethod); } -/* - * Local variables: - * tab-width: 4 - * c-basic-offset: 4 - * indent-tabs-mode: t - * End: - */ diff --git a/ext/opcache/jit/zend_jit_x86.h b/ext/opcache/jit/zend_jit_x86.h index 3bd812c4fb915..cec9b4a06b3a6 100644 --- a/ext/opcache/jit/zend_jit_x86.h +++ b/ext/opcache/jit/zend_jit_x86.h @@ -217,10 +217,3 @@ static zend_always_inline zend_bool zend_jit_same_addr(zend_jit_addr addr1, zend } #endif /* ZEND_JIT_X86_H */ -/* - * Local variables: - * tab-width: 4 - * c-basic-offset: 4 - * indent-tabs-mode: t - * End: - */ From f8dd0fe11927604bf86495b2f86415d108a87c3f Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 5 Feb 2019 10:45:24 +0300 Subject: [PATCH 536/569] Remove copyright years and change email address --- Zend/zend_gdb.h | 2 +- ext/opcache/jit/zend_elf.c | 4 ++-- ext/opcache/jit/zend_elf.h | 4 ++-- ext/opcache/jit/zend_jit.c | 4 ++-- ext/opcache/jit/zend_jit.h | 4 ++-- ext/opcache/jit/zend_jit_disasm_x86.c | 4 ++-- ext/opcache/jit/zend_jit_gdb.c | 4 ++-- ext/opcache/jit/zend_jit_helpers.c | 4 ++-- ext/opcache/jit/zend_jit_internal.h | 4 ++-- ext/opcache/jit/zend_jit_oprofile.c | 4 ++-- ext/opcache/jit/zend_jit_perf_dump.c | 4 ++-- ext/opcache/jit/zend_jit_vm_helpers.c | 4 ++-- ext/opcache/jit/zend_jit_vtune.c | 4 ++-- ext/opcache/jit/zend_jit_x86.dasc | 4 ++-- ext/opcache/jit/zend_jit_x86.h | 4 ++-- 15 files changed, 29 insertions(+), 29 deletions(-) diff --git a/Zend/zend_gdb.h b/Zend/zend_gdb.h index 7869960eda294..744add429b34a 100644 --- a/Zend/zend_gdb.h +++ b/Zend/zend_gdb.h @@ -2,7 +2,7 @@ +----------------------------------------------------------------------+ | Zend Engine | +----------------------------------------------------------------------+ - | Copyright (c) 1998-2017 Zend Technologies Ltd. (http://www.zend.com) | + | Copyright (c) Zend Technologies Ltd. (http://www.zend.com) | +----------------------------------------------------------------------+ | This source file is subject to version 2.00 of the Zend license, | | that is bundled with this package in the file LICENSE, and is | diff --git a/ext/opcache/jit/zend_elf.c b/ext/opcache/jit/zend_elf.c index aac3ab9181746..1ad1354f0364d 100644 --- a/ext/opcache/jit/zend_elf.c +++ b/ext/opcache/jit/zend_elf.c @@ -2,7 +2,7 @@ +----------------------------------------------------------------------+ | Zend JIT | +----------------------------------------------------------------------+ - | Copyright (c) 1998-2016 The PHP Group | + | Copyright (c) The PHP Group | +----------------------------------------------------------------------+ | This source file is subject to version 3.01 of the PHP license, | | that is bundled with this package in the file LICENSE, and is | @@ -12,7 +12,7 @@ | obtain it through the world-wide-web, please send a note to | | license@php.net so we can mail you a copy immediately. | +----------------------------------------------------------------------+ - | Authors: Dmitry Stogov | + | Authors: Dmitry Stogov | | Xinchen Hui | +----------------------------------------------------------------------+ */ diff --git a/ext/opcache/jit/zend_elf.h b/ext/opcache/jit/zend_elf.h index 6c75bd003b584..2c7e01ae12b69 100644 --- a/ext/opcache/jit/zend_elf.h +++ b/ext/opcache/jit/zend_elf.h @@ -2,7 +2,7 @@ +----------------------------------------------------------------------+ | Zend JIT | +----------------------------------------------------------------------+ - | Copyright (c) 1998-2016 The PHP Group | + | Copyright (c) The PHP Group | +----------------------------------------------------------------------+ | This source file is subject to version 3.01 of the PHP license, | | that is bundled with this package in the file LICENSE, and is | @@ -12,7 +12,7 @@ | obtain it through the world-wide-web, please send a note to | | license@php.net so we can mail you a copy immediately. | +----------------------------------------------------------------------+ - | Authors: Dmitry Stogov | + | Authors: Dmitry Stogov | | Xinchen Hui | +----------------------------------------------------------------------+ */ diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index d9f4a88e256e4..e743085166b17 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -2,7 +2,7 @@ +----------------------------------------------------------------------+ | Zend JIT | +----------------------------------------------------------------------+ - | Copyright (c) 1998-2016 The PHP Group | + | Copyright (c) The PHP Group | +----------------------------------------------------------------------+ | This source file is subject to version 3.01 of the PHP license, | | that is bundled with this package in the file LICENSE, and is | @@ -12,7 +12,7 @@ | obtain it through the world-wide-web, please send a note to | | license@php.net so we can mail you a copy immediately. | +----------------------------------------------------------------------+ - | Authors: Dmitry Stogov | + | Authors: Dmitry Stogov | +----------------------------------------------------------------------+ */ diff --git a/ext/opcache/jit/zend_jit.h b/ext/opcache/jit/zend_jit.h index 289ccc5afcf50..acbd3e038851f 100644 --- a/ext/opcache/jit/zend_jit.h +++ b/ext/opcache/jit/zend_jit.h @@ -2,7 +2,7 @@ +----------------------------------------------------------------------+ | Zend JIT | +----------------------------------------------------------------------+ - | Copyright (c) 1998-2016 The PHP Group | + | Copyright (c) The PHP Group | +----------------------------------------------------------------------+ | This source file is subject to version 3.01 of the PHP license, | | that is bundled with this package in the file LICENSE, and is | @@ -12,7 +12,7 @@ | obtain it through the world-wide-web, please send a note to | | license@php.net so we can mail you a copy immediately. | +----------------------------------------------------------------------+ - | Authors: Dmitry Stogov | + | Authors: Dmitry Stogov | +----------------------------------------------------------------------+ */ diff --git a/ext/opcache/jit/zend_jit_disasm_x86.c b/ext/opcache/jit/zend_jit_disasm_x86.c index 66e6adcba8bcf..91130d3b8a895 100644 --- a/ext/opcache/jit/zend_jit_disasm_x86.c +++ b/ext/opcache/jit/zend_jit_disasm_x86.c @@ -2,7 +2,7 @@ +----------------------------------------------------------------------+ | Zend JIT | +----------------------------------------------------------------------+ - | Copyright (c) 1998-2016 The PHP Group | + | Copyright (c) The PHP Group | +----------------------------------------------------------------------+ | This source file is subject to version 3.01 of the PHP license, | | that is bundled with this package in the file LICENSE, and is | @@ -12,7 +12,7 @@ | obtain it through the world-wide-web, please send a note to | | license@php.net so we can mail you a copy immediately. | +----------------------------------------------------------------------+ - | Authors: Dmitry Stogov | + | Authors: Dmitry Stogov | | Xinchen Hui | +----------------------------------------------------------------------+ */ diff --git a/ext/opcache/jit/zend_jit_gdb.c b/ext/opcache/jit/zend_jit_gdb.c index 1bbd5303be882..a554bc20d9456 100644 --- a/ext/opcache/jit/zend_jit_gdb.c +++ b/ext/opcache/jit/zend_jit_gdb.c @@ -2,7 +2,7 @@ +----------------------------------------------------------------------+ | Zend JIT | +----------------------------------------------------------------------+ - | Copyright (c) 1998-2016 The PHP Group | + | Copyright (c) The PHP Group | +----------------------------------------------------------------------+ | This source file is subject to version 3.01 of the PHP license, | | that is bundled with this package in the file LICENSE, and is | @@ -12,7 +12,7 @@ | obtain it through the world-wide-web, please send a note to | | license@php.net so we can mail you a copy immediately. | +----------------------------------------------------------------------+ - | Authors: Dmitry Stogov | + | Authors: Dmitry Stogov | | Xinchen Hui | +----------------------------------------------------------------------+ | Based on Mike Pall's implementation of GDB interface for LuaJIT. | diff --git a/ext/opcache/jit/zend_jit_helpers.c b/ext/opcache/jit/zend_jit_helpers.c index 6b6687b69d4a8..c2cdf3dcb1a21 100644 --- a/ext/opcache/jit/zend_jit_helpers.c +++ b/ext/opcache/jit/zend_jit_helpers.c @@ -2,7 +2,7 @@ +----------------------------------------------------------------------+ | Zend JIT | +----------------------------------------------------------------------+ - | Copyright (c) 1998-2016 The PHP Group | + | Copyright (c) The PHP Group | +----------------------------------------------------------------------+ | This source file is subject to version 3.01 of the PHP license, | | that is bundled with this package in the file LICENSE, and is | @@ -12,7 +12,7 @@ | obtain it through the world-wide-web, please send a note to | | license@php.net so we can mail you a copy immediately. | +----------------------------------------------------------------------+ - | Authors: Dmitry Stogov | + | Authors: Dmitry Stogov | +----------------------------------------------------------------------+ */ diff --git a/ext/opcache/jit/zend_jit_internal.h b/ext/opcache/jit/zend_jit_internal.h index e6c4f0c78d923..47a8eb54ea170 100644 --- a/ext/opcache/jit/zend_jit_internal.h +++ b/ext/opcache/jit/zend_jit_internal.h @@ -2,7 +2,7 @@ +----------------------------------------------------------------------+ | Zend JIT | +----------------------------------------------------------------------+ - | Copyright (c) 1998-2016 The PHP Group | + | Copyright (c) The PHP Group | +----------------------------------------------------------------------+ | This source file is subject to version 3.01 of the PHP license, | | that is bundled with this package in the file LICENSE, and is | @@ -12,7 +12,7 @@ | obtain it through the world-wide-web, please send a note to | | license@php.net so we can mail you a copy immediately. | +----------------------------------------------------------------------+ - | Authors: Dmitry Stogov | + | Authors: Dmitry Stogov | | Xinchen Hui | +----------------------------------------------------------------------+ */ diff --git a/ext/opcache/jit/zend_jit_oprofile.c b/ext/opcache/jit/zend_jit_oprofile.c index 7f9f849dfc5dd..50dd02727945f 100644 --- a/ext/opcache/jit/zend_jit_oprofile.c +++ b/ext/opcache/jit/zend_jit_oprofile.c @@ -2,7 +2,7 @@ +----------------------------------------------------------------------+ | Zend JIT | +----------------------------------------------------------------------+ - | Copyright (c) 1998-2016 The PHP Group | + | Copyright (c) The PHP Group | +----------------------------------------------------------------------+ | This source file is subject to version 3.01 of the PHP license, | | that is bundled with this package in the file LICENSE, and is | @@ -12,7 +12,7 @@ | obtain it through the world-wide-web, please send a note to | | license@php.net so we can mail you a copy immediately. | +----------------------------------------------------------------------+ - | Authors: Dmitry Stogov | + | Authors: Dmitry Stogov | +----------------------------------------------------------------------+ */ diff --git a/ext/opcache/jit/zend_jit_perf_dump.c b/ext/opcache/jit/zend_jit_perf_dump.c index b4d1f0435e781..3546208b170f5 100644 --- a/ext/opcache/jit/zend_jit_perf_dump.c +++ b/ext/opcache/jit/zend_jit_perf_dump.c @@ -2,7 +2,7 @@ +----------------------------------------------------------------------+ | Zend JIT | +----------------------------------------------------------------------+ - | Copyright (c) 1998-2016 The PHP Group | + | Copyright (c) The PHP Group | +----------------------------------------------------------------------+ | This source file is subject to version 3.01 of the PHP license, | | that is bundled with this package in the file LICENSE, and is | @@ -12,7 +12,7 @@ | obtain it through the world-wide-web, please send a note to | | license@php.net so we can mail you a copy immediately. | +----------------------------------------------------------------------+ - | Authors: Dmitry Stogov | + | Authors: Dmitry Stogov | +----------------------------------------------------------------------+ */ diff --git a/ext/opcache/jit/zend_jit_vm_helpers.c b/ext/opcache/jit/zend_jit_vm_helpers.c index 9b28a31cd33bb..24fd3b3de46e2 100644 --- a/ext/opcache/jit/zend_jit_vm_helpers.c +++ b/ext/opcache/jit/zend_jit_vm_helpers.c @@ -2,7 +2,7 @@ +----------------------------------------------------------------------+ | Zend JIT | +----------------------------------------------------------------------+ - | Copyright (c) 1998-2016 The PHP Group | + | Copyright (c) The PHP Group | +----------------------------------------------------------------------+ | This source file is subject to version 3.01 of the PHP license, | | that is bundled with this package in the file LICENSE, and is | @@ -12,7 +12,7 @@ | obtain it through the world-wide-web, please send a note to | | license@php.net so we can mail you a copy immediately. | +----------------------------------------------------------------------+ - | Authors: Dmitry Stogov | + | Authors: Dmitry Stogov | | Xinchen Hui | +----------------------------------------------------------------------+ */ diff --git a/ext/opcache/jit/zend_jit_vtune.c b/ext/opcache/jit/zend_jit_vtune.c index 853a00fb8cd58..877afa8606027 100644 --- a/ext/opcache/jit/zend_jit_vtune.c +++ b/ext/opcache/jit/zend_jit_vtune.c @@ -2,7 +2,7 @@ +----------------------------------------------------------------------+ | Zend JIT | +----------------------------------------------------------------------+ - | Copyright (c) 1998-2016 The PHP Group | + | Copyright (c) The PHP Group | +----------------------------------------------------------------------+ | This source file is subject to version 3.01 of the PHP license, | | that is bundled with this package in the file LICENSE, and is | @@ -12,7 +12,7 @@ | obtain it through the world-wide-web, please send a note to | | license@php.net so we can mail you a copy immediately. | +----------------------------------------------------------------------+ - | Authors: Dmitry Stogov | + | Authors: Dmitry Stogov | +----------------------------------------------------------------------+ */ diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 88ea13834203b..bb0617dba718d 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -2,7 +2,7 @@ * +----------------------------------------------------------------------+ * | Zend JIT | * +----------------------------------------------------------------------+ - * | Copyright (c) 1998-2016 The PHP Group | + * | Copyright (c) The PHP Group | * +----------------------------------------------------------------------+ * | This source file is subject to version 3.01 of the PHP license, | * | that is bundled with this package in the file LICENSE, and is | @@ -12,7 +12,7 @@ * | obtain it through the world-wide-web, please send a note to | * | license@php.net so we can mail you a copy immediately. | * +----------------------------------------------------------------------+ - * | Authors: Dmitry Stogov | + * | Authors: Dmitry Stogov | * | Xinchen Hui | * +----------------------------------------------------------------------+ */ diff --git a/ext/opcache/jit/zend_jit_x86.h b/ext/opcache/jit/zend_jit_x86.h index cec9b4a06b3a6..ff6e2c0c466f1 100644 --- a/ext/opcache/jit/zend_jit_x86.h +++ b/ext/opcache/jit/zend_jit_x86.h @@ -2,7 +2,7 @@ +----------------------------------------------------------------------+ | Zend JIT | +----------------------------------------------------------------------+ - | Copyright (c) 1998-2016 The PHP Group | + | Copyright (c) The PHP Group | +----------------------------------------------------------------------+ | This source file is subject to version 3.01 of the PHP license, | | that is bundled with this package in the file LICENSE, and is | @@ -12,7 +12,7 @@ | obtain it through the world-wide-web, please send a note to | | license@php.net so we can mail you a copy immediately. | +----------------------------------------------------------------------+ - | Authors: Dmitry Stogov | + | Authors: Dmitry Stogov | +----------------------------------------------------------------------+ */ From 9ae93844b96c0f90834e85f8ca440ba2476692fa Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 6 Feb 2019 15:52:58 +0300 Subject: [PATCH 537/569] Reverted unintentional change. --- ext/opcache/jit/zend_jit.c | 1 - 1 file changed, 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index e743085166b17..2bebf5770922a 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -477,7 +477,6 @@ static int zend_may_overflow(const zend_op *opline, zend_op_array *op_array, zen uint32_t num; int res; - return 1; if (!ssa->ops || !ssa->var_info) { return 1; } From df2f9cee67be694487d4d09618a5769005f2c89c Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 6 Feb 2019 16:32:17 +0300 Subject: [PATCH 538/569] typo --- ext/opcache/jit/zend_jit_x86.dasc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index bb0617dba718d..8bbd1ab64178e 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -795,7 +795,7 @@ static void* dasm_labels[zend_lb_MAX]; || if (Z_TYPE_P(zv) > IS_TRUE) { || if (Z_TYPE_P(zv) == IS_DOUBLE) { || zend_reg dst_reg = (Z_MODE(dst_addr) == IS_REG) ? -|| Z_REG(dst_addr) : ((Z_MODE(dst_addr) == IS_REG) ? Z_MODE(res_addr) : ZREG_XMM0); +|| Z_REG(dst_addr) : ((Z_MODE(res_addr) == IS_REG) ? Z_MODE(res_addr) : ZREG_XMM0); | .if X64 or SSE || if (Z_DVAL_P(zv) == 0.0 && !is_signed(Z_DVAL_P(zv))) { || if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { From f3fd52db7792cbe83f9e73e8000b4e03e376a5e4 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 6 Feb 2019 16:52:43 +0300 Subject: [PATCH 539/569] typo --- ext/opcache/jit/zend_jit_x86.dasc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 8bbd1ab64178e..027afc540cc60 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -1148,7 +1148,7 @@ static void* dasm_labels[zend_lb_MAX]; | add dword [Z_LVAL_P(zv)], 2 || } | .else -| add dword [Z_LVAL_P(zv)], 1 +| add dword [Z_LVAL_P(zv)], 2 | .endif |.endmacro From 5caca8da0dfb52581f4c50cc57fe1a4aa71483ef Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 6 Feb 2019 17:18:31 +0300 Subject: [PATCH 540/569] We no longer allow persistent strings in user zvals --- ext/opcache/jit/zend_jit_x86.dasc | 6 ------ 1 file changed, 6 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 027afc540cc60..fc30d540438b7 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -1195,13 +1195,7 @@ static void* dasm_labels[zend_lb_MAX]; || if (has_concrete_type((var_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { || zend_uchar type = concrete_type((var_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)); || if (type == IS_STRING && !ZEND_DEBUG) { -| test byte [FCARG1a + 5], IS_STR_PERSISTENT -| jnz >1 | EXT_CALL _efree, r0 -| jmp >2 -|1: -| EXT_CALL free, r0 -|2: || break; || } else if (type == IS_ARRAY) { || if (opline) { From 277fce745de1e898bf2ba1d3dc3b78d42c45b987 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 7 Feb 2019 10:31:37 +0300 Subject: [PATCH 541/569] Separation is handled by operation functions (e.g. add_function) since PHP-7.4. --- ext/opcache/jit/zend_jit_x86.dasc | 24 ++++++++---------------- 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index fc30d540438b7..59e722724700c 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -3033,8 +3033,7 @@ static int zend_jit_math_helper(dasm_State **Dst, uint32_t res_var, zend_jit_addr res_addr, uint32_t res_info, - uint32_t res_use_info, - zend_bool separate_op1) + uint32_t res_use_info) /* Labels: 1,2,3,4,5,6 */ { zend_bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); @@ -3178,9 +3177,6 @@ static int zend_jit_math_helper(dasm_State **Dst, } |6: | SAVE_VALID_OPLINE opline - if (separate_op1) { - | SEPARATE_ZVAL_NOREF op1_addr, op1_info, 0 - } if (Z_MODE(res_addr) == IS_REG) { zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); | LOAD_ZVAL_ADDR FCARG1a, real_addr @@ -3285,7 +3281,7 @@ static int zend_jit_math(dasm_State **Dst, const zend_op *opline, int *opnum, ze res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, ra, ssa->ops[opline - op_array->opcodes].result_def); } - if (!zend_jit_math_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, opline->result.var, res_addr, RES_INFO(), res_use_info, 0)) { + if (!zend_jit_math_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, opline->result.var, res_addr, RES_INFO(), res_use_info)) { return 0; } if (ra && !zend_jit_store_ssa_var_if_necessary(Dst, ssa, ra, res_addr, ssa->ops[opline - op_array->opcodes].result_def, ssa->ops[opline - op_array->opcodes].result_use)) { @@ -3315,8 +3311,7 @@ static int zend_jit_long_math_helper(dasm_State **Dst, uint32_t res_var, zend_jit_addr res_addr, uint32_t res_info, - uint32_t res_use_info, - zend_bool separate_op1) + uint32_t res_use_info) /* Labels: 6 */ { zend_bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); @@ -3524,9 +3519,6 @@ static int zend_jit_long_math_helper(dasm_State **Dst, } |6: | SAVE_VALID_OPLINE opline - if (separate_op1) { - | SEPARATE_ZVAL_NOREF op1_addr, op1_info, 0 - } if (Z_MODE(res_addr) == IS_REG) { zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); | LOAD_ZVAL_ADDR FCARG1a, real_addr @@ -3638,7 +3630,7 @@ static int zend_jit_long_math(dasm_State **Dst, const zend_op *opline, int *opnu res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, ra, ssa->ops[opline - op_array->opcodes].result_def); } - if (!zend_jit_long_math_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, op1_addr, op1_info, op1_ssa_var, opline->op2_type, opline->op2, op2_addr, op2_info, op2_ssa_var, opline->result.var, res_addr, RES_INFO(), res_use_info, 0)) { + if (!zend_jit_long_math_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, op1_addr, op1_info, op1_ssa_var, opline->op2_type, opline->op2, op2_addr, op2_info, op2_ssa_var, opline->result.var, res_addr, RES_INFO(), res_use_info)) { return 0; } if (ra && !zend_jit_store_ssa_var_if_necessary(Dst, ssa, ra, res_addr, ssa->ops[opline - op_array->opcodes].result_def, ssa->ops[opline - op_array->opcodes].result_use)) { @@ -4679,7 +4671,7 @@ static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, zend_ case ZEND_ASSIGN_SUB: case ZEND_ASSIGN_MUL: case ZEND_ASSIGN_DIV: - if (!zend_jit_math_helper(Dst, opline, op_array, ssa, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, op3_addr, OP1_DATA_INFO(), 0, var_addr, var_def_info, var_info, 1)) { + if (!zend_jit_math_helper(Dst, opline, op_array, ssa, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, op3_addr, OP1_DATA_INFO(), 0, var_addr, var_def_info, var_info)) { return 0; } break; @@ -4689,7 +4681,7 @@ static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, zend_ case ZEND_ASSIGN_SL: case ZEND_ASSIGN_SR: case ZEND_ASSIGN_MOD: - if (!zend_jit_long_math_helper(Dst, opline, op_array, ssa, IS_CV, opline->op1, var_addr, var_info, -1, (opline+1)->op1_type, (opline+1)->op1, op3_addr, OP1_DATA_INFO(), op3_ssa_var, 0, var_addr, OP1_DEF_INFO(), var_info, 1)) { + if (!zend_jit_long_math_helper(Dst, opline, op_array, ssa, IS_CV, opline->op1, var_addr, var_info, -1, (opline+1)->op1_type, (opline+1)->op1, op3_addr, OP1_DATA_INFO(), op3_ssa_var, 0, var_addr, OP1_DEF_INFO(), var_info)) { return 0; } break; @@ -4893,14 +4885,14 @@ static int zend_jit_assign_op(dasm_State **Dst, const zend_op *opline, zend_op_a case ZEND_ASSIGN_SUB: case ZEND_ASSIGN_MUL: case ZEND_ASSIGN_DIV: - return zend_jit_math_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, opline->op1.var, op1_addr, OP1_DEF_INFO(), op1_info, 1); + return zend_jit_math_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, opline->op1.var, op1_addr, OP1_DEF_INFO(), op1_info); case ZEND_ASSIGN_BW_OR: case ZEND_ASSIGN_BW_AND: case ZEND_ASSIGN_BW_XOR: case ZEND_ASSIGN_SL: case ZEND_ASSIGN_SR: case ZEND_ASSIGN_MOD: - return zend_jit_long_math_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, op1_addr, op1_info, op1_ssa_var, opline->op2_type, opline->op2, op2_addr, op2_info, op2_ssa_var, opline->op1.var, op1_addr, OP1_DEF_INFO(), op1_info, 1); + return zend_jit_long_math_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, op1_addr, op1_info, op1_ssa_var, opline->op2_type, opline->op2, op2_addr, op2_info, op2_ssa_var, opline->op1.var, op1_addr, OP1_DEF_INFO(), op1_info); case ZEND_ASSIGN_CONCAT: return zend_jit_concat_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, op1_addr, OP1_DEF_INFO()); default: From 821c76c4b29455d9bf48d0bb31c27504389f10e3 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 7 Feb 2019 14:55:12 +0300 Subject: [PATCH 542/569] Merge removing --disable-opcache-filecache option --- acinclude.m4 | 2 +- ext/opcache/ZendAccelerator.c | 50 ++------------------------- ext/opcache/ZendAccelerator.h | 6 +--- ext/opcache/config.m4 | 7 ---- ext/opcache/config.w32 | 6 ---- ext/opcache/zend_accelerator_module.c | 16 --------- ext/opcache/zend_file_cache.c | 4 --- ext/opcache/zend_persist.c | 6 ---- 8 files changed, 4 insertions(+), 93 deletions(-) diff --git a/acinclude.m4 b/acinclude.m4 index 21f44980f634d..aa5b04cec0e1c 100644 --- a/acinclude.m4 +++ b/acinclude.m4 @@ -2119,7 +2119,7 @@ dnl dnl Common setup macro for ICU dnl AC_DEFUN([PHP_SETUP_ICU],[ - PKG_CHECK_MODULES([ICU], [icu-io >= 50.1]) + PKG_CHECK_MODULES([ICU], [icu-uc >= 50.1 icu-io icu-i18n]) PHP_EVAL_INCLINE($ICU_CFLAGS) PHP_EVAL_LIBLINE($ICU_LIBS, $1) diff --git a/ext/opcache/ZendAccelerator.c b/ext/opcache/ZendAccelerator.c index c0b3cded8d5b6..41f5658440585 100644 --- a/ext/opcache/ZendAccelerator.c +++ b/ext/opcache/ZendAccelerator.c @@ -41,13 +41,10 @@ #include "zend_virtual_cwd.h" #include "zend_accelerator_util_funcs.h" #include "zend_accelerator_hash.h" +#include "zend_file_cache.h" #include "ext/pcre/php_pcre.h" #include "ext/standard/md5.h" -#ifdef HAVE_OPCACHE_FILE_CACHE -# include "zend_file_cache.h" -#endif - #ifdef HAVE_JIT # include "jit/zend_jit.h" #endif @@ -118,9 +115,7 @@ zend_accel_shared_globals *accel_shared_globals = NULL; zend_bool accel_startup_ok = 0; static char *zps_failure_reason = NULL; char *zps_api_failure_reason = NULL; -#ifdef HAVE_OPCACHE_FILE_CACHE zend_bool file_cache_only = 0; /* process uses file cache only */ -#endif #if ENABLE_FILE_CACHE_FALLBACK zend_bool fallback_process = 0; /* process uses file cache fallback */ #endif @@ -461,11 +456,9 @@ zend_string* ZEND_FASTCALL accel_new_interned_string(zend_string *str) uint32_t pos, *hash_slot; zend_string *s; -#ifdef HAVE_OPCACHE_FILE_CACHE if (UNEXPECTED(file_cache_only)) { return str; } -#endif if (IS_ACCEL_INTERNED(str)) { /* this is already an interned string */ @@ -1282,11 +1275,9 @@ int zend_accel_invalidate(const char *filename, size_t filename_len, zend_bool f return FAILURE; } -#ifdef HAVE_OPCACHE_FILE_CACHE if (ZCG(accel_directives).file_cache) { zend_file_cache_invalidate(realpath); } -#endif persistent_script = zend_accel_hash_find(&ZCSG(hash), realpath); if (persistent_script && !persistent_script->corrupted) { @@ -1353,7 +1344,6 @@ static zend_always_inline zend_bool is_phar_file(zend_string *filename) !strstr(ZSTR_VAL(filename), "://"); } -#ifdef HAVE_OPCACHE_FILE_CACHE static zend_persistent_script *store_script_in_file_cache(zend_persistent_script *new_persistent_script) { uint32_t memory_used; @@ -1419,7 +1409,6 @@ static zend_persistent_script *cache_script_in_file_cache(zend_persistent_script *from_shared_memory = 1; return store_script_in_file_cache(new_persistent_script); } -#endif static zend_persistent_script *cache_script_in_shared_memory(zend_persistent_script *new_persistent_script, const char *key, unsigned int key_length, int *from_shared_memory) { @@ -1433,11 +1422,9 @@ static zend_persistent_script *cache_script_in_shared_memory(zend_persistent_scr } orig_compiler_options = CG(compiler_options); -#ifdef HAVE_OPCACHE_FILE_CACHE if (ZCG(accel_directives).file_cache) { CG(compiler_options) |= ZEND_COMPILE_WITH_FILE_CACHE; } -#endif if (!zend_optimize_script(&new_persistent_script->script, ZCG(accel_directives).optimization_level, ZCG(accel_directives).opt_debug_level)) { CG(compiler_options) = orig_compiler_options; return new_persistent_script; @@ -1470,12 +1457,10 @@ static zend_persistent_script *cache_script_in_shared_memory(zend_persistent_scr ZSMMG(memory_exhausted) = 1; zend_accel_schedule_restart_if_necessary(ACCEL_RESTART_HASH); zend_shared_alloc_unlock(); -#ifdef HAVE_OPCACHE_FILE_CACHE if (ZCG(accel_directives).file_cache) { new_persistent_script = store_script_in_file_cache(new_persistent_script); *from_shared_memory = 1; } -#endif return new_persistent_script; } @@ -1530,12 +1515,10 @@ static zend_persistent_script *cache_script_in_shared_memory(zend_persistent_scr zend_shared_alloc_destroy_xlat_table(); zend_accel_schedule_restart_if_necessary(ACCEL_RESTART_OOM); zend_shared_alloc_unlock(); -#ifdef HAVE_OPCACHE_FILE_CACHE if (ZCG(accel_directives).file_cache) { new_persistent_script = store_script_in_file_cache(new_persistent_script); *from_shared_memory = 1; } -#endif return new_persistent_script; } @@ -1585,13 +1568,11 @@ static zend_persistent_script *cache_script_in_shared_memory(zend_persistent_scr zend_shared_alloc_unlock(); -#ifdef HAVE_OPCACHE_FILE_CACHE if (ZCG(accel_directives).file_cache) { SHM_PROTECT(); zend_file_cache_script_store(new_persistent_script, 1); SHM_UNPROTECT(); } -#endif *from_shared_memory = 1; return new_persistent_script; @@ -1738,11 +1719,9 @@ static zend_persistent_script *opcache_compile_file(zend_file_handle *file_handl CG(compiler_options) |= ZEND_COMPILE_DELAYED_BINDING; CG(compiler_options) |= ZEND_COMPILE_NO_CONSTANT_SUBSTITUTION; CG(compiler_options) |= ZEND_COMPILE_IGNORE_OTHER_FILES; -#ifdef HAVE_OPCACHE_FILE_CACHE if (ZCG(accel_directives).file_cache) { CG(compiler_options) |= ZEND_COMPILE_WITH_FILE_CACHE; } -#endif op_array = *op_array_p = accelerator_orig_compile_file(file_handle, type); CG(compiler_options) = orig_compiler_options; } zend_catch { @@ -1805,7 +1784,6 @@ static zend_persistent_script *opcache_compile_file(zend_file_handle *file_handl return new_persistent_script; } -#ifdef HAVE_OPCACHE_FILE_CACHE zend_op_array *file_cache_compile_file(zend_file_handle *file_handle, int type) { zend_persistent_script *persistent_script; @@ -1877,7 +1855,6 @@ zend_op_array *file_cache_compile_file(zend_file_handle *file_handle, int type) return op_array; } -#endif /* zend_compile() replacement */ zend_op_array *persistent_compile_file(zend_file_handle *file_handle, int type) @@ -1890,17 +1867,13 @@ zend_op_array *persistent_compile_file(zend_file_handle *file_handle, int type) if (!file_handle->filename || !ZCG(enabled) || !accel_startup_ok) { /* The Accelerator is disabled, act as if without the Accelerator */ return accelerator_orig_compile_file(file_handle, type); -#ifdef HAVE_OPCACHE_FILE_CACHE } else if (file_cache_only) { return file_cache_compile_file(file_handle, type); -#endif } else if ((!ZCG(counted) && !ZCSG(accelerator_enabled)) || (ZCSG(restart_in_progress) && accel_restart_is_active())) { -#ifdef HAVE_OPCACHE_FILE_CACHE if (ZCG(accel_directives).file_cache) { return file_cache_compile_file(file_handle, type); } -#endif return accelerator_orig_compile_file(file_handle, type); } @@ -1984,11 +1957,9 @@ zend_op_array *persistent_compile_file(zend_file_handle *file_handle, int type) */ if (!ZCG(counted)) { if (accel_activate_add() == FAILURE) { -#ifdef HAVE_OPCACHE_FILE_CACHE if (ZCG(accel_directives).file_cache) { return file_cache_compile_file(file_handle, type); } -#endif return accelerator_orig_compile_file(file_handle, type); } ZCG(counted) = 1; @@ -2055,12 +2026,10 @@ zend_op_array *persistent_compile_file(zend_file_handle *file_handle, int type) } } -#ifdef HAVE_OPCACHE_FILE_CACHE /* Check the second level cache */ if (!persistent_script && ZCG(accel_directives).file_cache) { persistent_script = zend_file_cache_script_load(file_handle); } -#endif /* If script was not found or invalidated by validate_timestamps */ if (!persistent_script) { @@ -2074,11 +2043,9 @@ zend_op_array *persistent_compile_file(zend_file_handle *file_handle, int type) if (ZSMMG(memory_exhausted) || ZCSG(restart_pending)) { SHM_PROTECT(); HANDLE_UNBLOCK_INTERRUPTIONS(); -#ifdef HAVE_OPCACHE_FILE_CACHE if (ZCG(accel_directives).file_cache) { return file_cache_compile_file(file_handle, type); } -#endif return accelerator_orig_compile_file(file_handle, type); } @@ -2190,9 +2157,7 @@ static int persistent_stream_open_function(const char *filename, zend_file_handl static zend_string* persistent_zend_resolve_path(const char *filename, size_t filename_len) { if (ZCG(enabled) && accel_startup_ok && -#ifdef HAVE_OPCACHE_FILE_CACHE !file_cache_only && -#endif (ZCG(counted) || ZCSG(accelerator_enabled)) && !ZCSG(restart_in_progress)) { @@ -2314,11 +2279,9 @@ int accel_activate(INIT_FUNC_ARGS) ZCG(cwd_key_len) = 0; ZCG(cwd_check) = 1; -#ifdef HAVE_OPCACHE_FILE_CACHE if (file_cache_only) { return SUCCESS; } -#endif #ifndef ZEND_WIN32 if (ZCG(accel_directives).validate_root) { @@ -2850,13 +2813,8 @@ static int accel_post_startup(void) /********************************************/ /* End of non-SHM dependent initializations */ /********************************************/ -#ifdef HAVE_OPCACHE_FILE_CACHE file_cache_only = ZCG(accel_directives).file_cache_only; - if (!file_cache_only) -#else - if (1) -#endif - { + if (!file_cache_only) { switch (zend_shared_alloc_startup(ZCG(accel_directives).memory_consumption)) { case ALLOC_SUCCESS: if (zend_accel_init_shm() == FAILURE) { @@ -2913,7 +2871,6 @@ static int accel_post_startup(void) zend_shared_alloc_unlock(); SHM_PROTECT(); -#ifdef HAVE_OPCACHE_FILE_CACHE } else if (!ZCG(accel_directives).file_cache) { accel_startup_ok = 0; zend_accel_error(ACCEL_LOG_FATAL, "opcache.file_cache_only is set without a proper setting of opcache.file_cache"); @@ -2923,7 +2880,6 @@ static int accel_post_startup(void) /* Init auto-global strings */ zend_accel_init_auto_globals(); -#endif } #if ENABLE_FILE_CACHE_FALLBACK file_cache_fallback: @@ -3013,9 +2969,7 @@ void accel_shutdown(void) preload_shutdown(); } -#ifdef HAVE_OPCACHE_FILE_CACHE _file_cache_only = file_cache_only; -#endif accel_reset_pcre_cache(); diff --git a/ext/opcache/ZendAccelerator.h b/ext/opcache/ZendAccelerator.h index e72734f6905af..3377a9856a039 100644 --- a/ext/opcache/ZendAccelerator.h +++ b/ext/opcache/ZendAccelerator.h @@ -104,7 +104,7 @@ extern int lock_file; # endif #endif -#if defined(HAVE_OPCACHE_FILE_CACHE) && defined(ZEND_WIN32) +#if defined(ZEND_WIN32) # define ENABLE_FILE_CACHE_FALLBACK 1 #else # define ENABLE_FILE_CACHE_FALLBACK 0 @@ -188,11 +188,9 @@ typedef struct _zend_accel_directives { #ifndef ZEND_WIN32 char *lockfile_path; #endif -#ifdef HAVE_OPCACHE_FILE_CACHE char *file_cache; zend_bool file_cache_only; zend_bool file_cache_consistency_checks; -#endif #if ENABLE_FILE_CACHE_FALLBACK zend_bool file_cache_fallback; #endif @@ -289,9 +287,7 @@ typedef struct _zend_accel_shared_globals { } zend_accel_shared_globals; extern zend_bool accel_startup_ok; -#ifdef HAVE_OPCACHE_FILE_CACHE extern zend_bool file_cache_only; -#endif #if ENABLE_FILE_CACHE_FALLBACK extern zend_bool fallback_process; #endif diff --git a/ext/opcache/config.m4 b/ext/opcache/config.m4 index 0261fbaa7cca7..5b1c0e745f484 100644 --- a/ext/opcache/config.m4 +++ b/ext/opcache/config.m4 @@ -3,9 +3,6 @@ dnl config.m4 for extension opcache PHP_ARG_ENABLE(opcache, whether to enable Zend OPcache support, [ --disable-opcache Disable Zend OPcache support], yes) -PHP_ARG_ENABLE(opcache-file, whether to enable file based caching, -[ --disable-opcache-file Disable file based caching], yes, no) - PHP_ARG_ENABLE(huge-code-pages, whether to enable copying PHP CODE pages into HUGE PAGES, [ --disable-huge-code-pages Disable copying PHP CODE pages into HUGE PAGES], yes, no) @@ -18,10 +15,6 @@ if test "$PHP_OPCACHE" != "no"; then dnl Always build as shared extension ext_shared=yes - if test "$PHP_OPCACHE_FILE" = "yes"; then - AC_DEFINE(HAVE_OPCACHE_FILE_CACHE, 1, [Define to enable file based caching (experimental)]) - fi - if test "$PHP_HUGE_CODE_PAGES" = "yes"; then AC_DEFINE(HAVE_HUGE_CODE_PAGES, 1, [Define to enable copying PHP CODE pages into HUGE PAGES (experimental)]) fi diff --git a/ext/opcache/config.w32 b/ext/opcache/config.w32 index 1d78f7e548732..fba1b1bef1be9 100644 --- a/ext/opcache/config.w32 +++ b/ext/opcache/config.w32 @@ -1,15 +1,9 @@ ARG_ENABLE("opcache", "whether to enable Zend OPcache support", "yes"); -ARG_ENABLE("opcache-file", "whether to enable file based caching", "yes"); - /* var PHP_OPCACHE_PGO = false; */ if (PHP_OPCACHE != "no") { - if (PHP_OPCACHE_FILE == "yes") { - AC_DEFINE('HAVE_OPCACHE_FILE_CACHE', 1, 'Define to enable file based caching (experimental)'); - } - ZEND_EXTENSION('opcache', "\ ZendAccelerator.c \ zend_accelerator_blacklist.c \ diff --git a/ext/opcache/zend_accelerator_module.c b/ext/opcache/zend_accelerator_module.c index 0033223115f10..e3199432cb087 100644 --- a/ext/opcache/zend_accelerator_module.c +++ b/ext/opcache/zend_accelerator_module.c @@ -249,7 +249,6 @@ static ZEND_INI_MH(OnEnable) } } -#ifdef HAVE_OPCACHE_FILE_CACHE static ZEND_INI_MH(OnUpdateFileCache) { if (new_value) { @@ -274,7 +273,6 @@ static ZEND_INI_MH(OnUpdateFileCache) OnUpdateString(entry, new_value, mh_arg1, mh_arg2, mh_arg3, stage); return SUCCESS; } -#endif ZEND_INI_BEGIN() STD_PHP_INI_BOOLEAN("opcache.enable" , "1", PHP_INI_ALL, OnEnable, enabled , zend_accel_globals, accel_globals) @@ -316,11 +314,9 @@ ZEND_INI_BEGIN() STD_PHP_INI_ENTRY("opcache.mmap_base", NULL, PHP_INI_SYSTEM, OnUpdateString, accel_directives.mmap_base, zend_accel_globals, accel_globals) #endif -#ifdef HAVE_OPCACHE_FILE_CACHE STD_PHP_INI_ENTRY("opcache.file_cache" , NULL , PHP_INI_SYSTEM, OnUpdateFileCache, accel_directives.file_cache, zend_accel_globals, accel_globals) STD_PHP_INI_ENTRY("opcache.file_cache_only" , "0" , PHP_INI_SYSTEM, OnUpdateBool, accel_directives.file_cache_only, zend_accel_globals, accel_globals) STD_PHP_INI_ENTRY("opcache.file_cache_consistency_checks" , "1" , PHP_INI_SYSTEM, OnUpdateBool, accel_directives.file_cache_consistency_checks, zend_accel_globals, accel_globals) -#endif #if ENABLE_FILE_CACHE_FALLBACK STD_PHP_INI_ENTRY("opcache.file_cache_fallback" , "1" , PHP_INI_SYSTEM, OnUpdateBool, accel_directives.file_cache_fallback, zend_accel_globals, accel_globals) #endif @@ -413,12 +409,10 @@ void zend_accel_override_file_functions(void) { zend_function *old_function; if (ZCG(enabled) && accel_startup_ok && ZCG(accel_directives).file_override_enabled) { -#ifdef HAVE_OPCACHE_FILE_CACHE if (file_cache_only) { zend_accel_error(ACCEL_LOG_WARNING, "file_override_enabled has no effect when file_cache_only is set"); return; } -#endif /* override file_exists */ if ((old_function = zend_hash_str_find_ptr(CG(function_table), "file_exists", sizeof("file_exists")-1)) != NULL) { orig_file_exists = old_function->internal_function.handler; @@ -449,11 +443,7 @@ void zend_accel_info(ZEND_MODULE_INFO_FUNC_ARGS) php_info_print_table_start(); if (ZCG(enabled) && accel_startup_ok && -#ifdef HAVE_OPCACHE_FILE_CACHE ((ZCG(counted) || ZCSG(accelerator_enabled)) || file_cache_only) -#else - (ZCG(counted) || ZCSG(accelerator_enabled)) -#endif ) { php_info_print_table_row(2, "Opcode Caching", "Up and Running"); } else { @@ -464,7 +454,6 @@ void zend_accel_info(ZEND_MODULE_INFO_FUNC_ARGS) } else { php_info_print_table_row(2, "Optimization", "Disabled"); } -#ifdef HAVE_OPCACHE_FILE_CACHE if (!file_cache_only) { php_info_print_table_row(2, "SHM Cache", "Enabled"); } else { @@ -482,7 +471,6 @@ void zend_accel_info(ZEND_MODULE_INFO_FUNC_ARGS) php_info_print_table_row(2, "Startup", "OK"); } } else -#endif if (ZCG(enabled)) { if (!accel_startup_ok || zps_api_failure_reason) { php_info_print_table_row(2, "Startup Failed", zps_api_failure_reason); @@ -620,7 +608,6 @@ static ZEND_FUNCTION(opcache_get_status) /* Trivia */ add_assoc_bool(return_value, "opcache_enabled", ZCG(enabled) && (ZCG(counted) || ZCSG(accelerator_enabled))); -#ifdef HAVE_OPCACHE_FILE_CACHE if (ZCG(accel_directives).file_cache) { add_assoc_string(return_value, "file_cache", ZCG(accel_directives).file_cache); } @@ -628,7 +615,6 @@ static ZEND_FUNCTION(opcache_get_status) add_assoc_bool(return_value, "file_cache_only", 1); return; } -#endif add_assoc_bool(return_value, "cache_full", ZSMMG(memory_exhausted)); add_assoc_bool(return_value, "restart_pending", ZCSG(restart_pending)); @@ -782,11 +768,9 @@ static ZEND_FUNCTION(opcache_get_configuration) add_assoc_string(&directives, "opcache.lockfile_path", STRING_NOT_NULL(ZCG(accel_directives).lockfile_path)); #endif -#ifdef HAVE_OPCACHE_FILE_CACHE add_assoc_string(&directives, "opcache.file_cache", ZCG(accel_directives).file_cache ? ZCG(accel_directives).file_cache : ""); add_assoc_bool(&directives, "opcache.file_cache_only", ZCG(accel_directives).file_cache_only); add_assoc_bool(&directives, "opcache.file_cache_consistency_checks", ZCG(accel_directives).file_cache_consistency_checks); -#endif add_assoc_zval(return_value, "directives", &directives); diff --git a/ext/opcache/zend_file_cache.c b/ext/opcache/zend_file_cache.c index df725fc3b1de2..09b029ad9a31d 100644 --- a/ext/opcache/zend_file_cache.c +++ b/ext/opcache/zend_file_cache.c @@ -27,8 +27,6 @@ #include "ext/standard/md5.h" #endif -#ifdef HAVE_OPCACHE_FILE_CACHE - #include "ZendAccelerator.h" #include "zend_file_cache.h" #include "zend_shared_alloc.h" @@ -1696,5 +1694,3 @@ void zend_file_cache_invalidate(zend_string *full_path) zend_file_cache_unlink(filename); efree(filename); } - -#endif /* HAVE_OPCACHE_FILE_CACHE */ diff --git a/ext/opcache/zend_persist.c b/ext/opcache/zend_persist.c index 0cd7a6d2ffe60..c7d6e6ef14525 100644 --- a/ext/opcache/zend_persist.c +++ b/ext/opcache/zend_persist.c @@ -33,7 +33,6 @@ # include "jit/zend_jit.h" #endif -#ifdef HAVE_OPCACHE_FILE_CACHE #define zend_set_str_gc_flags(str) do { \ if (file_cache_only) { \ GC_TYPE_INFO(str) = IS_STRING | (IS_STR_INTERNED << GC_FLAGS_SHIFT); \ @@ -41,11 +40,6 @@ GC_TYPE_INFO(str) = IS_STRING | ((IS_STR_INTERNED | IS_STR_PERMANENT) << GC_FLAGS_SHIFT); \ } \ } while (0) -#else -#define zend_set_str_gc_flags(str) do {\ - GC_TYPE_INFO(str) = IS_STRING | ((IS_STR_INTERNED | IS_STR_PERMANENT) << GC_FLAGS_SHIFT); \ -} while (0) -#endif #define zend_accel_store_string(str) do { \ zend_string *new_str = zend_shared_alloc_get_xlat_entry(str); \ From 9274dcbf840abb9653564ed607a005a47d4042c3 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 7 Feb 2019 21:26:41 +0300 Subject: [PATCH 543/569] Removed JIT support for overloaded functions --- ext/opcache/jit/zend_jit_x86.dasc | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 59e722724700c..beedade6aa2c0 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -7177,26 +7177,6 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar if (!func || func->type == ZEND_INTERNAL_FUNCTION) { if (!func) { |8: - if (opline->opcode == ZEND_DO_FCALL) { - | jg >8 - |.cold_code - |8: - if (!RETURN_VALUE_USED(opline)) { - | sub r4, 16 /* alloca() */ - } - | // ZVAL_NULL(EX_VAR(opline->result.var)); - | LOAD_ZVAL_ADDR FCARG2a, res_addr - | SET_Z_TYPE_INFO FCARG2a, IS_NULL - | mov FCARG1a, RX - | EXT_CALL zend_do_fcall_overloaded, r0 - | test r0, r0 - | jne >8 - if (!RETURN_VALUE_USED(opline)) { - | add r4, 16 /* revert alloca() */ - } - | jmp ->exception_handler - |.code - } } if (opline->opcode == ZEND_DO_FCALL_BY_NAME) { if (!func) { From 35b5c0d11d75ecdff8ee01479d5bf5652ed32b4b Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 8 Feb 2019 16:15:18 +0300 Subject: [PATCH 544/569] Fixed resolution of stub names in disassemble output --- ext/opcache/jit/zend_jit.c | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 2bebf5770922a..69202171d16c0 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -309,15 +309,17 @@ static void *dasm_link_and_encode(dasm_State **dasm_state, size); } } else { - if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_ASM_STUBS) { + if (ZCG(accel_directives).jit_debug & (ZEND_JIT_DEBUG_ASM_STUBS|ZEND_JIT_DEBUG_ASM)) { zend_jit_disasm_add_symbol(name, (uintptr_t)entry, size); - zend_jit_disasm( - name, - (op_array && op_array->filename) ? ZSTR_VAL(op_array->filename) : NULL, - op_array, - &ssa->cfg, - entry, - size); + if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_ASM_STUBS) { + zend_jit_disasm( + name, + (op_array && op_array->filename) ? ZSTR_VAL(op_array->filename) : NULL, + op_array, + &ssa->cfg, + entry, + size); + } } # endif } From 755ca65858939a2f66bc63d600f7dd6e6dcc7454 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 13 Feb 2019 14:20:46 +0300 Subject: [PATCH 545/569] ws --- ext/opcache/jit/zend_jit_x86.dasc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index beedade6aa2c0..cec85f05b853f 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -1711,7 +1711,7 @@ static int zend_jit_undefined_offset_stub(dasm_State **Dst) | jne >2 |.if X64 | movsxd r1, dword OP:r0->op2.constant - | add r0, r1 + | add r0, r1 |.else | mov r0, aword OP:r0->op2.zv |.endif From b895271852ffb32c6c054b2371d89fb4c4902ba8 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 13 Feb 2019 14:27:22 +0300 Subject: [PATCH 546/569] Added asserts --- ext/opcache/jit/zend_jit_x86.dasc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index cec85f05b853f..a519fd5733ea2 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -1989,8 +1989,10 @@ static int zend_jit_hybrid_func_counter_stub(dasm_State **Dst) | sub r0, aword [r1 + offsetof(zend_op_array, opcodes)] | // divide by sizeof(zend_op) | .if X64 + || ZEND_ASSERT(sizeof(zend_op) == 32); | sar r0, 5 | .else + || ZEND_ASSERT(sizeof(zend_op) == 28); | sar r0, 2 | imul r0, 0xb6db6db7 | .endif @@ -2065,8 +2067,10 @@ static int zend_jit_hybrid_loop_counter_stub(dasm_State **Dst) | sub r0, aword [r1 + offsetof(zend_op_array, opcodes)] | // divide by sizeof(zend_op) | .if X64 + || ZEND_ASSERT(sizeof(zend_op) == 32); | sar r0, 5 | .else + || ZEND_ASSERT(sizeof(zend_op) == 28); | sar r0, 2 | imul r0, 0xb6db6db7 | .endif From 99a3752efdf8778595808ef1344f1c5dc3e31b63 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 13 Feb 2019 14:49:34 +0300 Subject: [PATCH 547/569] Added comments --- ext/opcache/jit/zend_jit_x86.dasc | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index a519fd5733ea2..cf82245199589 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -1942,6 +1942,30 @@ static int zend_jit_hybrid_profile_jit_stub(dasm_State **Dst) return 1; } +/* + * This code is based Mike Pall's "Hashed profile counters" idea, implemented + * in LuaJIT. The full description may be found in "LuaJIT 2.0 intellectual + * property disclosure and research opportunities" email + * at http://lua-users.org/lists/lua-l/2009-11/msg00089.html + * + * In addition we use a variation of Knuth's multiplicative hash function + * described at https://code.i-harness.com/en/q/a21ce + * + * uint64_t hash(uint64_t x) { + * x = (x ^ (x >> 30)) * 0xbf58476d1ce4e5b9; + * x = (x ^ (x >> 27)) * 0x94d049bb133111eb; + * x = x ^ (x >> 31); + * return x; + * } + * + * uint_32_t hash(uint32_t x) { + * x = ((x >> 16) ^ x) * 0x45d9f3b; + * x = ((x >> 16) ^ x) * 0x45d9f3b; + * x = (x >> 16) ^ x; + * return x; + * } + * + */ static int zend_jit_hybrid_func_counter_stub(dasm_State **Dst) { |->hybrid_func_counter: From 1c8fb1e76e0e25fafd141bf54bf5ff7c33e859c3 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 13 Feb 2019 15:00:14 +0300 Subject: [PATCH 548/569] Added comments about CPU stack alignment --- ext/opcache/jit/zend_jit_x86.dasc | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index cf82245199589..f42e92c20aa4f 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -41,11 +41,11 @@ |.define CARG4d, ecx |.define CARG5d, r8d |.define CARG6d, r9d - |.define FCARG1a, CARG1 // Simulate x86 fastcall. + |.define FCARG1a, CARG1 // Simulate x86 fastcall. |.define FCARG2a, CARG2 |.define FCARG1d, CARG1d |.define FCARG2d, CARG2d - |.define SPAD, 8 + |.define SPAD, 8 // padding for CPU stack alignment |.define SSE, 1 |.else |.define FP, esi @@ -57,11 +57,17 @@ |.define FCARG2a, edx |.define FCARG1d, ecx |.define FCARG2d, edx - |.define SPAD, 12 + |.define SPAD, 12 // padding for CPU stack alignment |.define SSE, 1 |.endif -|.define HYBRID_SPAD, 16 +|.define HYBRID_SPAD, 16 // padding for stack alignment + +/* According to x86 and x86_64 ABI, CPU stack has to be 16 byte aligned to + * guarantee proper alignment of 128-bit SSE data allocated on stack. + * With broken alignment any execution of SSE code, including calls to + * memcpy() and others, may lead to crash. + */ #include "Zend/zend_cpuinfo.h" #include "jit/zend_jit_x86.h" From 4d2bd073d23fc6549a31d37bf6c74c97e91e76b4 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 13 Feb 2019 15:20:11 +0300 Subject: [PATCH 549/569] Added comments. --- ext/opcache/jit/zend_jit_x86.dasc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index f42e92c20aa4f..72298235beea5 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -2377,6 +2377,11 @@ static int zend_jit_jmp(dasm_State **Dst, unsigned int target_label) static int zend_jit_cond_jmp(dasm_State **Dst, const zend_op *next_opline, unsigned int target_label) { +/* In 64-bit build we compare only low 32-bits. + * x86_64 cmp instruction doesn't support immediate 64-bit operand, and full + * comparison would require additinal load of 64-bit address into register. + * This is not a problem at all, while JIT buffer size is less than 4GB. + */ | cmp IPl, next_opline | jne =>target_label From 77e42ca6cc0e30ac0ac3c676e80052881188cb6f Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 19 Feb 2019 11:58:31 +0300 Subject: [PATCH 550/569] It's not a problem to generate JIT for op_array with debug info, it's just slower. --- ext/opcache/jit/zend_jit.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 69202171d16c0..89100257027e2 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -690,10 +690,12 @@ static int zend_jit_op_array_analyze1(zend_op_array *op_array, zend_script *scri return FAILURE; } +#if 0 /* TODO: debugger and profiler supports? */ if ((ssa->cfg.flags & ZEND_FUNC_HAS_EXTENDED_INFO)) { return FAILURE; } +#endif if ((zend_jit_level >= ZEND_JIT_LEVEL_OPT_FUNC) && ssa->cfg.blocks From 44428f995bf11bc9f89ed7ffd38fa4a0bea22b74 Mon Sep 17 00:00:00 2001 From: Anatol Belski Date: Fri, 2 Sep 2016 17:03:22 +0200 Subject: [PATCH 551/569] Base work for Windows compatibility --- ext/opcache/config.w32 | 22 ++++++++++++++-- ext/opcache/jit/Makefile.frag.w32 | 16 ++++++++++++ ext/opcache/jit/zend_jit.c | 36 ++++++++++++++------------- ext/opcache/jit/zend_jit.h | 18 +++++++------- ext/opcache/jit/zend_jit_disasm_x86.c | 16 +++++++++++- ext/opcache/jit/zend_jit_internal.h | 8 ++++++ ext/opcache/jit/zend_jit_vm_helpers.c | 17 +++++++++++++ ext/opcache/jit/zend_jit_x86.dasc | 11 +++++++- ext/opcache/jit/zend_jit_x86.h | 20 +++++++++++++++ 9 files changed, 134 insertions(+), 30 deletions(-) create mode 100644 ext/opcache/jit/Makefile.frag.w32 diff --git a/ext/opcache/config.w32 b/ext/opcache/config.w32 index fba1b1bef1be9..27941ed3bd402 100644 --- a/ext/opcache/config.w32 +++ b/ext/opcache/config.w32 @@ -1,9 +1,10 @@ ARG_ENABLE("opcache", "whether to enable Zend OPcache support", "yes"); -/* var PHP_OPCACHE_PGO = false; */ if (PHP_OPCACHE != "no") { + ARG_ENABLE("opcache-jit", "whether to enable JIT", "yes"); + ZEND_EXTENSION('opcache', "\ ZendAccelerator.c \ zend_accelerator_blacklist.c \ @@ -17,9 +18,26 @@ if (PHP_OPCACHE != "no") { zend_shared_alloc.c \ shared_alloc_win32.c", true, "/DZEND_ENABLE_STATIC_TSRMLS_CACHE=1"); - ADD_SOURCES(configure_module_dirname + "/Optimizer", "zend_optimizer.c pass1_5.c pass2.c pass3.c optimize_func_calls.c block_pass.c optimize_temp_vars_5.c nop_removal.c compact_literals.c zend_cfg.c zend_dfg.c dfa_pass.c zend_ssa.c zend_inference.c zend_func_info.c zend_call_graph.c sccp.c scdf.c dce.c escape_analysis.c compact_vars.c zend_dump.c", "opcache", "OptimizerObj"); + if (PHP_OPCACHE_JIT == "yes") { + if (CHECK_HEADER_ADD_INCLUDE("dynasm/dasm_x86.h", "CFLAGS_OPCACHE", PHP_OPCACHE + ";ext\\opcache\\jit")) { + var dasm_flags = X64 ? "-D X64=1" : ""; + DEFINE("DASM_FLAGS", dasm_flags); + + AC_DEFINE('HAVE_JIT', 1, 'Define to enable JIT'); + /* XXX read this dynamically */ + /*ADD_FLAG("CFLAGS_OPCACHE", "/D DASM_VERSION=10400");*/ + + ADD_MAKEFILE_FRAGMENT(configure_module_dirname + "\\jit\\Makefile.frag.w32"); + ADD_SOURCES(configure_module_dirname + "\\jit", "zend_jit.c zend_jit_vm_helpers.c", "opcache", "opcache_jit"); + } else { + WARNING("JIT not enabled, headers not found"); + } + } + + ADD_SOURCES(configure_module_dirname + "/Optimizer", "zend_optimizer.c pass1_5.c pass2.c pass3.c optimize_func_calls.c block_pass.c optimize_temp_vars_5.c nop_removal.c compact_literals.c zend_cfg.c zend_dfg.c dfa_pass.c zend_ssa.c zend_inference.c zend_func_info.c zend_call_graph.c zend_dump.c escape_analysis.c compact_vars.c dce.c sccp.c scdf.c", "opcache", "OptimizerObj"); ADD_FLAG('CFLAGS_OPCACHE', "/I " + configure_module_dirname); } + diff --git a/ext/opcache/jit/Makefile.frag.w32 b/ext/opcache/jit/Makefile.frag.w32 new file mode 100644 index 0000000000000..0b5ca3d83852d --- /dev/null +++ b/ext/opcache/jit/Makefile.frag.w32 @@ -0,0 +1,16 @@ +$(BUILD_DIR)\\minilua.exe: ext\opcache\jit\dynasm\minilua.c + @if exist $(BUILD_DIR)\\minilua.exe del $(BUILD_DIR)\\minilua.exe + $(PHP_CL) /Fo$(BUILD_DIR)\ /Fd$(BUILD_DIR)\ /Fp$(BUILD_DIR)\ /FR$(BUILD_DIR) /Fe$(BUILD_DIR)\minilua.exe ext\opcache\jit\dynasm\minilua.c + +ext\opcache\jit\zend_jit_x86.c: $(BUILD_DIR)\\minilua.exe + @if exist ext\opcache\jit\zend_jit_x86.c del ext\opcache\jit\zend_jit_x86.c + $(BUILD_DIR)\\minilua.exe ext/opcache/jit/dynasm/dynasm.lua $(DASM_FLAGS) -o $@ ext/opcache/jit/zend_jit_x86.dasc + +$(BUILD_DIR)\opcache_jit\zend_jit.obj: \ + ext/opcache/jit/zend_jit_x86.c \ + ext/opcache/jit/zend_jit_helpers.c \ + ext/opcache/jit/zend_jit_disasm_x86.c \ + ext/opcache/jit/zend_jit_gdb.c \ + ext/opcache/jit/zend_jit_perf_dump.c \ + ext/opcache/jit/zend_jit_oprofile.c \ + ext/opcache/jit/zend_jit_vtune.c diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 89100257027e2..b873aac1a4ed3 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -129,8 +129,10 @@ static zend_bool zend_long_is_power_of_two(zend_long x) #include "jit/zend_jit_helpers.c" #include "jit/zend_jit_x86.c" #include "jit/zend_jit_disasm_x86.c" +#ifndef _WIN32 #include "jit/zend_jit_gdb.c" #include "jit/zend_jit_perf_dump.c" +#endif #ifdef HAVE_OPROFILE # include "jit/zend_jit_oprofile.c" #endif @@ -147,7 +149,7 @@ static zend_bool zend_long_is_power_of_two(zend_long x) #define DASM_ALIGNMENT 16 -ZEND_API void zend_jit_status(zval *ret) +ZEND_EXT_API void zend_jit_status(zval *ret) { zval stats; array_init(&stats); @@ -2824,7 +2826,7 @@ static int zend_needs_manual_jit(const zend_op_array *op_array) return 0; } -ZEND_API int zend_jit_op_array(zend_op_array *op_array, zend_script *script) +ZEND_EXT_API int zend_jit_op_array(zend_op_array *op_array, zend_script *script) { if (dasm_ptr == NULL) { return FAILURE; @@ -2869,7 +2871,7 @@ ZEND_API int zend_jit_op_array(zend_op_array *op_array, zend_script *script) } } -ZEND_API int zend_jit_script(zend_script *script) +ZEND_EXT_API int zend_jit_script(zend_script *script) { void *checkpoint; zend_call_graph call_graph; @@ -2989,7 +2991,7 @@ ZEND_API int zend_jit_script(zend_script *script) return FAILURE; } -ZEND_API void zend_jit_unprotect(void) +ZEND_EXT_API void zend_jit_unprotect(void) { #ifdef HAVE_MPROTECT if (!(ZCG(accel_directives).jit_debug & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP))) { @@ -3000,7 +3002,7 @@ ZEND_API void zend_jit_unprotect(void) #endif } -ZEND_API void zend_jit_protect(void) +ZEND_EXT_API void zend_jit_protect(void) { #ifdef HAVE_MPROTECT if (!(ZCG(accel_directives).jit_debug & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP))) { @@ -3076,7 +3078,7 @@ static int zend_jit_make_stubs(void) return 1; } -ZEND_API int zend_jit_startup(zend_long jit, size_t size) +ZEND_EXT_API int zend_jit_startup(zend_long jit, size_t size) { size_t page_size = jit_page_size(); int shared = 1; @@ -3168,7 +3170,7 @@ ZEND_API int zend_jit_startup(zend_long jit, size_t size) return SUCCESS; } -ZEND_API void zend_jit_shutdown(void) +ZEND_EXT_API void zend_jit_shutdown(void) { #ifdef HAVE_OPROFILE if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_OPROFILE) { @@ -3199,7 +3201,7 @@ ZEND_API void zend_jit_shutdown(void) } } -ZEND_API void zend_jit_activate(void) +ZEND_EXT_API void zend_jit_activate(void) { if (zend_jit_trigger == ZEND_JIT_ON_HOT_COUNTERS) { int i; @@ -3210,7 +3212,7 @@ ZEND_API void zend_jit_activate(void) } } -ZEND_API void zend_jit_deactivate(void) +ZEND_EXT_API void zend_jit_deactivate(void) { if (zend_jit_trigger == ZEND_JIT_ON_PROF_REQUEST) { if (!zend_jit_profile_counter) { @@ -3241,38 +3243,38 @@ ZEND_API void zend_jit_deactivate(void) #else /* HAVE_JIT */ -ZEND_API int zend_jit_op_array(zend_op_array *op_array, zend_script *script) +ZEND_EXT_API int zend_jit_op_array(zend_op_array *op_array, zend_script *script) { return FAILURE; } -ZEND_API int zend_jit_script(zend_script *script) +ZEND_EXT_API int zend_jit_script(zend_script *script) { return FAILURE; } -ZEND_API void zend_jit_unprotect(void) +ZEND_EXT_API void zend_jit_unprotect(void) { } -ZEND_API void zend_jit_protect(void) +ZEND_EXT_API void zend_jit_protect(void) { } -ZEND_API int zend_jit_startup(zend_long jit, size_t size) +ZEND_EXT_API int zend_jit_startup(zend_long jit, size_t size) { return FAILURE; } -ZEND_API void zend_jit_shutdown(void) +ZEND_EXT_API void zend_jit_shutdown(void) { } -ZEND_API void zend_jit_activate(void) +ZEND_EXT_API void zend_jit_activate(void) { } -ZEND_API void zend_jit_deactivate(void) +ZEND_EXT_API void zend_jit_deactivate(void) { } diff --git a/ext/opcache/jit/zend_jit.h b/ext/opcache/jit/zend_jit.h index acbd3e038851f..ab43ac6889399 100644 --- a/ext/opcache/jit/zend_jit.h +++ b/ext/opcache/jit/zend_jit.h @@ -75,15 +75,15 @@ #define ZEND_JIT_DEBUG_GDB (1<<8) -ZEND_API int zend_jit_op_array(zend_op_array *op_array, zend_script *script); -ZEND_API int zend_jit_script(zend_script *script); -ZEND_API void zend_jit_unprotect(void); -ZEND_API void zend_jit_protect(void); -ZEND_API int zend_jit_startup(zend_long jit, size_t size); -ZEND_API void zend_jit_shutdown(void); -ZEND_API void zend_jit_activate(void); -ZEND_API void zend_jit_deactivate(void); -ZEND_API void zend_jit_status(zval *ret); +ZEND_EXT_API int zend_jit_op_array(zend_op_array *op_array, zend_script *script); +ZEND_EXT_API int zend_jit_script(zend_script *script); +ZEND_EXT_API void zend_jit_unprotect(void); +ZEND_EXT_API void zend_jit_protect(void); +ZEND_EXT_API int zend_jit_startup(zend_long jit, size_t size); +ZEND_EXT_API void zend_jit_shutdown(void); +ZEND_EXT_API void zend_jit_activate(void); +ZEND_EXT_API void zend_jit_deactivate(void); +ZEND_EXT_API void zend_jit_status(zval *ret); typedef struct _zend_lifetime_interval zend_lifetime_interval; typedef struct _zend_life_range zend_life_range; diff --git a/ext/opcache/jit/zend_jit_disasm_x86.c b/ext/opcache/jit/zend_jit_disasm_x86.c index 91130d3b8a895..b4432a03e7657 100644 --- a/ext/opcache/jit/zend_jit_disasm_x86.c +++ b/ext/opcache/jit/zend_jit_disasm_x86.c @@ -34,7 +34,9 @@ static void zend_jit_disasm_add_symbol(const char *name, uint64_t addr, uint64_t size); -#include "jit/zend_elf.c" +#ifndef _WIN32 +# include "jit/zend_elf.c" +#endif #include "zend_sort.h" @@ -42,7 +44,9 @@ static void zend_jit_disasm_add_symbol(const char *name, # define _GNU_SOURCE #endif +#ifndef _WIN32 #include +#endif static struct ud ud; @@ -211,6 +215,7 @@ static const char* zend_jit_disasm_resolver(struct ud *ud, uint64_t addr, int64_t *offset) { +#ifndef _WIN32 ((void)ud); const char *name; void *a = (void*)(zend_uintptr_t)(addr); @@ -226,6 +231,13 @@ static const char* zend_jit_disasm_resolver(struct ud *ud, && info.dli_saddr == a) { return info.dli_sname; } +#else + const char *name; + name = zend_jit_disasm_find_symbol(addr, offset); + if (name) { + return name; + } +#endif return NULL; } @@ -444,7 +456,9 @@ static int zend_jit_disasm_init(void) REGISTER_HELPER(zend_jit_hot_func); #undef REGISTER_HELPER +#ifndef _WIN32 zend_elf_load_symbols(); +#endif return 1; } diff --git a/ext/opcache/jit/zend_jit_internal.h b/ext/opcache/jit/zend_jit_internal.h index 47a8eb54ea170..64c435feba1e3 100644 --- a/ext/opcache/jit/zend_jit_internal.h +++ b/ext/opcache/jit/zend_jit_internal.h @@ -34,7 +34,11 @@ extern int16_t zend_jit_hot_counters[ZEND_HOT_COUNTERS_COUNT]; void zend_jit_hot_func(zend_execute_data *execute_data, const zend_op *opline); /* VM handlers */ +#if HAVE_GCC_GLOBAL_REGS typedef void (ZEND_FASTCALL *zend_vm_opcode_handler_t)(void); +#else +typedef int (ZEND_FASTCALL *zend_vm_opcode_handler_t)(void); +#endif extern const zend_op *zend_jit_halt_op; @@ -43,7 +47,11 @@ void ZEND_FASTCALL zend_jit_leave_nested_func_helper(uint32_t call_info); void ZEND_FASTCALL zend_jit_leave_top_func_helper(uint32_t call_info); void ZEND_FASTCALL zend_jit_copy_extra_args_helper(void); void ZEND_FASTCALL zend_jit_deprecated_or_abstract_helper(void); +#if HAVE_GCC_GLOBAL_REGS void ZEND_FASTCALL zend_jit_profile_helper(void); +#else +int ZEND_FASTCALL zend_jit_profile_helper(void); +#endif void ZEND_FASTCALL zend_jit_func_counter_helper(void); void ZEND_FASTCALL zend_jit_loop_counter_helper(void); void ZEND_FASTCALL zend_jit_get_constant(const zval *key, uint32_t flags); diff --git a/ext/opcache/jit/zend_jit_vm_helpers.c b/ext/opcache/jit/zend_jit_vm_helpers.c index 24fd3b3de46e2..da92c104b243f 100644 --- a/ext/opcache/jit/zend_jit_vm_helpers.c +++ b/ext/opcache/jit/zend_jit_vm_helpers.c @@ -29,6 +29,7 @@ #include "zend_jit.h" #include "zend_jit_internal.h" +#ifndef _WIN32 #pragma GCC diagnostic ignored "-Wvolatile-register-var" #if defined(__x86_64__) register zend_execute_data* volatile execute_data __asm__("%r14"); @@ -38,6 +39,10 @@ register zend_execute_data* volatile execute_data __asm__("%esi"); register const zend_op* volatile opline __asm__("%edi"); #endif #pragma GCC diagnostic warning "-Wvolatile-register-var" +#else +static zend_execute_data* volatile execute_data; +static const zend_op* volatile opline; +#endif void ZEND_FASTCALL zend_jit_leave_nested_func_helper(uint32_t call_info) { @@ -147,13 +152,25 @@ void ZEND_FASTCALL zend_jit_deprecated_or_abstract_helper(void) } } +#if HAVE_GCC_GLOBAL_REGS void ZEND_FASTCALL zend_jit_profile_helper(void) +#else +int ZEND_FASTCALL zend_jit_profile_helper(void) +#endif { zend_op_array *op_array = (zend_op_array*)EX(func); const void *handler = (const void*)ZEND_FUNC_INFO(op_array); +#ifndef _WIN32 ++(ZEND_COUNTER_INFO(op_array)); +#else + ++((char*)ZEND_COUNTER_INFO(op_array)); +#endif ++zend_jit_profile_counter; +#if HAVE_GCC_GLOBAL_REGS + ((zend_vm_opcode_handler_t)handler)(); +#else return ((zend_vm_opcode_handler_t)handler)(); +#endif } static zend_always_inline zend_long _op_array_hash(const zend_op_array *op_array) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 72298235beea5..c55321c2cddfe 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -73,7 +73,7 @@ #include "jit/zend_jit_x86.h" const char* zend_reg_name[] = { -#ifdef __x86_64__ +#if defined(__x86_64__) || defined(_M_X64) "rax", "rcx", "rdx", "rbx", "rsp", "rbp", "rsi", "rdi", "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7", @@ -1379,7 +1379,11 @@ static void* dasm_labels[zend_lb_MAX]; | add r4, 4 | .endif ||#else +||#ifdef HAVE_BUILTIN_CONSTANT_P | EXT_CALL _efree_24, r0 +||#else +| EXT_CALL _efree, r0 +||#endif ||#endif |.endmacro @@ -1408,12 +1412,17 @@ static void* dasm_labels[zend_lb_MAX]; | add r4, 4 | .endif ||#else +||#ifdef HAVE_BUILTIN_CONSTANT_P || if (size == 24) { | EXT_CALL _emalloc_24, r0 || } else { | mov FCARG1a, size | EXT_CALL _emalloc, r0 || } +||#else +| mov FCARG1a, size +| EXT_CALL _emalloc, r0 +||#endif ||#endif |.endmacro diff --git a/ext/opcache/jit/zend_jit_x86.h b/ext/opcache/jit/zend_jit_x86.h index ff6e2c0c466f1..0a54d6498b090 100644 --- a/ext/opcache/jit/zend_jit_x86.h +++ b/ext/opcache/jit/zend_jit_x86.h @@ -145,8 +145,28 @@ typedef uint32_t zend_regset; (ZEND_REGSET(ZREG_RBX) | ZEND_REGSET(ZREG_RBP)) #endif +#ifndef _WIN32 #define ZEND_REGSET_FIRST(set) ((zend_reg)__builtin_ctz(set)) #define ZEND_REGSET_LAST(set) ((zend_reg)(__builtin_clz(set)^31))) +#else +#include +uint32_t __inline __zend_jit_ctz( uint32_t value ) { + DWORD trailing_zero = 0; + if (_BitScanForward(&trailing_zero, value)) { + return trailing_zero; + } + return 32; +} +uint32_t __inline __zend_jit_clz(uint32_t value) { + DWORD leading_zero = 0; + if (_BitScanReverse(&leading_zero, value)) { + return 31 - leading_zero; + } + return 32; +} +#define ZEND_REGSET_FIRST(set) ((zend_reg)__zend_jit_ctz(set)) +#define ZEND_REGSET_LAST(set) ((zend_reg)(__zend_jit_clz(set)^31))) +#endif #define ZEND_REGSET_FOREACH(set, reg) \ do { \ From 89ba2af51ba092235f3a87dcf0fcc5a05f097bb3 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 20 Feb 2019 01:34:49 +0300 Subject: [PATCH 552/569] Fixed test --- ext/opcache/tests/jit/fetch_obj_001.phpt | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ext/opcache/tests/jit/fetch_obj_001.phpt b/ext/opcache/tests/jit/fetch_obj_001.phpt index d68dd94a6c052..0213be47797c8 100644 --- a/ext/opcache/tests/jit/fetch_obj_001.phpt +++ b/ext/opcache/tests/jit/fetch_obj_001.phpt @@ -62,6 +62,7 @@ function bar() { bar(); ?> --EXPECTF-- +Warning: Creating default object from empty value in %s on line 14 object(stdClass)#%d (1) { ["a"]=> int(2) @@ -88,6 +89,8 @@ object(stdClass)#%d (2) { array(0) { } } + +Warning: Creating default object from empty value in %s on line 27 object(stdClass)#%d (1) { ["a"]=> int(2) @@ -118,10 +121,14 @@ object(stdClass)#%d (2) { Warning: Attempt to modify property 'abc' of non-object in %sfetch_obj_001.php on line 40 array(0) { } + +Warning: Creating default object from empty value in %s on line 44 object(stdClass)#%d (1) { ["abc"]=> int(2) } + +Warning: Creating default object from empty value in %s on line 48 object(stdClass)#%d (1) { ["abc"]=> int(2) From 17d7dada1ddff46ac5c02c5cfdbc892301a09825 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 21 Feb 2019 13:54:15 +0300 Subject: [PATCH 553/569] Make JIT VM helpers to be compatible with VM without GCC global registers --- ext/opcache/jit/zend_jit_internal.h | 66 ++++++++++++++++++++------- ext/opcache/jit/zend_jit_vm_helpers.c | 65 ++++++++++++++------------ 2 files changed, 87 insertions(+), 44 deletions(-) diff --git a/ext/opcache/jit/zend_jit_internal.h b/ext/opcache/jit/zend_jit_internal.h index 64c435feba1e3..c06a5d0b989fe 100644 --- a/ext/opcache/jit/zend_jit_internal.h +++ b/ext/opcache/jit/zend_jit_internal.h @@ -33,27 +33,61 @@ extern int16_t zend_jit_hot_counters[ZEND_HOT_COUNTERS_COUNT]; void zend_jit_hot_func(zend_execute_data *execute_data, const zend_op *opline); -/* VM handlers */ -#if HAVE_GCC_GLOBAL_REGS -typedef void (ZEND_FASTCALL *zend_vm_opcode_handler_t)(void); +extern const zend_op *zend_jit_halt_op; + +#ifdef HAVE_GCC_GLOBAL_REGS +# define EXECUTE_DATA_D void +# define EXECUTE_DATA_C +# define EXECUTE_DATA_DC +# define EXECUTE_DATA_CC +# define OPLINE_D void +# define OPLINE_C +# define OPLINE_DC +# define OPLINE_CC +# define ZEND_OPCODE_HANDLER_RET void +# define ZEND_OPCODE_HANDLER_ARGS EXECUTE_DATA_D +# define ZEND_OPCODE_HANDLER_ARGS_PASSTHRU +# define ZEND_OPCODE_HANDLER_ARGS_DC +# define ZEND_OPCODE_HANDLER_ARGS_PASSTHRU_CC +# define ZEND_OPCODE_RETURN() return +# define ZEND_OPCODE_TAIL_CALL(handler) do { \ + handler(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU); \ + return; \ + } while(0) #else -typedef int (ZEND_FASTCALL *zend_vm_opcode_handler_t)(void); +# define EXECUTE_DATA_D zend_execute_data* execute_data +# define EXECUTE_DATA_C execute_data +# define EXECUTE_DATA_DC , EXECUTE_DATA_D +# define EXECUTE_DATA_CC , EXECUTE_DATA_C +# define OPLINE_D const zend_op* opline +# define OPLINE_C opline +# define OPLINE_DC , OPLINE_D +# define OPLINE_CC , OPLINE_C +# define ZEND_OPCODE_HANDLER_RET int +# define ZEND_OPCODE_HANDLER_ARGS EXECUTE_DATA_D +# define ZEND_OPCODE_HANDLER_ARGS_PASSTHRU EXECUTE_DATA_C +# define ZEND_OPCODE_HANDLER_ARGS_DC EXECUTE_DATA_DC +# define ZEND_OPCODE_HANDLER_ARGS_PASSTHRU_CC EXECUTE_DATA_CC +# define ZEND_OPCODE_RETURN() return 0 +# define ZEND_OPCODE_TAIL_CALL(handler) do { \ + return handler(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU); \ + } while(0) #endif -extern const zend_op *zend_jit_halt_op; +/* VM handlers */ +typedef ZEND_OPCODE_HANDLER_RET (ZEND_FASTCALL *zend_vm_opcode_handler_t)(ZEND_OPCODE_HANDLER_ARGS); /* VM helpers */ -void ZEND_FASTCALL zend_jit_leave_nested_func_helper(uint32_t call_info); -void ZEND_FASTCALL zend_jit_leave_top_func_helper(uint32_t call_info); -void ZEND_FASTCALL zend_jit_copy_extra_args_helper(void); -void ZEND_FASTCALL zend_jit_deprecated_or_abstract_helper(void); -#if HAVE_GCC_GLOBAL_REGS -void ZEND_FASTCALL zend_jit_profile_helper(void); -#else -int ZEND_FASTCALL zend_jit_profile_helper(void); -#endif -void ZEND_FASTCALL zend_jit_func_counter_helper(void); -void ZEND_FASTCALL zend_jit_loop_counter_helper(void); +void ZEND_FASTCALL zend_jit_leave_nested_func_helper(uint32_t call_info EXECUTE_DATA_DC); +void ZEND_FASTCALL zend_jit_leave_top_func_helper(uint32_t call_info EXECUTE_DATA_DC); +void ZEND_FASTCALL zend_jit_copy_extra_args_helper(EXECUTE_DATA_D); +void ZEND_FASTCALL zend_jit_deprecated_or_abstract_helper(OPLINE_D); + +ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_profile_helper(ZEND_OPCODE_HANDLER_ARGS); + +ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_func_counter_helper(ZEND_OPCODE_HANDLER_ARGS); +ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_loop_counter_helper(ZEND_OPCODE_HANDLER_ARGS); + void ZEND_FASTCALL zend_jit_get_constant(const zval *key, uint32_t flags); int ZEND_FASTCALL zend_jit_check_constant(const zval *key); diff --git a/ext/opcache/jit/zend_jit_vm_helpers.c b/ext/opcache/jit/zend_jit_vm_helpers.c index da92c104b243f..18ffd0ad4eacc 100644 --- a/ext/opcache/jit/zend_jit_vm_helpers.c +++ b/ext/opcache/jit/zend_jit_vm_helpers.c @@ -29,22 +29,19 @@ #include "zend_jit.h" #include "zend_jit_internal.h" -#ifndef _WIN32 -#pragma GCC diagnostic ignored "-Wvolatile-register-var" -#if defined(__x86_64__) +#ifdef HAVE_GCC_GLOBAL_REGS +# pragma GCC diagnostic ignored "-Wvolatile-register-var" +# if defined(__x86_64__) register zend_execute_data* volatile execute_data __asm__("%r14"); register const zend_op* volatile opline __asm__("%r15"); -#else +# else register zend_execute_data* volatile execute_data __asm__("%esi"); register const zend_op* volatile opline __asm__("%edi"); -#endif -#pragma GCC diagnostic warning "-Wvolatile-register-var" -#else -static zend_execute_data* volatile execute_data; -static const zend_op* volatile opline; +# endif +# pragma GCC diagnostic warning "-Wvolatile-register-var" #endif -void ZEND_FASTCALL zend_jit_leave_nested_func_helper(uint32_t call_info) +void ZEND_FASTCALL zend_jit_leave_nested_func_helper(uint32_t call_info EXECUTE_DATA_DC) { zend_execute_data *old_execute_data; @@ -76,11 +73,13 @@ void ZEND_FASTCALL zend_jit_leave_nested_func_helper(uint32_t call_info) } } else { EX(opline)++; +#ifdef HAVE_GCC_GLOBAL_REGS opline = EX(opline); +#endif } } -void ZEND_FASTCALL zend_jit_leave_top_func_helper(uint32_t call_info) +void ZEND_FASTCALL zend_jit_leave_top_func_helper(uint32_t call_info EXECUTE_DATA_DC) { if (UNEXPECTED(call_info & (ZEND_CALL_HAS_SYMBOL_TABLE|ZEND_CALL_FREE_EXTRA_ARGS))) { if (UNEXPECTED(call_info & ZEND_CALL_HAS_SYMBOL_TABLE)) { @@ -93,10 +92,12 @@ void ZEND_FASTCALL zend_jit_leave_top_func_helper(uint32_t call_info) OBJ_RELEASE(ZEND_CLOSURE_OBJECT(EX(func))); } execute_data = EG(current_execute_data); +#ifdef HAVE_GCC_GLOBAL_REGS opline = zend_jit_halt_op; +#endif } -void ZEND_FASTCALL zend_jit_copy_extra_args_helper(void) +void ZEND_FASTCALL zend_jit_copy_extra_args_helper(EXECUTE_DATA_D) { zend_op_array *op_array = &EX(func)->op_array; @@ -108,7 +109,9 @@ void ZEND_FASTCALL zend_jit_copy_extra_args_helper(void) if (EXPECTED((op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0)) { /* Skip useless ZEND_RECV and ZEND_RECV_INIT opcodes */ +#ifdef HAVE_GCC_GLOBAL_REGS opline += first_extra_arg; +#endif } /* move extra args into separate array after all CV and TMP vars */ @@ -138,7 +141,7 @@ void ZEND_FASTCALL zend_jit_copy_extra_args_helper(void) } } -void ZEND_FASTCALL zend_jit_deprecated_or_abstract_helper(void) +void ZEND_FASTCALL zend_jit_deprecated_or_abstract_helper(OPLINE_D) { zend_function *fbc = ((zend_execute_data*)(opline))->func; @@ -152,25 +155,17 @@ void ZEND_FASTCALL zend_jit_deprecated_or_abstract_helper(void) } } -#if HAVE_GCC_GLOBAL_REGS -void ZEND_FASTCALL zend_jit_profile_helper(void) -#else -int ZEND_FASTCALL zend_jit_profile_helper(void) -#endif +ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_profile_helper(ZEND_OPCODE_HANDLER_ARGS) { zend_op_array *op_array = (zend_op_array*)EX(func); - const void *handler = (const void*)ZEND_FUNC_INFO(op_array); + zend_vm_opcode_handler_t handler = (zend_vm_opcode_handler_t)ZEND_FUNC_INFO(op_array); #ifndef _WIN32 ++(ZEND_COUNTER_INFO(op_array)); #else ++((char*)ZEND_COUNTER_INFO(op_array)); #endif ++zend_jit_profile_counter; -#if HAVE_GCC_GLOBAL_REGS - ((zend_vm_opcode_handler_t)handler)(); -#else - return ((zend_vm_opcode_handler_t)handler)(); -#endif + ZEND_OPCODE_TAIL_CALL(handler); } static zend_always_inline zend_long _op_array_hash(const zend_op_array *op_array) @@ -194,8 +189,11 @@ static zend_always_inline zend_long _op_array_hash(const zend_op_array *op_array return x; } -void ZEND_FASTCALL zend_jit_func_counter_helper(void) +ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_func_counter_helper(ZEND_OPCODE_HANDLER_ARGS) { +#ifndef HAVE_GCC_GLOBAL_REGS + const zend_op *opline = EX(opline); +#endif unsigned int n = _op_array_hash(&EX(func)->op_array) % (sizeof(zend_jit_hot_counters) / sizeof(zend_jit_hot_counters[0])); @@ -204,15 +202,20 @@ void ZEND_FASTCALL zend_jit_func_counter_helper(void) if (UNEXPECTED(zend_jit_hot_counters[n] <= 0)) { zend_jit_hot_counters[n] = ZEND_JIT_HOT_COUNTER_INIT; zend_jit_hot_func(execute_data, opline); + ZEND_OPCODE_RETURN(); } else { zend_vm_opcode_handler_t *handlers = (zend_vm_opcode_handler_t*)ZEND_FUNC_INFO(&EX(func)->op_array); - handlers[opline - EX(func)->op_array.opcodes](); + zend_vm_opcode_handler_t handler = handlers[opline - EX(func)->op_array.opcodes]; + ZEND_OPCODE_TAIL_CALL(handler); } } -void ZEND_FASTCALL zend_jit_loop_counter_helper(void) +ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_loop_counter_helper(ZEND_OPCODE_HANDLER_ARGS) { +#ifndef HAVE_GCC_GLOBAL_REGS + const zend_op *opline = EX(opline); +#endif unsigned int n = _op_array_hash(&EX(func)->op_array) % (sizeof(zend_jit_hot_counters) / sizeof(zend_jit_hot_counters[0])); @@ -221,16 +224,22 @@ void ZEND_FASTCALL zend_jit_loop_counter_helper(void) if (UNEXPECTED(zend_jit_hot_counters[n] <= 0)) { zend_jit_hot_counters[n] = ZEND_JIT_HOT_COUNTER_INIT; zend_jit_hot_func(execute_data, opline); + ZEND_OPCODE_RETURN(); } else { zend_vm_opcode_handler_t *handlers = (zend_vm_opcode_handler_t*)ZEND_FUNC_INFO(&EX(func)->op_array); - handlers[opline - EX(func)->op_array.opcodes](); + zend_vm_opcode_handler_t handler = handlers[opline - EX(func)->op_array.opcodes]; + ZEND_OPCODE_TAIL_CALL(handler); } } static zend_always_inline int _zend_quick_get_constant( const zval *key, uint32_t flags, int check_defined_only) { +#ifndef HAVE_GCC_GLOBAL_REGS + zend_execute_data *execute_data = EG(current_execute_data); + const zend_op *opline = EX(opline); +#endif zval *zv; zend_constant *c = NULL; From 61249011e8770e8e81dfa8a6639027e7ae9d8e63 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 21 Feb 2019 14:43:00 +0300 Subject: [PATCH 554/569] Use fastcall calling convention --- ext/opcache/jit/zend_jit.c | 2 +- ext/opcache/jit/zend_jit_internal.h | 2 +- ext/opcache/jit/zend_jit_x86.dasc | 22 ++++++---------------- 3 files changed, 8 insertions(+), 18 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index b873aac1a4ed3..71cbc7d5024e9 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -2744,7 +2744,7 @@ void zend_jit_check_funcs(HashTable *function_table, zend_bool is_method) { } ZEND_HASH_FOREACH_END(); } -void zend_jit_hot_func(zend_execute_data *execute_data, const zend_op *opline) +void ZEND_FASTCALL zend_jit_hot_func(zend_execute_data *execute_data, const zend_op *opline) { zend_op_array *op_array = &EX(func)->op_array; const void **orig_handlers; diff --git a/ext/opcache/jit/zend_jit_internal.h b/ext/opcache/jit/zend_jit_internal.h index c06a5d0b989fe..7db4e1691b5ef 100644 --- a/ext/opcache/jit/zend_jit_internal.h +++ b/ext/opcache/jit/zend_jit_internal.h @@ -31,7 +31,7 @@ extern int zend_jit_profile_counter_rid; extern int16_t zend_jit_hot_counters[ZEND_HOT_COUNTERS_COUNT]; -void zend_jit_hot_func(zend_execute_data *execute_data, const zend_op *opline); +void ZEND_FASTCALL zend_jit_hot_func(zend_execute_data *execute_data, const zend_op *opline); extern const zend_op *zend_jit_halt_op; diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index c55321c2cddfe..b4c64ea03d3e1 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -2044,17 +2044,12 @@ static int zend_jit_hybrid_func_counter_stub(dasm_State **Dst) |1: | .if X64 | mov word [r0+r1*2], ZEND_JIT_HOT_COUNTER_INIT - | mov CARG1, FP - | mov CARG2, IP - | EXT_CALL zend_jit_hot_func, r0 | .else | mov word [r1*2+&zend_jit_hot_counters], ZEND_JIT_HOT_COUNTER_INIT - | sub r4, 8 - | push IP - | push FP - | EXT_CALL zend_jit_hot_func, r0 - | add r4, 16 | .endif + | mov FCARG1a, FP + | mov FCARG2a, IP + | EXT_CALL zend_jit_hot_func, r0 | jmp aword [IP] return 1; } @@ -2122,17 +2117,12 @@ static int zend_jit_hybrid_loop_counter_stub(dasm_State **Dst) |1: | .if X64 | mov word [r0+r1*2], ZEND_JIT_HOT_COUNTER_INIT - | mov CARG1, FP - | mov CARG2, IP - | EXT_CALL zend_jit_hot_func, r0 | .else | mov word [r1*2+&zend_jit_hot_counters], ZEND_JIT_HOT_COUNTER_INIT - | sub r4, 8 - | push IP - | push FP - | EXT_CALL zend_jit_hot_func, r0 - | add r4, 16 | .endif + | mov FCARG1a, FP + | mov FCARG2a, IP + | EXT_CALL zend_jit_hot_func, r0 | jmp aword [IP] return 1; } From b6ce247637c582ed74464299fa63f9740e6fe572 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 21 Feb 2019 15:09:06 +0300 Subject: [PATCH 555/569] Use macros for IP regiser access --- ext/opcache/jit/zend_jit_x86.dasc | 138 ++++++++++++++++++------------ 1 file changed, 82 insertions(+), 56 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index b4c64ea03d3e1..574f2d338a337 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -27,8 +27,7 @@ |.define FP, r14 |.define IP, r15 |.define IPl, r15d - |.define RX, IP // the same as VM IP reused as a general purpos reg - |.define RXl, IPl + |.define RX, r15 // the same as VM IP reused as a general purpos reg |.define CARG1, rdi // x64/POSIX C call arguments. |.define CARG2, rsi |.define CARG3, rdx @@ -51,8 +50,7 @@ |.define FP, esi |.define IP, edi |.define IPl, edi - |.define RX, IP // the same as VM IP reused as a general purpos reg - |.define RXl, IPl + |.define RX, edi // the same as VM IP reused as a general purpos reg |.define FCARG1a, ecx // x86 fastcall arguments. |.define FCARG2a, edx |.define FCARG1d, ecx @@ -87,7 +85,7 @@ const char* zend_reg_name[] = { static uint32_t zend_jit_x86_flags = 0; |.type EX, zend_execute_data, FP -|.type OP, zend_op, IP +|.type OP, zend_op |.type ZVAL, zval |.actionlist dasm_actions @@ -115,6 +113,43 @@ static void* dasm_labels[zend_lb_MAX]; | .endif |.endmacro +|.macro SAVE_OPLINE +| mov aword EX->opline, IP +|.endmacro + +|.macro LOAD_OPLINE +| mov IP, aword EX->opline +|.endmacro + +|.macro LOAD_IP_ADDR, addr +| LOAD_ADDR IP, addr +|.endmacro + +|.macro GET_IP, rm +| mov rm, IP +|.endmacro + +|.macro SET_IP, val +| mov IP, val +|.endmacro + +|.macro ADD_IP, val +| add IP, val +|.endmacro + +|.macro JMP_IP +| jmp aword [IP] +|.endmacro + +/* In 64-bit build we compare only low 32-bits. + * x86_64 cmp instruction doesn't support immediate 64-bit operand, and full + * comparison would require additinal load of 64-bit address into register. + * This is not a problem at all, while JIT buffer size is less than 4GB. + */ +|.macro CMP_IP, addr +| cmp IPl, next_opline +|.endmacro + |.macro ADDR_OP1, addr_ins, addr, tmp_reg | .if X64 || if (IS_32BIT(addr)) { @@ -1189,7 +1224,7 @@ static void* dasm_labels[zend_lb_MAX]; |.macro SAVE_VALID_OPLINE, op || if (op == last_valid_opline) { -| mov aword EX->opline, IP +| SAVE_OPLINE || } else { | ADDR_OP2_2 mov, aword EX->opline, op, r0 || } @@ -1539,8 +1574,7 @@ static int zend_jit_interrupt_handler_stub(dasm_State **Dst) |1: | //} else if (zend_interrupt_function) { if (zend_interrupt_function) { - | //SAVE_OPLINE(); - | mov EX->opline, IP + | SAVE_OPLINE | //zend_interrupt_function(execute_data); |.if X64 | mov CARG1, FP @@ -1554,9 +1588,7 @@ static int zend_jit_interrupt_handler_stub(dasm_State **Dst) | //ZEND_VM_ENTER(); | //execute_data = EG(current_execute_data); | MEM_OP2_2 mov, FP, aword, &EG(current_execute_data), r0 - | // LOAD_OPLINE(); - | mov IP, EX->opline - | //} + | LOAD_OPLINE } | //ZEND_VM_CONTINUE() if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { @@ -1564,7 +1596,7 @@ static int zend_jit_interrupt_handler_stub(dasm_State **Dst) } else { | add r4, SPAD // stack alignment } - | jmp aword [IP] + | JMP_IP return 1; } @@ -1577,7 +1609,7 @@ static int zend_jit_exception_handler_stub(dasm_State **Dst) | add r4, HYBRID_SPAD | EXT_CALL handler, r0 - | jmp aword [IP] + | JMP_IP } else { const void *handler = EG(exception_op)->handler; @@ -1605,11 +1637,11 @@ static int zend_jit_leave_function_stub(dasm_State **Dst) | jnz >1 | EXT_CALL zend_jit_leave_nested_func_helper, r0 | add r4, HYBRID_SPAD // stack alignment - | jmp aword [IP] + | JMP_IP |1: | EXT_CALL zend_jit_leave_top_func_helper, r0 | add r4, HYBRID_SPAD // stack alignment - | jmp aword [IP] + | JMP_IP } else { | add r4, SPAD | test FCARG1d, ZEND_CALL_TOP @@ -1632,7 +1664,7 @@ static int zend_jit_leave_throw_stub(dasm_State **Dst) | MEM_OP2_1 mov, aword, &EG(opline_before_exception), IP, r0 |5: | // opline = EG(exception_op); - | LOAD_ADDR IP, &EG(exception_op) + | LOAD_IP_ADDR &EG(exception_op) | // HANDLE_EXCEPTION() | jmp ->exception_handler @@ -1701,7 +1733,7 @@ static int zend_jit_throw_cannot_pass_by_ref_stub(dasm_State **Dst) static int zend_jit_undefined_offset_ex_stub(dasm_State **Dst) { |->undefined_offset_ex: - | mov aword EX->opline, IP + | SAVE_OPLINE | jmp ->undefined_offset return 1; @@ -1761,7 +1793,7 @@ static int zend_jit_undefined_offset_stub(dasm_State **Dst) static int zend_jit_undefined_index_ex_stub(dasm_State **Dst) { |->undefined_index_ex: - | mov aword EX->opline, IP + | SAVE_OPLINE | jmp ->undefined_index return 1; @@ -1824,7 +1856,7 @@ static int zend_jit_undefined_index_stub(dasm_State **Dst) static int zend_jit_cannot_add_element_ex_stub(dasm_State **Dst) { |->cannot_add_element_ex: - | mov aword EX->opline, IP + | SAVE_OPLINE | jmp ->cannot_add_element return 1; @@ -1886,7 +1918,7 @@ static int zend_jit_not_obj_stub(dasm_State **Dst) static int zend_jit_negative_shift_stub(dasm_State **Dst) { |->negative_shift: - | mov aword EX->opline, IP + | SAVE_OPLINE |.if X64 | LOAD_ADDR CARG1, zend_ce_arithmetic_error | LOAD_ADDR CARG2, "Bit shift by negative number" @@ -1905,7 +1937,7 @@ static int zend_jit_negative_shift_stub(dasm_State **Dst) static int zend_jit_mod_by_zero_stub(dasm_State **Dst) { |->mod_by_zero: - | mov aword EX->opline, IP + | SAVE_OPLINE |.if X64 | LOAD_ADDR CARG1, zend_ce_division_by_zero_error | LOAD_ADDR CARG2, "Modulo by zero" @@ -1932,7 +1964,7 @@ static int zend_jit_hybrid_runtime_jit_stub(dasm_State **Dst) { |->hybrid_runtime_jit: | EXT_CALL zend_runtime_jit, r0 - | jmp aword [IP] + | JMP_IP return 1; } @@ -2024,7 +2056,7 @@ static int zend_jit_hybrid_func_counter_stub(dasm_State **Dst) | .endif | jle >1 | mov r1, EX->func - | mov r0, IP + | GET_IP r0 | sub r0, aword [r1 + offsetof(zend_op_array, opcodes)] | // divide by sizeof(zend_op) | .if X64 @@ -2048,9 +2080,9 @@ static int zend_jit_hybrid_func_counter_stub(dasm_State **Dst) | mov word [r1*2+&zend_jit_hot_counters], ZEND_JIT_HOT_COUNTER_INIT | .endif | mov FCARG1a, FP - | mov FCARG2a, IP + | GET_IP FCARG2a | EXT_CALL zend_jit_hot_func, r0 - | jmp aword [IP] + | JMP_IP return 1; } @@ -2097,7 +2129,7 @@ static int zend_jit_hybrid_loop_counter_stub(dasm_State **Dst) | .endif | jle >1 | mov r1, EX->func - | mov r0, IP + | GET_IP r0 | sub r0, aword [r1 + offsetof(zend_op_array, opcodes)] | // divide by sizeof(zend_op) | .if X64 @@ -2121,9 +2153,9 @@ static int zend_jit_hybrid_loop_counter_stub(dasm_State **Dst) | mov word [r1*2+&zend_jit_hot_counters], ZEND_JIT_HOT_COUNTER_INIT | .endif | mov FCARG1a, FP - | mov FCARG2a, IP + | GET_IP FCARG2a | EXT_CALL zend_jit_hot_func, r0 - | jmp aword [IP] + | JMP_IP return 1; } @@ -2210,9 +2242,9 @@ static int zend_jit_save_call_chain(dasm_State **Dst, uint32_t call_level) static int zend_jit_set_ip(dasm_State **Dst, const zend_op *opline) { if (!last_valid_opline) { - | LOAD_ADDR IP, opline + | LOAD_IP_ADDR opline } else if (last_valid_opline != opline) { - | add IP, (opline - last_valid_opline) * sizeof(zend_op); + | ADD_IP (opline - last_valid_opline) * sizeof(zend_op); } return 1; } @@ -2244,7 +2276,7 @@ static int zend_jit_check_timeout(dasm_State **Dst, const zend_op *opline) | jne >1 |.cold_code |1: - | LOAD_ADDR IP, opline + | LOAD_IP_ADDR opline | jmp ->interrupt_handler |.code return 1; @@ -2334,7 +2366,7 @@ static int zend_jit_tail_handler(dasm_State **Dst, const zend_op *opline) | add r4, HYBRID_SPAD | EXT_CALL handler, r0 - | jmp aword [IP] + | JMP_IP } else { const void *handler = opline->handler; @@ -2376,12 +2408,7 @@ static int zend_jit_jmp(dasm_State **Dst, unsigned int target_label) static int zend_jit_cond_jmp(dasm_State **Dst, const zend_op *next_opline, unsigned int target_label) { -/* In 64-bit build we compare only low 32-bits. - * x86_64 cmp instruction doesn't support immediate 64-bit operand, and full - * comparison would require additinal load of 64-bit address into register. - * This is not a problem at all, while JIT buffer size is less than 4GB. - */ - | cmp IPl, next_opline + | CMP_IP next_opline | jne =>target_label last_valid_opline = next_opline; @@ -2394,9 +2421,9 @@ static int zend_jit_smart_branch(dasm_State **Dst, const zend_op *opline, unsign const zend_op *next_opline = opline + 1; const zend_op *target_opline = OP_JMP_ADDR(opline, opline->op2); - | cmp IPl, next_opline + | CMP_IP next_opline | je =>next_label - | cmp IPl, target_opline + | CMP_IP target_opline | je =>target_label return 1; @@ -2412,7 +2439,7 @@ static int zend_jit_context_threaded_call(dasm_State **Dst, const zend_op *oplin } else { const zend_op *next_opline = opline + 1; - | cmp IPl, next_opline + | CMP_IP next_opline | je >1 | call aword [IP] zend_jit_check_exception(Dst); @@ -7116,19 +7143,19 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar num_args = call_info->num_args; } if (func && zend_accel_in_shm(func->op_array.opcodes)) { - | LOAD_ADDR IP, (func->op_array.opcodes + num_args) + | LOAD_IP_ADDR (func->op_array.opcodes + num_args) } else { if (func) { | mov r0, EX->func } - | mov IP, aword [r0 + offsetof(zend_op_array, opcodes)] + | SET_IP aword [r0 + offsetof(zend_op_array, opcodes)] if (num_args) { - | add IP, (num_args * sizeof(zend_op)) + | ADD_IP (num_args * sizeof(zend_op)) } } } else { | // opline = op_array->opcodes - | mov IP, aword [r0 + offsetof(zend_op_array, opcodes)] + | SET_IP aword [r0 + offsetof(zend_op_array, opcodes)] | // first_extra_arg = op_array->num_args; | mov edx, dword [r0 + offsetof(zend_op_array, num_args)] | // num_args = EX_NUM_ARGS(); @@ -7153,7 +7180,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar |.else | imul r2, ecx, sizeof(zend_op) |.endif - | add IP, r2 + | ADD_IP r2 |1: | // if (EXPECTED((int)num_args < op_array->last_var)) { | mov edx, dword [r0 + offsetof(zend_op_array, last_var)] @@ -7207,7 +7234,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar } else { | add r4, SPAD // stack alignment } - | jmp aword [IP] + | JMP_IP #endif } } @@ -7350,7 +7377,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar | jne >1 |.cold_code |1: - | LOAD_ADDR IP, opline + | LOAD_IP_ADDR opline | jmp ->icall_throw_handler |.code @@ -7361,7 +7388,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar if (opline->opcode != ZEND_DO_ICALL) { uint32_t target_label = ssa->cfg.map[opline - op_array->opcodes] + 1; - | LOAD_ADDR IP, (opline + 1) + | LOAD_IP_ADDR (opline + 1) | jmp =>target_label } } @@ -7412,7 +7439,7 @@ static int zend_jit_new(dasm_State **Dst, const zend_op *opline, int *opnum, zen if (!ce || !(ce->ce_flags & ZEND_ACC_LINKED) || ce->constructor) { const zend_op *next_opline = opline + 1; - | cmp IPl, next_opline + | CMP_IP next_opline | jne >6 #if 0 zend_jit_call(Dst, next_opline); @@ -8283,10 +8310,10 @@ static int zend_jit_leave_func(dasm_State **Dst, const zend_op *opline, zend_op_ } | // if (EG(exception)) | MEM_OP2_1 cmp, aword, &EG(exception), 0, r0 - | mov IP, EX->opline + | LOAD_OPLINE | jne ->leave_throw_handler | // opline = EX(opline) + 1 - | add IP, sizeof(zend_op) + | ADD_IP sizeof(zend_op) if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { | add r4, HYBRID_SPAD } else { @@ -8295,7 +8322,7 @@ static int zend_jit_leave_func(dasm_State **Dst, const zend_op *opline, zend_op_ #ifdef CONTEXT_THREADED_JIT | ret #else - | jmp aword [IP] + | JMP_IP #endif return 1; @@ -9109,7 +9136,7 @@ static int zend_jit_recv_init(dasm_State **Dst, const zend_op *opline, zend_op_a } } if (is_last) { - | LOAD_ADDR IP, (opline + 1) + | LOAD_IP_ADDR (opline + 1) last_valid_opline = (opline + 1); } @@ -9487,8 +9514,7 @@ static int zend_jit_echo(dasm_State **Dst, const zend_op *opline, zend_op_array if (!zend_jit_set_valid_ip(Dst, opline)) { return 0; } - | //SAVE_OPLINE(); - | mov EX->opline, IP + | SAVE_OPLINE |.if X64 | LOAD_ADDR CARG1, str | LOAD_ADDR CARG2, len From 8ebe9857f33a70ce8152703bedafaf27c6bfdaa6 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 22 Feb 2019 01:56:49 +0300 Subject: [PATCH 556/569] Implemented JIT supprt for VM without GCC global register variables --- ext/opcache/jit/zend_jit_internal.h | 9 +- ext/opcache/jit/zend_jit_vm_helpers.c | 11 +- ext/opcache/jit/zend_jit_x86.dasc | 216 +++++++++++++++++++++----- 3 files changed, 191 insertions(+), 45 deletions(-) diff --git a/ext/opcache/jit/zend_jit_internal.h b/ext/opcache/jit/zend_jit_internal.h index 7db4e1691b5ef..bd42bdcc64cf9 100644 --- a/ext/opcache/jit/zend_jit_internal.h +++ b/ext/opcache/jit/zend_jit_internal.h @@ -78,16 +78,17 @@ extern const zend_op *zend_jit_halt_op; typedef ZEND_OPCODE_HANDLER_RET (ZEND_FASTCALL *zend_vm_opcode_handler_t)(ZEND_OPCODE_HANDLER_ARGS); /* VM helpers */ -void ZEND_FASTCALL zend_jit_leave_nested_func_helper(uint32_t call_info EXECUTE_DATA_DC); -void ZEND_FASTCALL zend_jit_leave_top_func_helper(uint32_t call_info EXECUTE_DATA_DC); -void ZEND_FASTCALL zend_jit_copy_extra_args_helper(EXECUTE_DATA_D); -void ZEND_FASTCALL zend_jit_deprecated_or_abstract_helper(OPLINE_D); +ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_leave_nested_func_helper(uint32_t call_info EXECUTE_DATA_DC); +ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_leave_top_func_helper(uint32_t call_info EXECUTE_DATA_DC); ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_profile_helper(ZEND_OPCODE_HANDLER_ARGS); ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_func_counter_helper(ZEND_OPCODE_HANDLER_ARGS); ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_loop_counter_helper(ZEND_OPCODE_HANDLER_ARGS); +void ZEND_FASTCALL zend_jit_copy_extra_args_helper(EXECUTE_DATA_D); +void ZEND_FASTCALL zend_jit_deprecated_or_abstract_helper(OPLINE_D); + void ZEND_FASTCALL zend_jit_get_constant(const zval *key, uint32_t flags); int ZEND_FASTCALL zend_jit_check_constant(const zval *key); diff --git a/ext/opcache/jit/zend_jit_vm_helpers.c b/ext/opcache/jit/zend_jit_vm_helpers.c index 18ffd0ad4eacc..9d543b90edfd7 100644 --- a/ext/opcache/jit/zend_jit_vm_helpers.c +++ b/ext/opcache/jit/zend_jit_vm_helpers.c @@ -41,7 +41,7 @@ register const zend_op* volatile opline __asm__("%edi"); # pragma GCC diagnostic warning "-Wvolatile-register-var" #endif -void ZEND_FASTCALL zend_jit_leave_nested_func_helper(uint32_t call_info EXECUTE_DATA_DC) +ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_leave_nested_func_helper(uint32_t call_info EXECUTE_DATA_DC) { zend_execute_data *old_execute_data; @@ -71,15 +71,20 @@ void ZEND_FASTCALL zend_jit_leave_nested_func_helper(uint32_t call_info EXECUTE_ if (old_opline->result_type != IS_UNDEF) { zval_ptr_dtor(EX_VAR(old_opline->result.var)); } +#ifndef HAVE_GCC_GLOBAL_REGS + return 2; // ZEND_VM_LEAVE +#endif } else { EX(opline)++; #ifdef HAVE_GCC_GLOBAL_REGS opline = EX(opline); +#else + return 2; // ZEND_VM_LEAVE #endif } } -void ZEND_FASTCALL zend_jit_leave_top_func_helper(uint32_t call_info EXECUTE_DATA_DC) +ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_leave_top_func_helper(uint32_t call_info EXECUTE_DATA_DC) { if (UNEXPECTED(call_info & (ZEND_CALL_HAS_SYMBOL_TABLE|ZEND_CALL_FREE_EXTRA_ARGS))) { if (UNEXPECTED(call_info & ZEND_CALL_HAS_SYMBOL_TABLE)) { @@ -94,6 +99,8 @@ void ZEND_FASTCALL zend_jit_leave_top_func_helper(uint32_t call_info EXECUTE_DAT execute_data = EG(current_execute_data); #ifdef HAVE_GCC_GLOBAL_REGS opline = zend_jit_halt_op; +#else + return -1; // ZEND_VM_RETURN #endif } diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 574f2d338a337..ae934c87fe862 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -44,7 +44,8 @@ |.define FCARG2a, CARG2 |.define FCARG1d, CARG1d |.define FCARG2d, CARG2d - |.define SPAD, 8 // padding for CPU stack alignment + |.define SPAD, 0x08 // padding for CPU stack alignment + |.define NR_SPAD, 0x18 // padding for CPU stack alignment |.define SSE, 1 |.else |.define FP, esi @@ -56,6 +57,7 @@ |.define FCARG1d, ecx |.define FCARG2d, edx |.define SPAD, 12 // padding for CPU stack alignment + |.define NR_SPAD, 12 // padding for CPU stack alignment |.define SSE, 1 |.endif @@ -82,6 +84,12 @@ const char* zend_reg_name[] = { #endif }; +#ifdef HAVE_GCC_GLOBAL_REGS +# define GCC_GLOBAL_REGS 1 +#else +# define GCC_GLOBAL_REGS 0 +#endif + static uint32_t zend_jit_x86_flags = 0; |.type EX, zend_execute_data, FP @@ -114,31 +122,49 @@ static void* dasm_labels[zend_lb_MAX]; |.endmacro |.macro SAVE_OPLINE -| mov aword EX->opline, IP +|| if (GCC_GLOBAL_REGS) { +| mov aword EX->opline, IP +|| } |.endmacro |.macro LOAD_OPLINE -| mov IP, aword EX->opline +|| if (GCC_GLOBAL_REGS) { +| mov IP, aword EX->opline +|| } |.endmacro |.macro LOAD_IP_ADDR, addr -| LOAD_ADDR IP, addr -|.endmacro - -|.macro GET_IP, rm -| mov rm, IP +|| if (GCC_GLOBAL_REGS) { +| LOAD_ADDR IP, addr +|| } else { +| LOAD_ADDR RX, addr +| mov aword EX->opline, RX +|| } |.endmacro -|.macro SET_IP, val -| mov IP, val +|.macro GET_IP, reg +|| if (GCC_GLOBAL_REGS) { +| mov reg, IP +|| } else { +| mov reg, aword EX->opline +|| } |.endmacro |.macro ADD_IP, val -| add IP, val +|| if (GCC_GLOBAL_REGS) { +| add IP, val +|| } else { +| add aword EX->opline, val +|| } |.endmacro |.macro JMP_IP -| jmp aword [IP] +|| if (GCC_GLOBAL_REGS) { +| jmp aword [IP] +|| } else { +| mov r0, aword EX:FCARG1a->opline +| jmp aword [r0] +|| } |.endmacro /* In 64-bit build we compare only low 32-bits. @@ -147,7 +173,11 @@ static void* dasm_labels[zend_lb_MAX]; * This is not a problem at all, while JIT buffer size is less than 4GB. */ |.macro CMP_IP, addr -| cmp IPl, next_opline +|| if (GCC_GLOBAL_REGS) { +| cmp IPl, addr +|| } else { +| cmp dword EX->opline, addr +|| } |.endmacro |.macro ADDR_OP1, addr_ins, addr, tmp_reg @@ -1593,10 +1623,17 @@ static int zend_jit_interrupt_handler_stub(dasm_State **Dst) | //ZEND_VM_CONTINUE() if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { | add r4, HYBRID_SPAD - } else { + | JMP_IP + } else if (GCC_GLOBAL_REGS) { | add r4, SPAD // stack alignment + | JMP_IP + } else { + | mov FP, aword [r4 + sizeof(void*)] // restore FP + | mov RX, aword [r4 + sizeof(void*) * 2] // restore IP + | add r4, NR_SPAD // stack alignment + | mov r0, 1 // ZEND_VM_ENTER + | ret } - | JMP_IP return 1; } @@ -1613,7 +1650,14 @@ static int zend_jit_exception_handler_stub(dasm_State **Dst) } else { const void *handler = EG(exception_op)->handler; - | add r4, SPAD // stack alignment + if (GCC_GLOBAL_REGS) { + | add r4, SPAD // stack alignment + } else { + | mov FCARG1a, FP + | mov FP, aword [r4 + sizeof(void*)] // restore FP + | mov RX, aword [r4 + sizeof(void*) * 2] // restore IP + | add r4, NR_SPAD // stack alignment + } | EXT_JMP handler, r0 } @@ -1643,7 +1687,14 @@ static int zend_jit_leave_function_stub(dasm_State **Dst) | add r4, HYBRID_SPAD // stack alignment | JMP_IP } else { - | add r4, SPAD + if (GCC_GLOBAL_REGS) { + | add r4, SPAD + } else { + | mov FCARG2a, FP + | mov FP, aword [r4 + sizeof(void*)] // restore FP + | mov RX, aword [r4 + sizeof(void*) * 2] // restore IP + | add r4, NR_SPAD + } | test FCARG1d, ZEND_CALL_TOP | jnz >1 | EXT_JMP zend_jit_leave_nested_func_helper, r0 @@ -1658,15 +1709,31 @@ static int zend_jit_leave_throw_stub(dasm_State **Dst) { |->leave_throw_handler: | // if (opline->opcode != ZEND_HANDLE_EXCEPTION) { - | cmp byte OP:IP->opcode, ZEND_HANDLE_EXCEPTION - | je >5 - | // EG(opline_before_exception) = opline; - | MEM_OP2_1 mov, aword, &EG(opline_before_exception), IP, r0 - |5: - | // opline = EG(exception_op); - | LOAD_IP_ADDR &EG(exception_op) - | // HANDLE_EXCEPTION() - | jmp ->exception_handler + if (GCC_GLOBAL_REGS) { + | cmp byte OP:IP->opcode, ZEND_HANDLE_EXCEPTION + | je >5 + | // EG(opline_before_exception) = opline; + | MEM_OP2_1 mov, aword, &EG(opline_before_exception), IP, r0 + |5: + | // opline = EG(exception_op); + | LOAD_IP_ADDR &EG(exception_op) + | // HANDLE_EXCEPTION() + | jmp ->exception_handler + } else { + | GET_IP FCARG1a + | cmp byte OP:FCARG1a->opcode, ZEND_HANDLE_EXCEPTION + | je >5 + | // EG(opline_before_exception) = opline; + | MEM_OP2_1 mov, aword, &EG(opline_before_exception), FCARG1a, r0 + |5: + | // opline = EG(exception_op); + | LOAD_IP_ADDR &EG(exception_op) + | mov FP, aword [r4 + sizeof(void*)] // restore FP + | mov RX, aword [r4 + sizeof(void*) * 2] // restore IP + | add r4, NR_SPAD // stack alignment + | mov r0, 2 // ZEND_VM_LEAVE + | ret + } return 1; } @@ -2210,8 +2277,13 @@ static int zend_jit_prologue(dasm_State **Dst) { if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { | sub r4, HYBRID_SPAD - } else { + } else if (GCC_GLOBAL_REGS) { | sub r4, SPAD // stack alignment + } else { + | sub r4, NR_SPAD // stack alignment + | mov aword [r4 + sizeof(void*)], FP // save FP + | mov aword [r4 + sizeof(void*) * 2], RX // save IP + | mov FP, FCARG1a } return 1; } @@ -2244,7 +2316,11 @@ static int zend_jit_set_ip(dasm_State **Dst, const zend_op *opline) if (!last_valid_opline) { | LOAD_IP_ADDR opline } else if (last_valid_opline != opline) { - | ADD_IP (opline - last_valid_opline) * sizeof(zend_op); + if (GCC_GLOBAL_REGS) { + | ADD_IP (opline - last_valid_opline) * sizeof(zend_op); + } else { + | LOAD_IP_ADDR opline + } } return 1; } @@ -2318,6 +2394,9 @@ static int zend_jit_handler(dasm_State **Dst, const zend_op *opline, int may_thr if (!zend_jit_set_valid_ip(Dst, opline)) { return 0; } + if (!GCC_GLOBAL_REGS) { + | mov FCARG1a, FP + } | EXT_CALL handler, r0 if (may_throw) { zend_jit_check_exception(Dst); @@ -2370,7 +2449,14 @@ static int zend_jit_tail_handler(dasm_State **Dst, const zend_op *opline) } else { const void *handler = opline->handler; - | add r4, SPAD // stack alignment + if (GCC_GLOBAL_REGS) { + | add r4, SPAD // stack alignment + } else { + | mov FCARG1a, FP + | mov FP, aword [r4 + sizeof(void*)] // restore FP + | mov RX, aword [r4 + sizeof(void*) * 2] // restore IP + | add r4, NR_SPAD // stack alignment + } | EXT_JMP handler, r0 } last_valid_opline = NULL; @@ -7054,6 +7140,9 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar | jnz >1 |.cold_code |1: + if (!GCC_GLOBAL_REGS) { + | mov FCARG1a, RX + } | EXT_CALL zend_jit_deprecated_or_abstract_helper, r0 | MEM_OP2_1 cmp, aword, &EG(exception), 0, r0 | jne ->exception_handler @@ -7062,9 +7151,15 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar |.code |1: } else if (func->common.fn_flags & ZEND_ACC_ABSTRACT) { + if (!GCC_GLOBAL_REGS) { + | mov FCARG1a, RX + } | EXT_CALL zend_jit_deprecated_or_abstract_helper, r0 | jmp ->exception_handler } else if (func->common.fn_flags & ZEND_ACC_DEPRECATED) { + if (!GCC_GLOBAL_REGS) { + | mov FCARG1a, RX + } | EXT_CALL zend_jit_deprecated_or_abstract_helper, r0 | MEM_OP2_1 cmp, aword, &EG(exception), 0, r0 | jne ->exception_handler @@ -7148,14 +7243,27 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar if (func) { | mov r0, EX->func } - | SET_IP aword [r0 + offsetof(zend_op_array, opcodes)] - if (num_args) { - | ADD_IP (num_args * sizeof(zend_op)) + if (GCC_GLOBAL_REGS) { + | mov IP, aword [r0 + offsetof(zend_op_array, opcodes)] + if (num_args) { + | add IP, (num_args * sizeof(zend_op)) + } + } else { + | mov FCARG1a, aword [r0 + offsetof(zend_op_array, opcodes)] + if (num_args) { + | add FCARG1a, (num_args * sizeof(zend_op)) + } + | mov aword EX->opline, FCARG1a } } } else { | // opline = op_array->opcodes - | SET_IP aword [r0 + offsetof(zend_op_array, opcodes)] + if (GCC_GLOBAL_REGS) { + | mov IP, aword [r0 + offsetof(zend_op_array, opcodes)] + } else { + | mov FCARG1a, aword [r0 + offsetof(zend_op_array, opcodes)] + | mov aword EX->opline, FCARG1a + } | // first_extra_arg = op_array->num_args; | mov edx, dword [r0 + offsetof(zend_op_array, num_args)] | // num_args = EX_NUM_ARGS(); @@ -7165,6 +7273,9 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar | jl >1 |.cold_code |1: + if (!GCC_GLOBAL_REGS) { + | mov FCARG1a, FP + } | EXT_CALL zend_jit_copy_extra_args_helper, r0 | mov r0, EX->func // reload | mov ecx, dword [FP + offsetof(zend_execute_data, This.u2.num_args)] // reload @@ -7231,10 +7342,17 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar #else if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { | add r4, HYBRID_SPAD - } else { + | JMP_IP + } else if (GCC_GLOBAL_REGS) { | add r4, SPAD // stack alignment + | JMP_IP + } else { + | mov FP, aword [r4 + sizeof(void*)] // restore FP + | mov RX, aword [r4 + sizeof(void*) * 2] // restore IP + | add r4, NR_SPAD // stack alignment + | mov r0, 1 // ZEND_VM_ENTER + | ret } - | JMP_IP #endif } } @@ -7249,6 +7367,9 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar | jnz >1 |.cold_code |1: + if (!GCC_GLOBAL_REGS) { + | mov FCARG1a, RX + } | EXT_CALL zend_jit_deprecated_or_abstract_helper, r0 | MEM_OP2_1 cmp, aword, &EG(exception), 0, r0 | jne ->exception_handler @@ -7257,9 +7378,15 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar |.code |1: } else if (func->common.fn_flags & ZEND_ACC_ABSTRACT) { + if (!GCC_GLOBAL_REGS) { + | mov FCARG1a, RX + } | EXT_CALL zend_jit_deprecated_or_abstract_helper, r0 | jmp ->exception_handler } else if (func->common.fn_flags & ZEND_ACC_DEPRECATED) { + if (!GCC_GLOBAL_REGS) { + | mov FCARG1a, RX + } | EXT_CALL zend_jit_deprecated_or_abstract_helper, r0 | MEM_OP2_1 cmp, aword, &EG(exception), 0, r0 | jne ->exception_handler @@ -8316,14 +8443,25 @@ static int zend_jit_leave_func(dasm_State **Dst, const zend_op *opline, zend_op_ | ADD_IP sizeof(zend_op) if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { | add r4, HYBRID_SPAD - } else { +#ifdef CONTEXT_THREADED_JIT + | ret +#else + | JMP_IP +#endif + } else if (GCC_GLOBAL_REGS) { | add r4, SPAD // stack alignment - } #ifdef CONTEXT_THREADED_JIT - | ret + | ret #else - | JMP_IP + | JMP_IP #endif + } else { + | mov FP, aword [r4 + sizeof(void*)] // restore FP + | mov RX, aword [r4 + sizeof(void*) * 2] // restore IP + | add r4, NR_SPAD // stack alignment + | mov r0, 2 // ZEND_VM_LEAVE + | ret + } return 1; } From 3aa1756536d25863bae05772be931611d94dc5fa Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 22 Feb 2019 12:47:16 +0300 Subject: [PATCH 557/569] Fixed JIT after first profiling request --- ext/opcache/ZendAccelerator.c | 15 +++++++++++---- ext/opcache/jit/zend_jit.c | 10 ++++++---- ext/opcache/jit/zend_jit_internal.h | 3 ++- ext/opcache/jit/zend_jit_vm_helpers.c | 4 ---- ext/opcache/jit/zend_jit_x86.dasc | 18 ++++++++++++++++-- 5 files changed, 35 insertions(+), 15 deletions(-) diff --git a/ext/opcache/ZendAccelerator.c b/ext/opcache/ZendAccelerator.c index a9787eafbe8f7..68375bd9efd1a 100644 --- a/ext/opcache/ZendAccelerator.c +++ b/ext/opcache/ZendAccelerator.c @@ -2406,6 +2406,13 @@ int accel_activate(INIT_FUNC_ARGS) return SUCCESS; } +#ifdef HAVE_JIT +void accel_deactivate(void) +{ + zend_jit_deactivate(); +} +#endif + int accel_post_deactivate(void) { if (ZCG(cwd)) { @@ -2417,10 +2424,6 @@ int accel_post_deactivate(void) return SUCCESS; } -#ifdef HAVE_JIT - zend_jit_deactivate(); -#endif - zend_shared_alloc_safe_unlock(); /* be sure we didn't leave cache locked */ accel_unlock_all(); ZCG(counted) = 0; @@ -4259,7 +4262,11 @@ ZEND_EXT_API zend_extension zend_extension_entry = { accel_startup, /* startup */ NULL, /* shutdown */ NULL, /* per-script activation */ +#ifdef HAVE_JIT + accel_deactivate, /* per-script deactivation */ +#else NULL, /* per-script deactivation */ +#endif NULL, /* message handler */ NULL, /* op_array handler */ NULL, /* extended statement handler */ diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 71cbc7d5024e9..cf98ab4793d8c 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -2722,6 +2722,7 @@ void zend_jit_check_funcs(HashTable *function_table, zend_bool is_method) { zend_op *opline; zend_function *func; zend_op_array *op_array; + zend_ulong counter; ZEND_HASH_REVERSE_FOREACH_PTR(function_table, func) { if (func->type == ZEND_INTERNAL_FUNCTION) { @@ -2733,7 +2734,10 @@ void zend_jit_check_funcs(HashTable *function_table, zend_bool is_method) { opline++; } if (opline->handler == zend_jit_profile_jit_handler) { - zend_ulong counter = (zend_ulong)ZEND_COUNTER_INFO(op_array); + if (!RUN_TIME_CACHE(op_array)) { + continue; + } + counter = (zend_ulong)ZEND_COUNTER_INFO(op_array); ZEND_COUNTER_INFO(op_array) = 0; opline->handler = ZEND_FUNC_INFO(op_array); ZEND_SET_FUNC_INFO(op_array, NULL); @@ -3104,9 +3108,7 @@ ZEND_EXT_API int zend_jit_startup(zend_long jit, size_t size) } if (zend_jit_trigger == ZEND_JIT_ON_PROF_REQUEST) { - zend_extension dummy; - zend_jit_profile_counter_rid = zend_get_resource_handle(&dummy); - ZEND_ASSERT(zend_jit_profile_counter_rid != -1); + zend_jit_profile_counter_rid = zend_get_op_array_extension_handle(); } #ifdef HAVE_GDB diff --git a/ext/opcache/jit/zend_jit_internal.h b/ext/opcache/jit/zend_jit_internal.h index bd42bdcc64cf9..1252850ee041e 100644 --- a/ext/opcache/jit/zend_jit_internal.h +++ b/ext/opcache/jit/zend_jit_internal.h @@ -24,7 +24,8 @@ extern zend_ulong zend_jit_profile_counter; extern int zend_jit_profile_counter_rid; -#define ZEND_COUNTER_INFO(op_array) ((op_array)->reserved[zend_jit_profile_counter_rid]) +#define ZEND_COUNTER_INFO(op_array) \ + ZEND_OP_ARRAY_EXTENSION(op_array, zend_jit_profile_counter_rid) /* Hot Counters */ #define ZEND_HOT_COUNTERS_COUNT 128 diff --git a/ext/opcache/jit/zend_jit_vm_helpers.c b/ext/opcache/jit/zend_jit_vm_helpers.c index 9d543b90edfd7..0adaa3ce9e617 100644 --- a/ext/opcache/jit/zend_jit_vm_helpers.c +++ b/ext/opcache/jit/zend_jit_vm_helpers.c @@ -166,11 +166,7 @@ ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_profile_helper(ZEND_OPCODE_HANDLE { zend_op_array *op_array = (zend_op_array*)EX(func); zend_vm_opcode_handler_t handler = (zend_vm_opcode_handler_t)ZEND_FUNC_INFO(op_array); -#ifndef _WIN32 ++(ZEND_COUNTER_INFO(op_array)); -#else - ++((char*)ZEND_COUNTER_INFO(op_array)); -#endif ++zend_jit_profile_counter; ZEND_OPCODE_TAIL_CALL(handler); } diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index ae934c87fe862..b72bb4ff2fd9a 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -2047,8 +2047,22 @@ static int zend_jit_hybrid_profile_jit_stub(dasm_State **Dst) | .endif | // op_array = (zend_op_array*)EX(func); | mov r0, EX->func - | // ++(ZEND_COUNTER_INFO(op_array)); - | inc aword [r0 + offsetof(zend_op_array, reserved[zend_jit_profile_counter_rid])] + | // ++ZEND_COUNTER_INFO(op_array) + | mov r2, aword [r0 + offsetof(zend_op_array, run_time_cache__ptr)] +#if ZEND_MAP_PTR_KIND == ZEND_MAP_PTR_KIND_PTR + | mov r2, aword [r2] +#elif ZEND_MAP_PTR_KIND == ZEND_MAP_PTR_KIND_PTR_OR_OFFSET + | xor r1, r1 + | test r2, 1 + | jz >1 + | MEM_OP2_2 mov, r1, aword, &CG(map_ptr_base), r1 + | sub r1, 1 + |1: + | mov r2, aword [r1 + r2] +#else +# error "Unknown ZEND_MAP_PTR_KIND" +#endif + | inc aword [r2 + zend_jit_profile_counter_rid * sizeof(void)] | // handler = (const void*)ZEND_FUNC_INFO(op_array); | mov r0, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])] | // return ((zend_vm_opcode_handler_t)handler)(); From b84e926e88a3cde09e568d431b29728d4cd1e921 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 22 Feb 2019 13:07:42 +0300 Subject: [PATCH 558/569] typo --- ext/opcache/jit/zend_jit.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index cf98ab4793d8c..82d9a8cf5b47c 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -3075,7 +3075,7 @@ static int zend_jit_make_stubs(void) zend_jit_runtime_jit_handler = (const void*)zend_runtime_jit; zend_jit_profile_jit_handler = (const void*)zend_jit_profile_helper; zend_jit_func_counter_handler = (const void*)zend_jit_func_counter_helper; - zend_jit_loop_counter_handler = (const void*)zend_jit_loop_counter_handler; + zend_jit_loop_counter_handler = (const void*)zend_jit_loop_counter_helper; } dasm_free(&dasm_state); From 5466d2d215c4837a0760cc5ff467ebb39e150c81 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 25 Feb 2019 15:54:35 +0300 Subject: [PATCH 559/569] Fixed MSVC compatibility, with explicit type casting --- ext/opcache/jit/zend_jit.c | 4 ++-- ext/opcache/jit/zend_jit_vm_helpers.c | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 82d9a8cf5b47c..8563835d20d14 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -2722,7 +2722,7 @@ void zend_jit_check_funcs(HashTable *function_table, zend_bool is_method) { zend_op *opline; zend_function *func; zend_op_array *op_array; - zend_ulong counter; + uintptr_t counter; ZEND_HASH_REVERSE_FOREACH_PTR(function_table, func) { if (func->type == ZEND_INTERNAL_FUNCTION) { @@ -2737,7 +2737,7 @@ void zend_jit_check_funcs(HashTable *function_table, zend_bool is_method) { if (!RUN_TIME_CACHE(op_array)) { continue; } - counter = (zend_ulong)ZEND_COUNTER_INFO(op_array); + counter = (uintptr_t)ZEND_COUNTER_INFO(op_array); ZEND_COUNTER_INFO(op_array) = 0; opline->handler = ZEND_FUNC_INFO(op_array); ZEND_SET_FUNC_INFO(op_array, NULL); diff --git a/ext/opcache/jit/zend_jit_vm_helpers.c b/ext/opcache/jit/zend_jit_vm_helpers.c index 0adaa3ce9e617..a241d6f03a32a 100644 --- a/ext/opcache/jit/zend_jit_vm_helpers.c +++ b/ext/opcache/jit/zend_jit_vm_helpers.c @@ -166,7 +166,9 @@ ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_profile_helper(ZEND_OPCODE_HANDLE { zend_op_array *op_array = (zend_op_array*)EX(func); zend_vm_opcode_handler_t handler = (zend_vm_opcode_handler_t)ZEND_FUNC_INFO(op_array); - ++(ZEND_COUNTER_INFO(op_array)); + uintptr_t counter = (uintptr_t)ZEND_COUNTER_INFO(op_array); + + ZEND_COUNTER_INFO(op_array) = (void*)(counter + 1); ++zend_jit_profile_counter; ZEND_OPCODE_TAIL_CALL(handler); } From 7f1fff82ffd4f34d1cb3349bedd4e28ac7d27771 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 28 Feb 2019 15:22:55 +0300 Subject: [PATCH 560/569] Support for WIN64 (incomplete) --- ext/opcache/config.w32 | 2 +- ext/opcache/jit/Makefile.frag.w32 | 2 +- ext/opcache/jit/zend_jit_disasm_x86.c | 2 +- ext/opcache/jit/zend_jit_x86.dasc | 121 ++++++++++++++++++++------ ext/opcache/jit/zend_jit_x86.h | 25 +++++- 5 files changed, 116 insertions(+), 36 deletions(-) diff --git a/ext/opcache/config.w32 b/ext/opcache/config.w32 index 27941ed3bd402..e6e3016f34d2b 100644 --- a/ext/opcache/config.w32 +++ b/ext/opcache/config.w32 @@ -20,7 +20,7 @@ if (PHP_OPCACHE != "no") { if (PHP_OPCACHE_JIT == "yes") { if (CHECK_HEADER_ADD_INCLUDE("dynasm/dasm_x86.h", "CFLAGS_OPCACHE", PHP_OPCACHE + ";ext\\opcache\\jit")) { - var dasm_flags = X64 ? "-D X64=1" : ""; + var dasm_flags = (X64 ? "-D X64=1" : "") + (X64 ? " -D X64WIN=1" : "") + " -D WIN=1"; DEFINE("DASM_FLAGS", dasm_flags); AC_DEFINE('HAVE_JIT', 1, 'Define to enable JIT'); diff --git a/ext/opcache/jit/Makefile.frag.w32 b/ext/opcache/jit/Makefile.frag.w32 index 0b5ca3d83852d..282f81bb74270 100644 --- a/ext/opcache/jit/Makefile.frag.w32 +++ b/ext/opcache/jit/Makefile.frag.w32 @@ -2,7 +2,7 @@ $(BUILD_DIR)\\minilua.exe: ext\opcache\jit\dynasm\minilua.c @if exist $(BUILD_DIR)\\minilua.exe del $(BUILD_DIR)\\minilua.exe $(PHP_CL) /Fo$(BUILD_DIR)\ /Fd$(BUILD_DIR)\ /Fp$(BUILD_DIR)\ /FR$(BUILD_DIR) /Fe$(BUILD_DIR)\minilua.exe ext\opcache\jit\dynasm\minilua.c -ext\opcache\jit\zend_jit_x86.c: $(BUILD_DIR)\\minilua.exe +ext\opcache\jit\zend_jit_x86.c: ext\opcache\jit\zend_jit_x86.dasc $(BUILD_DIR)\\minilua.exe @if exist ext\opcache\jit\zend_jit_x86.c del ext\opcache\jit\zend_jit_x86.c $(BUILD_DIR)\\minilua.exe ext/opcache/jit/dynasm/dynasm.lua $(DASM_FLAGS) -o $@ ext/opcache/jit/zend_jit_x86.dasc diff --git a/ext/opcache/jit/zend_jit_disasm_x86.c b/ext/opcache/jit/zend_jit_disasm_x86.c index b4432a03e7657..59d147fc92e79 100644 --- a/ext/opcache/jit/zend_jit_disasm_x86.c +++ b/ext/opcache/jit/zend_jit_disasm_x86.c @@ -358,7 +358,7 @@ static int zend_jit_disasm(const char *name, static int zend_jit_disasm_init(void) { ud_init(&ud); -#ifdef __x86_64__ +#if defined(__x86_64__) || defined(_WIN64) ud_set_mode(&ud, 64); #else ud_set_mode(&ud, 32); diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index b72bb4ff2fd9a..36fd0187fd4b1 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -23,7 +23,27 @@ |.arch x86 |.endif -|.if X64 +|.if X64WIN + |.define FP, r14 + |.define IP, r15 + |.define IPl, r15d + |.define RX, r15 // the same as VM IP reused as a general purpos reg + |.define CARG1, rcx // x64/POSIX C call arguments. + |.define CARG2, rdx + |.define CARG3, r8 + |.define CARG4, r9 + |.define CARG1d, ecx + |.define CARG2d, edx + |.define CARG3d, r8d + |.define CARG4d, r9d + |.define FCARG1a, CARG1 // Simulate x86 fastcall. + |.define FCARG2a, CARG2 + |.define FCARG1d, CARG1d + |.define FCARG2d, CARG2d + |.define SPAD, 0x08 // padding for CPU stack alignment + |.define NR_SPAD, 0x18 // padding for CPU stack alignment + |.define SSE, 1 +|.elif X64 |.define FP, r14 |.define IP, r15 |.define IPl, r15d @@ -1429,18 +1449,23 @@ static void* dasm_labels[zend_lb_MAX]; ||#if ZEND_DEBUG || const char *filename = op_array->filename ? op_array->filename->val : NULL; | LOAD_ADDR FCARG2a, filename -| .if X64 +| .if X64WIN +| mov CARG3d, opline->lineno +| xor CARG4, CARG4 +| push 0 +| EXT_CALL _efree, r0 +| add r4, 8 +| .elif X64 | mov CARG3d, opline->lineno | xor CARG4, CARG4 | xor CARG5, CARG5 +| EXT_CALL _efree, r0 | .else | sub r4, 4 | push 0 | push 0 | push opline->lineno -| .endif -| EXT_CALL _efree, r0 -| .if not(X64) +| EXT_CALL _efree, r0 | add r4, 4 | .endif ||#else @@ -1462,18 +1487,23 @@ static void* dasm_labels[zend_lb_MAX]; || const char *filename = op_array->filename ? op_array->filename->val : NULL; | mov FCARG1a, size | LOAD_ADDR FCARG2a, filename -| .if X64 +| .if X64WIN +| mov CARG3d, opline->lineno +| xor CARG4, CARG4 +| push 0 +| EXT_CALL _emalloc, r0 +| add r4, 8 +| .elif X64 | mov CARG3d, opline->lineno | xor CARG4, CARG4 | xor CARG5, CARG5 +| EXT_CALL _emalloc, r0 | .else | sub r4, 4 | push 0 | push 0 | push opline->lineno -| .endif -| EXT_CALL _emalloc, r0 -| .if not(X64) +| EXT_CALL _emalloc, r0 | add r4, 4 | .endif ||#else @@ -6803,9 +6833,9 @@ static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, zen zend_jit_start_reuse_ip(); | // if (UNEXPECTED(used_stack > (size_t)(((char*)EG(vm_stack_end)) - (char*)call))) { - | MEM_OP2_2 mov, RX, aword, &EG(vm_stack_top), r1 + | MEM_OP2_2 mov, RX, aword, &EG(vm_stack_top), RX | // Check Stack Overflow - | MEM_OP2_2 mov, r2, aword, &EG(vm_stack_end), r1 + | MEM_OP2_2 mov, r2, aword, &EG(vm_stack_end), r2 | sub r2, RX if (func) { | cmp r2, used_stack @@ -6830,9 +6860,9 @@ static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, zen |.code if (func) { - | MEM_OP2_1 add, aword, &EG(vm_stack_top), used_stack, r1 + | MEM_OP2_1 add, aword, &EG(vm_stack_top), used_stack, r2 } else { - | MEM_OP2_1 add, aword, &EG(vm_stack_top), FCARG1a, r1 + | MEM_OP2_1 add, aword, &EG(vm_stack_top), FCARG1a, r2 } | // zend_vm_init_call_frame(call, call_info, func, num_args, called_scope, object); | // ZEND_SET_CALL_INFO(call, 0, call_info); @@ -9126,19 +9156,24 @@ static int zend_jit_recv(dasm_State **Dst, const zend_op *opline, zend_op_array | mov r0, EX->run_time_cache | add r0, opline->op2.num | LOAD_ADDR FCARG2a, (ptrdiff_t)op_array - |.if X64 + |.if X64WIN + | mov CARG3, arg_num + | LOAD_ADDR CARG4, (ptrdiff_t)arg_info + | push r0 + | EXT_CALL zend_jit_verify_arg_object, r0 + | add r4, 8 + |.elif X64 | mov CARG3, arg_num | LOAD_ADDR CARG4, (ptrdiff_t)arg_info | mov CARG5, r0 + | EXT_CALL zend_jit_verify_arg_object, r0 |.else | sub r4, 4 | push r0 | push (ptrdiff_t)arg_info | push arg_num - |.endif - | EXT_CALL zend_jit_verify_arg_object, r0 - |.if not(X64) - | add r4, 4 + | EXT_CALL zend_jit_verify_arg_object, r0 + | add r4, 4 |.endif if (!zend_jit_check_exception(Dst)) { return 0; @@ -9153,18 +9188,26 @@ static int zend_jit_recv(dasm_State **Dst, const zend_op *opline, zend_op_array | mov r0, EX->run_time_cache | add r0, opline->op2.num | LOAD_ADDR FCARG2a, (ptrdiff_t)op_array - |.if X64 + |.if X64WIN + | mov CARG3, arg_num + | LOAD_ADDR CARG4, (ptrdiff_t)arg_info + | push 0 + | push r0 + | EXT_CALL zend_jit_verify_arg_slow, r0 + | add r4, 16 + |.elif X64 | mov CARG3, arg_num | LOAD_ADDR CARG4, (ptrdiff_t)arg_info | mov CARG5, r0 | xor CARG6, CARG6 + | EXT_CALL zend_jit_verify_arg_slow, r0 |.else | push 0 | push r0 | push (ptrdiff_t)arg_info | push arg_num + | EXT_CALL zend_jit_verify_arg_slow, r0 |.endif - | EXT_CALL zend_jit_verify_arg_slow, r0 if (!zend_jit_check_exception(Dst)) { return 0; } @@ -9264,19 +9307,24 @@ static int zend_jit_recv_init(dasm_State **Dst, const zend_op *opline, zend_op_a | mov r0, EX->run_time_cache | lea r0, [r0 + opline->extended_value] | LOAD_ADDR FCARG2a, (ptrdiff_t)op_array - |.if X64 + |.if X64WIN + | mov CARG3, arg_num + | LOAD_ADDR CARG4, (ptrdiff_t)arg_info + | push r0 + | EXT_CALL zend_jit_verify_arg_object, r0 + | add r4, 8 + |.elif X64 | mov CARG3, arg_num | LOAD_ADDR CARG4, (ptrdiff_t)arg_info | mov CARG5, r0 + | EXT_CALL zend_jit_verify_arg_object, r0 |.else | sub r4, 4 | push r0 | push (ptrdiff_t)arg_info | push arg_num - |.endif - | EXT_CALL zend_jit_verify_arg_object, r0 - |.if not(X64) - | add r4, 4 + | EXT_CALL zend_jit_verify_arg_object, r0 + | add r4, 4 |.endif } } while (0); @@ -9311,18 +9359,26 @@ static int zend_jit_recv_init(dasm_State **Dst, const zend_op *opline, zend_op_a | mov r0, EX->run_time_cache | lea r0, [r0 + opline->extended_value] | LOAD_ADDR FCARG2a, (ptrdiff_t)op_array - |.if X64 + |.if X64WIN + | mov CARG3, arg_num + | LOAD_ADDR CARG4, (ptrdiff_t)arg_info + | push zv + | push r0 + | EXT_CALL zend_jit_verify_arg_slow, r0 + | add r4, 16 + |.elif X64 | mov CARG3, arg_num | LOAD_ADDR CARG4, (ptrdiff_t)arg_info | mov CARG5, r0 | LOAD_ADDR CARG6, zv + | EXT_CALL zend_jit_verify_arg_slow, r0 |.else | push zv | push r0 | push (ptrdiff_t)arg_info | push arg_num + | EXT_CALL zend_jit_verify_arg_slow, r0 |.endif - | EXT_CALL zend_jit_verify_arg_slow, r0 | jmp <9 } |.code @@ -9583,7 +9639,11 @@ static int zend_jit_fetch_obj_read(dasm_State **Dst, zend_op *opline, zend_op_ar if (offset == ZEND_WRONG_PROPERTY_OFFSET && may_be_dynamic) { |8: | mov FCARG2a, r0 - |.if X64 + |.if X64WIN + | LOAD_ADDR CARG3, member + | LOAD_ZVAL_ADDR CARG4, res_addr + | push (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + |.elif X64 | LOAD_ADDR CARG3, member | LOAD_ZVAL_ADDR CARG4, res_addr | mov CARG5, (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) @@ -9599,7 +9659,10 @@ static int zend_jit_fetch_obj_read(dasm_State **Dst, zend_op *opline, zend_op_ar } else if (opline->opcode == ZEND_FETCH_OBJ_IS) { | EXT_CALL zend_jit_fetch_obj_is_dynamic, r0 } - |.if not(X64) + |.if X64WIN + | add r4, 8 + |.elif X64 + |.else | add r4, 4 |.endif | jmp >9 diff --git a/ext/opcache/jit/zend_jit_x86.h b/ext/opcache/jit/zend_jit_x86.h index 0a54d6498b090..0bd445d22fc7b 100644 --- a/ext/opcache/jit/zend_jit_x86.h +++ b/ext/opcache/jit/zend_jit_x86.h @@ -31,7 +31,7 @@ typedef enum _zend_reg { ZREG_R6, ZREG_R7, -#ifdef __x86_64__ +#if defined(__x86_64__) || defined(_WIN64) ZREG_R8, ZREG_R9, ZREG_R10, @@ -51,7 +51,7 @@ typedef enum _zend_reg { ZREG_XMM6, ZREG_XMM7, -#ifdef __x86_64__ +#if defined(__x86_64__) || defined(_WIN64) ZREG_XMM8, ZREG_XMM9, ZREG_XMM10, @@ -74,7 +74,13 @@ typedef enum _zend_reg { #define ZREG_RSI ZREG_R6 #define ZREG_RDI ZREG_R7 -#ifdef __x86_64__ +#ifdef _WIN64 +# define ZREG_FP ZREG_R14 +# define ZREG_IP ZREG_R15 +# define ZREG_RX ZREG_IP +# define ZREG_FCARG1a ZREG_RCX +# define ZREG_FCARG2a ZREG_RDX +#elif defined(__x86_64__) # define ZREG_FP ZREG_R14 # define ZREG_IP ZREG_R15 # define ZREG_RX ZREG_IP @@ -121,7 +127,18 @@ typedef uint32_t zend_regset; #define ZEND_REGSET_DIFFERENCE(set1, set2) \ ((set1) & ~(set2)) -#ifdef __x86_64__ +#ifdef _WIN64 +# define ZEND_REGSET_FIXED \ + (ZEND_REGSET(ZREG_RSP) | ZEND_REGSET(ZREG_R14) | ZEND_REGSET(ZREG_R15)) +# define ZEND_REGSET_GP \ + ZEND_REGSET_DIFFERENCE(ZEND_REGSET_INTERVAL(ZREG_R0, ZREG_R15), ZEND_REGSET_FIXED) +# define ZEND_REGSET_FP \ + ZEND_REGSET_DIFFERENCE(ZEND_REGSET_INTERVAL(ZREG_XMM0, ZREG_XMM15), ZEND_REGSET_FIXED) +# define ZEND_REGSET_SCRATCH \ + (ZEND_REGSET(ZREG_RAX) | ZEND_REGSET(ZREG_RDX) | ZEND_REGSET(ZREG_RCX) | ZEND_REGSET_INTERVAL(ZREG_R8, ZREG_R11) | ZEND_REGSET_FP) +# define ZEND_REGSET_PRESERVED \ + (ZEND_REGSET(ZREG_RBX) | ZEND_REGSET(ZREG_RBP) | ZEND_REGSET(ZREG_R12) | ZEND_REGSET(ZREG_R13) | ZEND_REGSET(ZREG_RDI) | ZEND_REGSET(ZREG_RSI)) +#elif defined(__x86_64__) # define ZEND_REGSET_FIXED \ (ZEND_REGSET(ZREG_RSP) | ZEND_REGSET(ZREG_R14) | ZEND_REGSET(ZREG_R15)) # define ZEND_REGSET_GP \ From 744192445ea26ca4a19104daf9ee3605c968f71f Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 28 Feb 2019 16:37:25 +0300 Subject: [PATCH 561/569] Fixed size calculation --- ext/opcache/jit/zend_jit.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 8563835d20d14..f2b4e7ece7f29 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -1084,16 +1084,18 @@ static int zend_jit_compute_liveness(zend_op_array *op_array, zend_ssa *ssa, zen int *block_order; zend_ssa_phi *phi; zend_lifetime_interval **intervals; + size_t mem_size; ALLOCA_FLAG(use_heap); set_size = zend_bitset_len(ssa->vars_count); - intervals = do_alloca( + mem_size = ZEND_MM_ALIGNED_SIZE(ssa->vars_count * sizeof(zend_lifetime_interval*)) + - ZEND_MM_ALIGNED_SIZE((set_size * (ssa->cfg.blocks_count + 2)) * ZEND_BITSET_ELM_SIZE) + + ZEND_MM_ALIGNED_SIZE((set_size * ssa->cfg.blocks_count) * ZEND_BITSET_ELM_SIZE) + + ZEND_MM_ALIGNED_SIZE(set_size * ZEND_BITSET_ELM_SIZE) + + ZEND_MM_ALIGNED_SIZE(set_size * ZEND_BITSET_ELM_SIZE) + ZEND_MM_ALIGNED_SIZE(zend_bitset_len(op_array->last) * ZEND_BITSET_ELM_SIZE) + - ZEND_MM_ALIGNED_SIZE(ssa->cfg.blocks_count * sizeof(int)), - use_heap); - + ZEND_MM_ALIGNED_SIZE(ssa->cfg.blocks_count * sizeof(int)); + intervals = do_alloca(mem_size, use_heap); if (!intervals) { *list = NULL; return FAILURE; From 678cd533bc8cbf9b7b02e7f176c39deec63b9e76 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 1 Mar 2019 03:10:09 +0300 Subject: [PATCH 562/569] Support for WIN64 --- ext/opcache/jit/zend_jit_x86.dasc | 199 ++++++++++++++++++------------ 1 file changed, 117 insertions(+), 82 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 36fd0187fd4b1..c629454619288 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -41,8 +41,13 @@ |.define FCARG1d, CARG1d |.define FCARG2d, CARG2d |.define SPAD, 0x08 // padding for CPU stack alignment - |.define NR_SPAD, 0x18 // padding for CPU stack alignment + |.define NR_SPAD, 0x58 // padding for CPU stack alignment |.define SSE, 1 + |.define T3, [r4+0x50] + |.define T2, [r4+0x48] + |.define T1, [r4+0x40] + |.define A6, [r4+0x28] + |.define A5, [r4+0x20] |.elif X64 |.define FP, r14 |.define IP, r15 @@ -67,6 +72,9 @@ |.define SPAD, 0x08 // padding for CPU stack alignment |.define NR_SPAD, 0x18 // padding for CPU stack alignment |.define SSE, 1 + |.define T3, [r4+0x10] + |.define T2, [r4+0x08] + |.define T1, [r4] |.else |.define FP, esi |.define IP, edi @@ -79,6 +87,9 @@ |.define SPAD, 12 // padding for CPU stack alignment |.define NR_SPAD, 12 // padding for CPU stack alignment |.define SSE, 1 + |.define T3, [r4+0x10] + |.define T2, [r4+0x08] + |.define T1, [r4] |.endif |.define HYBRID_SPAD, 16 // padding for stack alignment @@ -1390,13 +1401,13 @@ static void* dasm_labels[zend_lb_MAX]; | GC_DELREF r0 |1: || if (Z_REG(addr) == ZREG_FCARG1a) { -| mov aword [r4], FCARG1a // save +| mov aword T1, FCARG1a // save || } else { | LOAD_ZVAL_ADDR FCARG1a, addr || } | EXT_CALL zval_copy_ctor_func, r0 || if (Z_REG(addr) == ZREG_FCARG1a) { -| mov FCARG1a, aword [r4] // restore +| mov FCARG1a, aword T1 // restore || } || if (cold) { | jmp >2 @@ -1426,13 +1437,13 @@ static void* dasm_labels[zend_lb_MAX]; | GC_DELREF Ra(tmp_reg) |1: || if (Z_REG(addr) == ZREG_FCARG1a) { -| mov aword [r4], FCARG1a // save +| mov aword T1, FCARG1a // save || } else { | LOAD_ZVAL_ADDR FCARG1a, addr || } | EXT_CALL zval_copy_ctor_func, r0 || if (Z_REG(addr) == ZREG_FCARG1a) { -| mov FCARG1a, aword [r4] // restore +| mov FCARG1a, aword T1 // restore || } || if (RC_MAY_BE_1(op_info)) { || if (cold) { @@ -1452,9 +1463,8 @@ static void* dasm_labels[zend_lb_MAX]; | .if X64WIN | mov CARG3d, opline->lineno | xor CARG4, CARG4 -| push 0 +| mov aword A5, 0 | EXT_CALL _efree, r0 -| add r4, 8 | .elif X64 | mov CARG3d, opline->lineno | xor CARG4, CARG4 @@ -1490,9 +1500,8 @@ static void* dasm_labels[zend_lb_MAX]; | .if X64WIN | mov CARG3d, opline->lineno | xor CARG4, CARG4 -| push 0 +| mov aword A5, 0 | EXT_CALL _emalloc, r0 -| add r4, 8 | .elif X64 | mov CARG3d, opline->lineno | xor CARG4, CARG4 @@ -1658,8 +1667,8 @@ static int zend_jit_interrupt_handler_stub(dasm_State **Dst) | add r4, SPAD // stack alignment | JMP_IP } else { - | mov FP, aword [r4 + sizeof(void*)] // restore FP - | mov RX, aword [r4 + sizeof(void*) * 2] // restore IP + | mov FP, aword T2 // restore FP + | mov RX, aword T3 // restore IP | add r4, NR_SPAD // stack alignment | mov r0, 1 // ZEND_VM_ENTER | ret @@ -1684,8 +1693,8 @@ static int zend_jit_exception_handler_stub(dasm_State **Dst) | add r4, SPAD // stack alignment } else { | mov FCARG1a, FP - | mov FP, aword [r4 + sizeof(void*)] // restore FP - | mov RX, aword [r4 + sizeof(void*) * 2] // restore IP + | mov FP, aword T2 // restore FP + | mov RX, aword T3 // restore IP | add r4, NR_SPAD // stack alignment } | EXT_JMP handler, r0 @@ -1721,8 +1730,8 @@ static int zend_jit_leave_function_stub(dasm_State **Dst) | add r4, SPAD } else { | mov FCARG2a, FP - | mov FP, aword [r4 + sizeof(void*)] // restore FP - | mov RX, aword [r4 + sizeof(void*) * 2] // restore IP + | mov FP, aword T2 // restore FP + | mov RX, aword T3 // restore IP | add r4, NR_SPAD } | test FCARG1d, ZEND_CALL_TOP @@ -1758,8 +1767,8 @@ static int zend_jit_leave_throw_stub(dasm_State **Dst) |5: | // opline = EG(exception_op); | LOAD_IP_ADDR &EG(exception_op) - | mov FP, aword [r4 + sizeof(void*)] // restore FP - | mov RX, aword [r4 + sizeof(void*) * 2] // restore IP + | mov FP, aword T2 // restore FP + | mov RX, aword T3 // restore IP | add r4, NR_SPAD // stack alignment | mov r0, 2 // ZEND_VM_LEAVE | ret @@ -1839,7 +1848,9 @@ static int zend_jit_undefined_offset_ex_stub(dasm_State **Dst) static int zend_jit_undefined_offset_stub(dasm_State **Dst) { |->undefined_offset: - |.if X64 + |.if X64WIN + | sub r4, 0x28 + |.elif X64 | sub r4, 8 |.else | sub r4, 12 @@ -1868,7 +1879,13 @@ static int zend_jit_undefined_offset_stub(dasm_State **Dst) |.endif | add r0, FP |3: - |.if X64 + |.if X64WIN + | mov CARG1, E_NOTICE + | LOAD_ADDR CARG2, "Undefined offset: " ZEND_LONG_FMT + | mov CARG3, aword [r0] + | EXT_CALL zend_error, r0 + | add r4, 0x28 // stack alignment + |.elif X64 | mov CARG1, E_NOTICE | LOAD_ADDR CARG2, "Undefined offset: " ZEND_LONG_FMT | mov CARG3, aword [r0] @@ -1899,7 +1916,9 @@ static int zend_jit_undefined_index_ex_stub(dasm_State **Dst) static int zend_jit_undefined_index_stub(dasm_State **Dst) { |->undefined_index: - |.if X64 + |.if X64WIN + | sub r4, 0x28 + |.elif X64 | sub r4, 8 |.else | sub r4, 12 @@ -1928,7 +1947,14 @@ static int zend_jit_undefined_index_stub(dasm_State **Dst) |.endif | add r0, FP |3: - |.if X64 + |.if X64WIN + | mov CARG1, E_NOTICE + | LOAD_ADDR CARG2, "Undefined index: %s" + | mov CARG3, aword [r0] + | add CARG3, offsetof(zend_string, val) + | EXT_CALL zend_error, r0 + | add r4, 0x28 + |.elif X64 | mov CARG1, E_NOTICE | LOAD_ADDR CARG2, "Undefined index: %s" | mov CARG3, aword [r0] @@ -1962,7 +1988,9 @@ static int zend_jit_cannot_add_element_ex_stub(dasm_State **Dst) static int zend_jit_cannot_add_element_stub(dasm_State **Dst) { |->cannot_add_element: - |.if X64 + |.if X64WIN + | sub r4, 0x28 + |.elif X64 | sub r4, 8 |.else | sub r4, 12 @@ -1977,7 +2005,12 @@ static int zend_jit_cannot_add_element_stub(dasm_State **Dst) |.endif | SET_Z_TYPE_INFO FP + r0, IS_NULL |1: - |.if X64 + |.if X64WIN + | mov CARG1, E_WARNING + | LOAD_ADDR CARG2, "Cannot add element to the array as the next element is already occupied" + | EXT_CALL zend_error, r0 + | add r4, 0x28 + |.elif X64 | mov CARG1, E_WARNING | LOAD_ADDR CARG2, "Cannot add element to the array as the next element is already occupied" | EXT_CALL zend_error, r0 @@ -2325,8 +2358,8 @@ static int zend_jit_prologue(dasm_State **Dst) | sub r4, SPAD // stack alignment } else { | sub r4, NR_SPAD // stack alignment - | mov aword [r4 + sizeof(void*)], FP // save FP - | mov aword [r4 + sizeof(void*) * 2], RX // save IP + | mov aword T2, FP // save FP + | mov aword T3, RX // save IP | mov FP, FCARG1a } return 1; @@ -2497,8 +2530,8 @@ static int zend_jit_tail_handler(dasm_State **Dst, const zend_op *opline) | add r4, SPAD // stack alignment } else { | mov FCARG1a, FP - | mov FP, aword [r4 + sizeof(void*)] // restore FP - | mov RX, aword [r4 + sizeof(void*) * 2] // restore IP + | mov FP, aword T2 // restore FP + | mov RX, aword T3 // restore IP | add r4, NR_SPAD // stack alignment } | EXT_JMP handler, r0 @@ -4347,13 +4380,13 @@ static int zend_jit_simple_assign(dasm_State **Dst, } | // zend_error(E_NOTICE, "Undefined variable: %s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); if (Z_REG(var_addr) != ZREG_FP) { - | mov aword [r4], Ra(Z_REG(var_addr)) // save + | mov aword T1, Ra(Z_REG(var_addr)) // save } | SAVE_VALID_OPLINE opline | mov FCARG1d, val.var | EXT_CALL zend_jit_undefined_op_helper, r0 if (Z_REG(var_addr) != ZREG_FP) { - | mov Ra(Z_REG(var_addr)), aword [r4] // restore + | mov Ra(Z_REG(var_addr)), aword T1 // restore } | SET_ZVAL_TYPE_INFO var_addr, IS_NULL if (res_addr) { @@ -4500,11 +4533,11 @@ static int zend_jit_assign_to_variable(dasm_State **Dst, if (RC_MAY_BE_N(var_info)) { | jnz >4 } - | mov aword [r4], r0 // save + | mov aword T1, r0 // save if (!zend_jit_simple_assign(Dst, opline, op_array, ssa, var_addr, var_info, val_type, val, val_addr, val_info, res_addr, in_cold)) { return 0; } - | mov FCARG1a, aword [r4] // restore + | mov FCARG1a, aword T1 // restore if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { | cmp dword [FCARG1a], 0 | jnz >8 @@ -4520,16 +4553,16 @@ static int zend_jit_assign_to_variable(dasm_State **Dst, } else if (Z_REG(var_addr) != ZREG_FCARG1a) { | GET_ZVAL_PTR FCARG1a, var_addr | IF_GC_MAY_NOT_LEAK FCARG1a, eax, >5 - | mov [r4], Ra(Z_REG(var_addr)) // save + | mov T1, Ra(Z_REG(var_addr)) // save } else { | GET_ZVAL_PTR r0, var_addr | IF_GC_MAY_NOT_LEAK r0, eax, >5 - | mov [r4], Ra(Z_REG(var_addr)) // save + | mov T1, Ra(Z_REG(var_addr)) // save | GET_ZVAL_PTR FCARG1a, var_addr } | EXT_CALL gc_possible_root, r0 if (Z_REG(var_addr) != ZREG_FP) { - | mov Ra(Z_REG(var_addr)), [r4] // restore + | mov Ra(Z_REG(var_addr)), T1 // restore } if (in_cold) { | jmp >5 @@ -4594,11 +4627,11 @@ static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, zend_op_ } | // ZVAL_ARR(container, zend_new_array(8)); if (Z_REG(op1_addr) != ZREG_FP) { - | mov [r4], Ra(Z_REG(op1_addr)) // save + | mov T1, Ra(Z_REG(op1_addr)) // save } | EXT_CALL _zend_new_array_0, r0 if (Z_REG(op1_addr) != ZREG_FP) { - | mov Ra(Z_REG(op1_addr)), [r4] // restore + | mov Ra(Z_REG(op1_addr)), T1 // restore } | SET_ZVAL_LVAL op1_addr, r0 | SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX @@ -4671,11 +4704,11 @@ static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, zend_op_ } | // ZVAL_ARR(container, zend_new_array(8)); if (Z_REG(op1_addr) != ZREG_FP) { - | mov [r4], Ra(Z_REG(op1_addr)) // save + | mov T1, Ra(Z_REG(op1_addr)) // save } | EXT_CALL _zend_new_array_0, r0 if (Z_REG(op1_addr) != ZREG_FP) { - | mov Ra(Z_REG(op1_addr)), [r4] // restore + | mov Ra(Z_REG(op1_addr)), T1 // restore } | SET_ZVAL_LVAL op1_addr, r0 | SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX @@ -4814,11 +4847,11 @@ static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, zend_ } | // ZVAL_ARR(container, zend_new_array(8)); if (Z_REG(op1_addr) != ZREG_FP) { - | mov [r4], Ra(Z_REG(op1_addr)) // save + | mov T1, Ra(Z_REG(op1_addr)) // save } | EXT_CALL _zend_new_array_0, r0 if (Z_REG(op1_addr) != ZREG_FP) { - | mov Ra(Z_REG(op1_addr)), [r4] // restore + | mov Ra(Z_REG(op1_addr)), T1 // restore } | SET_ZVAL_LVAL op1_addr, r0 | SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX @@ -4915,11 +4948,11 @@ static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, zend_ } | // ZVAL_ARR(container, zend_new_array(8)); if (Z_REG(op1_addr) != ZREG_FP) { - | mov [r4], Ra(Z_REG(op1_addr)) // save + | mov T1, Ra(Z_REG(op1_addr)) // save } | EXT_CALL _zend_new_array_0, r0 if (Z_REG(op1_addr) != ZREG_FP) { - | mov Ra(Z_REG(op1_addr)), [r4] // restore + | mov Ra(Z_REG(op1_addr)), T1 // restore } | SET_ZVAL_LVAL op1_addr, r0 | SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX @@ -5956,10 +5989,10 @@ static int zend_jit_cmp(dasm_State **Dst, const zend_op *opline, int b, int *opn } if (opline->op2_type == IS_CV && (op2_info & MAY_BE_UNDEF)) { | IF_NOT_ZVAL_TYPE op2_addr, IS_UNDEF, >1 - | mov [r4], FCARG2a // save + | mov T1, FCARG2a // save | mov FCARG1a, opline->op2.var | EXT_CALL zend_jit_undefined_op_helper, r0 - | mov FCARG2a, [r4] // restore + | mov FCARG2a, T1 // restore |.if X64 | LOAD_ADDR CARG3, &EG(uninitialized_zval) |.else @@ -6091,13 +6124,13 @@ static int zend_jit_identical(dasm_State **Dst, const zend_op *opline, int b, in |1: | // zend_error(E_NOTICE, "Undefined variable: %s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); | SAVE_VALID_OPLINE opline - | mov aword [r4], FCARG1a // save + | mov aword T1, FCARG1a // save | mov FCARG1d, opline->op2.var | EXT_CALL zend_jit_undefined_op_helper, r0 if (zend_may_throw(opline, op_array, ssa)) { zend_jit_check_exception_undef_result(Dst, opline); } - | mov FCARG1a, aword [r4] // restore + | mov FCARG1a, aword T1 // restore | LOAD_ADDR FCARG2a, &EG(uninitialized_zval) | jmp >1 |.code @@ -6320,12 +6353,12 @@ static int zend_jit_identical(dasm_State **Dst, const zend_op *opline, int b, in (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) || ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) && (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)))) { - | mov aword [r4], r0 // save + | mov aword T1, r0 // save | SAVE_VALID_OPLINE opline | FREE_OP opline->op1_type, opline->op1, op1_info, 1, op_array, opline | FREE_OP opline->op2_type, opline->op2, op2_info, 1, op_array, opline zend_jit_check_exception_undef_result(Dst, opline); - | mov r0, aword [r4] // restore + | mov r0, aword T1 // restore } | test r0, r0 if (smart_branch) { @@ -6695,9 +6728,9 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, int b, | GET_ZVAL_PTR FCARG1a, op1_addr | GC_DELREF FCARG1a | jnz >3 - | mov aword [r4], r0 // save + | mov aword T1, r0 // save | ZVAL_DTOR_FUNC op1_info, opline - | mov r0, aword [r4] // restore + | mov r0, aword T1 // restore |3: } if (zend_may_throw(opline, op_array, ssa)) { @@ -7114,8 +7147,13 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar if (RETURN_VALUE_USED(opline)) { res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1); } else { +#ifdef _WIN64 + /* Reuse reserved arguments stack */ + res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R4, 0x20); +#else /* CPU stack alocated temorary zval */ res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R4, 8); +#endif } if (info) { @@ -7391,8 +7429,8 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar | add r4, SPAD // stack alignment | JMP_IP } else { - | mov FP, aword [r4 + sizeof(void*)] // restore FP - | mov RX, aword [r4 + sizeof(void*) * 2] // restore IP + | mov FP, aword T2 // restore FP + | mov RX, aword T3 // restore IP | add r4, NR_SPAD // stack alignment | mov r0, 1 // ZEND_VM_ENTER | ret @@ -7439,7 +7477,9 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar } if (!RETURN_VALUE_USED(opline)) { + |.if not(X64WIN) | sub r4, 16 /* alloca() */ + |.endif } | // ZVAL_NULL(EX_VAR(opline->result.var)); @@ -7540,7 +7580,9 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar if (func_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { | ZVAL_PTR_DTOR res_addr, func_info, 1, 1, 0, opline } + |.if not(X64WIN) | add r4, 16 /* revert alloca() */ + |.endif } | // if (UNEXPECTED(EG(exception) != NULL)) { @@ -7764,7 +7806,7 @@ static int zend_jit_send_ref(dasm_State **Dst, const zend_op *opline, zend_op_ar |2: | // ZVAL_NEW_REF(arg, varptr); if (opline->op1_type == IS_VAR) { - | mov aword [r4], r0 // save + | mov aword T1, r0 // save } | EMALLOC sizeof(zend_reference), op_array, opline | mov dword [r0], 2 @@ -7774,7 +7816,7 @@ static int zend_jit_send_ref(dasm_State **Dst, const zend_op *opline, zend_op_ar if (opline->op1_type == IS_VAR) { zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R1, 0); - | mov r1, aword [r4] // restore + | mov r1, aword T1 // restore | ZVAL_COPY_VALUE ref_addr, -1, val_addr, op1_info, ZREG_R2, ZREG_R2 | SET_ZVAL_PTR val_addr, r0 | SET_ZVAL_TYPE_INFO val_addr, IS_REFERENCE_EX @@ -8200,10 +8242,10 @@ static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, int b, i } else { | mov al, byte [FP + opline->op1.var + 8] } - | mov byte [r4], al // save + | mov byte T1, al // save | // zval_dtor_func(r); | ZVAL_DTOR_FUNC op1_info, opline - | mov cl, byte [r4] // restore + | mov cl, byte T1 // restore |jmp >2 } if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { @@ -8277,10 +8319,10 @@ static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, int b, i } else { | mov al, byte [FP + opline->op1.var + 8] } - | mov byte [r4], al // save + | mov byte T1, al // save | // zval_dtor_func(r); | ZVAL_DTOR_FUNC op1_info, opline - | mov cl, byte [r4] // restore + | mov cl, byte T1 // restore |jmp >2 } if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { @@ -8412,7 +8454,7 @@ static int zend_jit_leave_func(dasm_State **Dst, const zend_op *opline, zend_op_ if (op_array->scope || (op_array->fn_flags & ZEND_ACC_CLOSURE)) { | // EG(current_execute_data) = EX(prev_execute_data); | mov r0, EX->prev_execute_data - | MEM_OP2_1 mov, aword, &EG(current_execute_data), r0, r1 + | MEM_OP2_1 mov, aword, &EG(current_execute_data), r0, r2 if (op_array->scope) { | // if (call_info & ZEND_CALL_RELEASE_THIS) | test FCARG1d, ZEND_CALL_RELEASE_THIS @@ -8500,8 +8542,8 @@ static int zend_jit_leave_func(dasm_State **Dst, const zend_op *opline, zend_op_ | JMP_IP #endif } else { - | mov FP, aword [r4 + sizeof(void*)] // restore FP - | mov RX, aword [r4 + sizeof(void*) * 2] // restore IP + | mov FP, aword T2 // restore FP + | mov RX, aword T3 // restore IP | add r4, NR_SPAD // stack alignment | mov r0, 2 // ZEND_VM_LEAVE | ret @@ -9001,7 +9043,7 @@ static int zend_jit_bind_global(dasm_State **Dst, const zend_op *opline, zend_op | jne >9 //EXPECTED(memcmp(ZSTR_VAL(p->key), Z_STRVAL_P(varname), Z_STRLEN_P(varname)) == 0) | add r1, offsetof(zend_string, val) - | mov [r4], r0 + | mov T1, r0 |.if X64 | mov CARG1, r1 | LOAD_ADDR CARG2, Z_STRVAL_P(varname) @@ -9016,7 +9058,7 @@ static int zend_jit_bind_global(dasm_State **Dst, const zend_op *opline, zend_op | add r4, 16 |.endif | test al, al - | mov r0, aword [r4] + | mov r0, aword T1 | jnz >9 | jmp >2 |.code @@ -9062,17 +9104,17 @@ static int zend_jit_bind_global(dasm_State **Dst, const zend_op *opline, zend_op if (op1_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) { | jnz >3 } - | mov aword [r4], r0 // save + | mov aword T1, r0 // save | ZVAL_DTOR_FUNC op1_info, opline - | mov r0, aword [r4] // restore + | mov r0, aword T1 // restore | jmp >5 if (op1_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) { |3: // GC_ZVAL_CHECK_POSSIBLE_ROOT(variable_ptr) | IF_GC_MAY_NOT_LEAK FCARG1a, edx, >5 - | mov aword [r4], r0 //save + | mov aword T1, r0 //save | EXT_CALL gc_possible_root, r1 - | mov r0, aword [r4] // restore + | mov r0, aword T1 // restore | jmp >5 } |4: @@ -9159,9 +9201,8 @@ static int zend_jit_recv(dasm_State **Dst, const zend_op *opline, zend_op_array |.if X64WIN | mov CARG3, arg_num | LOAD_ADDR CARG4, (ptrdiff_t)arg_info - | push r0 + | mov aword A5, r0 | EXT_CALL zend_jit_verify_arg_object, r0 - | add r4, 8 |.elif X64 | mov CARG3, arg_num | LOAD_ADDR CARG4, (ptrdiff_t)arg_info @@ -9191,10 +9232,9 @@ static int zend_jit_recv(dasm_State **Dst, const zend_op *opline, zend_op_array |.if X64WIN | mov CARG3, arg_num | LOAD_ADDR CARG4, (ptrdiff_t)arg_info - | push 0 - | push r0 + | mov aword A5, r0 + | mov aword A6, 0 | EXT_CALL zend_jit_verify_arg_slow, r0 - | add r4, 16 |.elif X64 | mov CARG3, arg_num | LOAD_ADDR CARG4, (ptrdiff_t)arg_info @@ -9310,9 +9350,8 @@ static int zend_jit_recv_init(dasm_State **Dst, const zend_op *opline, zend_op_a |.if X64WIN | mov CARG3, arg_num | LOAD_ADDR CARG4, (ptrdiff_t)arg_info - | push r0 + | mov aword A5, r0 | EXT_CALL zend_jit_verify_arg_object, r0 - | add r4, 8 |.elif X64 | mov CARG3, arg_num | LOAD_ADDR CARG4, (ptrdiff_t)arg_info @@ -9362,10 +9401,9 @@ static int zend_jit_recv_init(dasm_State **Dst, const zend_op *opline, zend_op_a |.if X64WIN | mov CARG3, arg_num | LOAD_ADDR CARG4, (ptrdiff_t)arg_info - | push zv - | push r0 + | mov aword A5, r0 + | ADDR_OP2_2 mov, aword A6, zv, r0 | EXT_CALL zend_jit_verify_arg_slow, r0 - | add r4, 16 |.elif X64 | mov CARG3, arg_num | LOAD_ADDR CARG4, (ptrdiff_t)arg_info @@ -9642,7 +9680,7 @@ static int zend_jit_fetch_obj_read(dasm_State **Dst, zend_op *opline, zend_op_ar |.if X64WIN | LOAD_ADDR CARG3, member | LOAD_ZVAL_ADDR CARG4, res_addr - | push (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + | mov aword A5, (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) |.elif X64 | LOAD_ADDR CARG3, member | LOAD_ZVAL_ADDR CARG4, res_addr @@ -9659,10 +9697,7 @@ static int zend_jit_fetch_obj_read(dasm_State **Dst, zend_op *opline, zend_op_ar } else if (opline->opcode == ZEND_FETCH_OBJ_IS) { | EXT_CALL zend_jit_fetch_obj_is_dynamic, r0 } - |.if X64WIN - | add r4, 8 - |.elif X64 - |.else + |.if not(X64) | add r4, 4 |.endif | jmp >9 From 65215c367a51e45c2dd32a30e521c0427241a85d Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 1 Mar 2019 03:24:15 +0300 Subject: [PATCH 563/569] typo --- ext/opcache/jit/zend_jit.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index f2b4e7ece7f29..cde623a6cab11 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -2035,10 +2035,9 @@ static void zend_calc_checked_this_r(zend_bitset checked_this, zend_op_array *op default: break; } - opline++; } - if (cfg->blocks[b].flags && ZEND_BB_TRY) { + if (cfg->blocks[b].flags & ZEND_BB_TRY) { checked = old_checked; } From 1c6d1bf7de6e1a6d7143d4a05bc57b2c9e846778 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 5 Mar 2019 16:45:48 +0300 Subject: [PATCH 564/569] JIT support for Windows --- ext/opcache/ZendAccelerator.c | 41 ++++++- ext/opcache/jit/zend_jit.c | 193 +++++++++++------------------- ext/opcache/jit/zend_jit.h | 2 +- ext/opcache/jit/zend_jit_x86.dasc | 32 +++++ ext/opcache/shared_alloc_mmap.c | 62 +++++++--- ext/opcache/shared_alloc_win32.c | 30 ++++- ext/opcache/zend_shared_alloc.c | 31 +++-- ext/opcache/zend_shared_alloc.h | 7 +- 8 files changed, 241 insertions(+), 157 deletions(-) diff --git a/ext/opcache/ZendAccelerator.c b/ext/opcache/ZendAccelerator.c index 77afda8010491..033be463daa2f 100644 --- a/ext/opcache/ZendAccelerator.c +++ b/ext/opcache/ZendAccelerator.c @@ -2833,7 +2833,38 @@ static int accel_post_startup(void) /********************************************/ file_cache_only = ZCG(accel_directives).file_cache_only; if (!file_cache_only) { - switch (zend_shared_alloc_startup(ZCG(accel_directives).memory_consumption)) { + size_t shm_size = ZCG(accel_directives).memory_consumption; +#ifdef HAVE_JIT + size_t jit_size = 0; + zend_bool reattached = 0; + + if (ZCG(accel_directives).jit && + ZCG(accel_directives).jit_buffer_size) { + size_t page_size; + +# ifdef _WIN32 + SYSTEM_INFO system_info; + GetSystemInfo(&system_info); + page_size = system_info.dwPageSize; +# else + page_size = getpagesize(); +# endif + if (!page_size || (page_size & (page_size - 1))) { + zend_accel_error(ACCEL_LOG_FATAL, "Failure to initialize shared memory structures - can't get page size."); + abort(); + } + jit_size = ZCG(accel_directives).jit_buffer_size; + jit_size = ZEND_MM_ALIGNED_SIZE_EX(jit_size, page_size); + shm_size += jit_size; + } else { + ZCG(accel_directives).jit = 0; + ZCG(accel_directives).jit_buffer_size = 0; + } + + switch (zend_shared_alloc_startup(shm_size, jit_size)) { +#else + switch (zend_shared_alloc_startup(shm_size, 0)) { +#endif case ALLOC_SUCCESS: if (zend_accel_init_shm() == FAILURE) { accel_startup_ok = 0; @@ -2845,6 +2876,9 @@ static int accel_post_startup(void) zend_accel_error(ACCEL_LOG_FATAL, "Failure to initialize shared memory structures - probably not enough shared memory."); return SUCCESS; case SUCCESSFULLY_REATTACHED: +#ifdef HAVE_JIT + reattached = 1; +#endif zend_shared_alloc_lock(); accel_shared_globals = (zend_accel_shared_globals *) ZSMMG(app_shared_globals); zend_interned_strings_set_request_storage_handlers(accel_new_interned_string_for_php, accel_init_interned_string_for_php); @@ -2878,8 +2912,9 @@ static int accel_post_startup(void) zend_shared_alloc_lock(); #ifdef HAVE_JIT if (ZCG(accel_directives).jit && - ZCG(accel_directives).jit_buffer_size) { - zend_jit_startup(ZCG(accel_directives).jit, ZCG(accel_directives).jit_buffer_size); + ZCG(accel_directives).jit_buffer_size && + ZSMMG(reserved)) { + zend_jit_startup(ZCG(accel_directives).jit, ZSMMG(reserved), jit_size, reattached); } else { ZCG(accel_directives).jit = 0; ZCG(accel_directives).jit_buffer_size = 0; diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index cde623a6cab11..194e7386d00a6 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -35,7 +35,6 @@ #include "Optimizer/zend_dump.h" //#define CONTEXT_THREADED_JIT -#define PREFER_MAP_32BIT #define ZEND_JIT_USE_RC_INFERENCE #ifdef ZEND_JIT_USE_RC_INFERENCE @@ -80,6 +79,8 @@ static void *dasm_buf = NULL; static void *dasm_end = NULL; static void **dasm_ptr = NULL; +static size_t dasm_size = 0; + static const void *zend_jit_runtime_jit_handler = NULL; static const void *zend_jit_profile_jit_handler = NULL; static const void *zend_jit_func_counter_handler = NULL; @@ -385,97 +386,6 @@ static void *dasm_link_and_encode(dasm_State **dasm_state, return entry; } -static size_t jit_page_size(void) -{ -#ifdef _WIN32 - SYSTEM_INFO system_info; - GetSystemInfo(&system_info); - return system_info.dwPageSize; -#else - return getpagesize(); -#endif -} - -static void *jit_alloc(size_t size, int shared) -{ -#ifdef _WIN32 - return VirtualAlloc(0, size, MEM_COMMIT | MEM_RESERVE, PAGE_EXECUTE_READWRITE); -#else - void *p; - int prot; -# ifdef MAP_HUGETLB - size_t huge_page_size = 2 * 1024 * 1024; -# endif - -# ifdef HAVE_MPROTECT - if (ZCG(accel_directives).jit_debug & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP)) { - prot = PROT_EXEC | PROT_READ | PROT_WRITE; - } else { - prot = PROT_NONE; - } -# else - prot = PROT_EXEC | PROT_READ | PROT_WRITE; -# endif - - shared = shared? MAP_SHARED : MAP_PRIVATE; - -# ifdef MAP_HUGETLB - if (size >= huge_page_size && size % huge_page_size == 0) { -# if defined(PREFER_MAP_32BIT) && defined(__x86_64__) && defined(MAP_32BIT) - void *p2; - /* to got HUGE PAGES in low 32-bit address we have to reseve address - space and then remap it using MAP_HUGETLB */ - p = mmap(NULL, size, prot, shared | MAP_ANONYMOUS | MAP_32BIT, -1, 0); - if (p != MAP_FAILED) { - munmap(p, size); - p = (void*)(ZEND_MM_ALIGNED_SIZE_EX((ptrdiff_t)p, huge_page_size)); - p2 = mmap(p, size, prot, shared | MAP_ANONYMOUS | MAP_HUGETLB | MAP_FIXED, -1, 0); - if (p2 != MAP_FAILED) { - return p2; - } else { - p = mmap(NULL, size, prot, shared | MAP_ANONYMOUS | MAP_32BIT, -1, 0); - if (p != MAP_FAILED) { - return p; - } - } - } -# endif - p = mmap(NULL, size, prot, shared | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0); - if (p != MAP_FAILED) { - return p; - } -# if defined(PREFER_MAP_32BIT) && defined(__x86_64__) && defined(MAP_32BIT) - } else { - p = mmap(NULL, size, prot, shared | MAP_ANONYMOUS | MAP_32BIT, -1, 0); - if (p != MAP_FAILED) { - return p; - } -# endif - } -# elif defined(PREFER_MAP_32BIT) && defined(__x86_64__) && defined(MAP_32BIT) - p = mmap(NULL, size, prot, shared | MAP_ANONYMOUS | MAP_32BIT, -1, 0); - if (p != MAP_FAILED) { - return p; - } -# endif - p = mmap(NULL, size, prot, shared | MAP_ANONYMOUS, -1, 0); - if (p == MAP_FAILED) { - return NULL; - } - - return (void*)p; -#endif -} - -static void jit_free(void *p, size_t size) -{ -#ifdef _WIN32 - VirtualFree(p, 0, MEM_RELEASE); -#else - munmap(p, size); -#endif -} - static int zend_may_overflow(const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) { uint32_t num; @@ -3000,10 +2910,18 @@ ZEND_EXT_API void zend_jit_unprotect(void) { #ifdef HAVE_MPROTECT if (!(ZCG(accel_directives).jit_debug & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP))) { - if (mprotect(dasm_buf, ((char*)dasm_end) - ((char*)dasm_buf), PROT_READ | PROT_WRITE) != 0) { + if (mprotect(dasm_buf, dasm_size, PROT_READ | PROT_WRITE) != 0) { fprintf(stderr, "mprotect() failed [%d] %s\n", errno, strerror(errno)); } } +#elif _WIN32 + if (!(ZCG(accel_directives).jit_debug & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP))) { + DWORD old; + + if (!VirtualProtect(dasm_buf, dasm_size, PAGE_READWRITE, &old)) { + fprintf(stderr, "VirtualProtect() failed\n"); + } + } #endif } @@ -3011,10 +2929,18 @@ ZEND_EXT_API void zend_jit_protect(void) { #ifdef HAVE_MPROTECT if (!(ZCG(accel_directives).jit_debug & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP))) { - if (mprotect(dasm_buf, ((char*)dasm_end) - ((char*)dasm_buf), PROT_READ | PROT_EXEC) != 0) { + if (mprotect(dasm_buf, dasm_size, PROT_READ | PROT_EXEC) != 0) { fprintf(stderr, "mprotect() failed [%d] %s\n", errno, strerror(errno)); } } +#elif _WIN32 + if (!(ZCG(accel_directives).jit_debug & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP))) { + DWORD old; + + if (!VirtualProtect(dasm_buf, dasm_size, PAGE_EXECUTE_READ, &old)) { + fprintf(stderr, "VirtualProtect() failed\n"); + } + } #endif } @@ -3083,10 +3009,8 @@ static int zend_jit_make_stubs(void) return 1; } -ZEND_EXT_API int zend_jit_startup(zend_long jit, size_t size) +ZEND_EXT_API int zend_jit_startup(zend_long jit, void *buf, size_t size, zend_bool reattached) { - size_t page_size = jit_page_size(); - int shared = 1; int ret; zend_jit_level = ZEND_JIT_LEVEL(jit); @@ -3098,7 +3022,7 @@ ZEND_EXT_API int zend_jit_startup(zend_long jit, size_t size) if (zend_jit_vm_kind != ZEND_VM_KIND_CALL && zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) { // TODO: error reporting and cleanup ??? - return FAILURE; + return FAILURE; } zend_jit_halt_op = zend_get_halt_op(); @@ -3118,7 +3042,6 @@ ZEND_EXT_API int zend_jit_startup(zend_long jit, size_t size) #ifdef HAVE_OPROFILE if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_OPROFILE) { - shared = 0; if (!zend_jit_oprofile_startup()) { // TODO: error reporting and cleanup ??? return FAILURE; @@ -3126,25 +3049,45 @@ ZEND_EXT_API int zend_jit_startup(zend_long jit, size_t size) } #endif - /* Round up to the page size, which should be a power of two. */ - page_size = jit_page_size(); + dasm_buf = buf; + dasm_size = size; - if (!page_size || (page_size & (page_size - 1))) { - abort(); +#ifdef HAVE_MPROTECT + if (ZCG(accel_directives).jit_debug & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP)) { + if (mprotect(dasm_buf, dasm_size, PROT_READ | PROT_WRITE | PROT_EXEC) != 0) { + fprintf(stderr, "mprotect() failed [%d] %s\n", errno, strerror(errno)); + } + } else { + if (mprotect(dasm_buf, dasm_size, PROT_READ | PROT_EXEC) != 0) { + fprintf(stderr, "mprotect() failed [%d] %s\n", errno, strerror(errno)); + } } +#elif _WIN32 + if (ZCG(accel_directives).jit_debug & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP)) { + DWORD old; - size = ZEND_MM_ALIGNED_SIZE_EX(size, page_size); - - dasm_buf = jit_alloc(size, shared); + if (!VirtualProtect(dasm_buf, dasm_size, PAGE_EXECUTE_READWRITE, &old)) { + fprintf(stderr, "VirtualProtect() failed\n"); + } + } else { + DWORD old; - if (!dasm_buf) { - return FAILURE; + if (!VirtualProtect(dasm_buf, dasm_size, PAGE_EXECUTE_READ, &old)) { + fprintf(stderr, "VirtualProtect() failed\n"); + } } +#endif dasm_ptr = dasm_end = (void*)(((char*)dasm_buf) + size - sizeof(*dasm_ptr)); - zend_jit_unprotect(); - *dasm_ptr = dasm_buf; - zend_jit_protect(); + if (!reattached) { + zend_jit_unprotect(); + *dasm_ptr = dasm_buf; +#if _WIN32 + /* reserve space for global labels */ + *dasm_ptr = (void**)*dasm_ptr + zend_lb_MAX; +#endif + zend_jit_protect(); + } #ifdef HAVE_DISASM if (ZCG(accel_directives).jit_debug & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_ASM_STUBS)) { @@ -3161,13 +3104,23 @@ ZEND_EXT_API int zend_jit_startup(zend_long jit, size_t size) } #endif - zend_jit_unprotect(); - ret = zend_jit_make_stubs(); - zend_jit_protect(); - - if (!ret) { - // TODO: error reporting and cleanup ??? - return FAILURE; + if (!reattached) { + zend_jit_unprotect(); + ret = zend_jit_make_stubs(); +#if _WIN32 + /* save global labels */ + memcpy(dasm_buf, dasm_labels, sizeof(void*) * zend_lb_MAX); +#endif + zend_jit_protect(); + if (!ret) { + // TODO: error reporting and cleanup ??? + return FAILURE; + } + } else { +#if _WIN32 + /* restore global labels */ + memcpy(dasm_labels, dasm_buf, sizeof(void*) * zend_lb_MAX); +#endif } return SUCCESS; @@ -3198,10 +3151,6 @@ ZEND_EXT_API void zend_jit_shutdown(void) zend_jit_perf_jitdump_close(); } #endif - - if (dasm_buf) { - jit_free(dasm_buf, ((char*)dasm_end) - ((char*)dasm_buf)); - } } ZEND_EXT_API void zend_jit_activate(void) diff --git a/ext/opcache/jit/zend_jit.h b/ext/opcache/jit/zend_jit.h index ab43ac6889399..fa65ed78408d7 100644 --- a/ext/opcache/jit/zend_jit.h +++ b/ext/opcache/jit/zend_jit.h @@ -79,7 +79,7 @@ ZEND_EXT_API int zend_jit_op_array(zend_op_array *op_array, zend_script *script ZEND_EXT_API int zend_jit_script(zend_script *script); ZEND_EXT_API void zend_jit_unprotect(void); ZEND_EXT_API void zend_jit_protect(void); -ZEND_EXT_API int zend_jit_startup(zend_long jit, size_t size); +ZEND_EXT_API int zend_jit_startup(zend_long jit, void *jit_buffer, size_t size, zend_bool reattached); ZEND_EXT_API void zend_jit_shutdown(void); ZEND_EXT_API void zend_jit_activate(void); ZEND_EXT_API void zend_jit_deactivate(void); diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index c629454619288..86151b417e97c 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -2050,13 +2050,23 @@ static int zend_jit_negative_shift_stub(dasm_State **Dst) |->negative_shift: | SAVE_OPLINE |.if X64 + |.if WIN + | LOAD_ADDR CARG1, &zend_ce_arithmetic_error + | mov CARG1, aword [CARG1] + |.else | LOAD_ADDR CARG1, zend_ce_arithmetic_error + |.endif | LOAD_ADDR CARG2, "Bit shift by negative number" | EXT_CALL zend_throw_error, r0 |.else | sub r4, 8 | push "Bit shift by negative number" + |.if WIN + | LOAD_ADDR r0, &zend_ce_arithmetic_error + | push aword [r0] + |.else | PUSH_ADDR zend_ce_arithmetic_error, r0 + |.endif | EXT_CALL zend_throw_error, r0 | add r4, 16 |.endif @@ -2069,13 +2079,23 @@ static int zend_jit_mod_by_zero_stub(dasm_State **Dst) |->mod_by_zero: | SAVE_OPLINE |.if X64 + |.if WIN + | LOAD_ADDR CARG1, &zend_ce_division_by_zero_error + | mov CARG1, aword [CARG1] + |.else | LOAD_ADDR CARG1, zend_ce_division_by_zero_error + |.endif | LOAD_ADDR CARG2, "Modulo by zero" | EXT_CALL zend_throw_error, r0 |.else | sub r4, 8 | push "Modulo by zero" + |.if WIN + | LOAD_ADDR r0, &zend_ce_division_by_zero_error + | push aword [r0] + |.else | PUSH_ADDR zend_ce_division_by_zero_error, r0 + |.endif | EXT_CALL zend_throw_error, r0 | add r4, 16 |.endif @@ -6882,7 +6902,11 @@ static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, zen if (func) { | mov FCARG1d, used_stack } +#ifdef _WIN32 + if (0) { +#else if (func && func->type == ZEND_INTERNAL_FUNCTION) { +#endif | EXT_CALL zend_jit_int_extend_stack_helper, r0 } else { | mov FCARG2a, r0 @@ -6901,7 +6925,11 @@ static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, zen | // ZEND_SET_CALL_INFO(call, 0, call_info); | mov dword EX:RX->This.u1.type_info, (IS_UNDEF | (ZEND_CALL_NESTED_FUNCTION << ZEND_CALL_INFO_SHIFT)) | // call->func = func; +#ifdef _WIN32 + if (0) { +#else if (func && func->type == ZEND_INTERNAL_FUNCTION) { +#endif |1: | ADDR_OP2_2 mov, aword EX:RX->func, func, r1 } else { @@ -7047,7 +7075,11 @@ static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, uint32_t } } +#ifdef _WIN32 + if (0) { +#else if (func && func->type == ZEND_INTERNAL_FUNCTION) { +#endif /* load constant address later */ } else if (func && op_array == &func->op_array) { /* recursive call */ diff --git a/ext/opcache/shared_alloc_mmap.c b/ext/opcache/shared_alloc_mmap.c index dc02d038f5a5b..266fbebd030a9 100644 --- a/ext/opcache/shared_alloc_mmap.c +++ b/ext/opcache/shared_alloc_mmap.c @@ -39,17 +39,10 @@ static int create_segments(size_t requested_size, zend_shared_segment ***shared_segments_p, int *shared_segments_count, char **error_in) { zend_shared_segment *shared_segment; - - *shared_segments_count = 1; - *shared_segments_p = (zend_shared_segment **) calloc(1, sizeof(zend_shared_segment) + sizeof(void *)); - if (!*shared_segments_p) { - *error_in = "calloc"; - return ALLOC_FAILURE; - } - shared_segment = (zend_shared_segment *)((char *)(*shared_segments_p) + sizeof(void *)); - (*shared_segments_p)[0] = shared_segment; - + void *p; #ifdef MAP_HUGETLB + size_t huge_page_size = 2 * 1024 * 1024; + /* Try to allocate huge pages first to reduce dTLB misses. * OSes has to be configured properly * on Linux @@ -60,21 +53,56 @@ static int create_segments(size_t requested_size, zend_shared_segment ***shared_ * sysctl vm.pmap.pg_ps_enabled entry * (boot time config only, but enabled by default on most arches). */ - shared_segment->p = mmap(0, requested_size, PROT_READ | PROT_WRITE, MAP_SHARED|MAP_ANONYMOUS|MAP_HUGETLB, -1, 0); - if (shared_segment->p != MAP_FAILED) { - shared_segment->pos = 0; - shared_segment->size = requested_size; + if (requested_size >= huge_page_size && requested_size % huge_page_size == 0) { +# if defined(__x86_64__) && defined(MAP_32BIT) + /* to got HUGE PAGES in low 32-bit address we have to reserve address + space and then remap it using MAP_HUGETLB */ - return ALLOC_SUCCESS; + p = mmap(NULL, requested_size, PROT_READ | PROT_WRITE, MAP_SHARED|MAP_ANONYMOUS|MAP_32BIT, -1, 0); + if (p != MAP_FAILED) { + munmap(p, requested_size); + p = (void*)(ZEND_MM_ALIGNED_SIZE_EX((ptrdiff_t)p, huge_page_size)); + p = mmap(p, requested_size, PROT_READ | PROT_WRITE, MAP_SHARED|MAP_ANONYMOUS|MAP_32BIT|MAP_HUGETLB|MAP_FIXED, -1, 0); + if (p != MAP_FAILED) { + goto success; + } else { + p = mmap(NULL, requested_size, PROT_READ | PROT_WRITE, MAP_SHARED|MAP_ANONYMOUS|MAP_32BIT, -1, 0); + if (p != MAP_FAILED) { + goto success; + } + } + } +# endif + p = mmap(0, requested_size, PROT_READ | PROT_WRITE, MAP_SHARED|MAP_ANONYMOUS|MAP_HUGETLB, -1, 0); + if (p != MAP_FAILED) { + goto success; + } + } +#elif defined(PREFER_MAP_32BIT) && defined(__x86_64__) && defined(MAP_32BIT) + p = mmap(NULL, requested_size, PROT_READ | PROT_WRITE, MAP_SHARED|MAP_ANONYMOUS|MAP_32BIT, -1, 0); + if (p != MAP_FAILED) { + goto success; } #endif - shared_segment->p = mmap(0, requested_size, PROT_READ | PROT_WRITE, MAP_SHARED|MAP_ANONYMOUS, -1, 0); - if (shared_segment->p == MAP_FAILED) { + p = mmap(0, requested_size, PROT_READ | PROT_WRITE, MAP_SHARED|MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { *error_in = "mmap"; return ALLOC_FAILURE; } +success: + *shared_segments_count = 1; + *shared_segments_p = (zend_shared_segment **) calloc(1, sizeof(zend_shared_segment) + sizeof(void *)); + if (!*shared_segments_p) { + munmap(p, requested_size); + *error_in = "calloc"; + return ALLOC_FAILURE; + } + shared_segment = (zend_shared_segment *)((char *)(*shared_segments_p) + sizeof(void *)); + (*shared_segments_p)[0] = shared_segment; + + shared_segment->p = p; shared_segment->pos = 0; shared_segment->size = requested_size; diff --git a/ext/opcache/shared_alloc_win32.c b/ext/opcache/shared_alloc_win32.c index 27f3d6cd13c47..76533cc4f4da8 100644 --- a/ext/opcache/shared_alloc_win32.c +++ b/ext/opcache/shared_alloc_win32.c @@ -186,7 +186,7 @@ static int zend_shared_alloc_reattach(size_t requested_size, char **error_in) return ALLOC_FAILURE; } - mapping_base = MapViewOfFileEx(memfile, FILE_MAP_ALL_ACCESS, 0, 0, 0, wanted_mapping_base); + mapping_base = MapViewOfFileEx(memfile, FILE_MAP_ALL_ACCESS|FILE_MAP_EXECUTE, 0, 0, 0, wanted_mapping_base); if (mapping_base == NULL) { err = GetLastError(); @@ -195,7 +195,16 @@ static int zend_shared_alloc_reattach(size_t requested_size, char **error_in) return ALLOC_FAILURE; } return ALLOC_FAIL_MAPPING; + } else { + DWORD old; + + if (!VirtualProtect(mapping_base, requested_size, PAGE_READWRITE, &old)) { + err = GetLastError(); + zend_win_error_message(ACCEL_LOG_FATAL, "VirtualProtect() failed", err); + return ALLOC_FAIL_MAPPING; + } } + smm_shared_globals = (zend_smm_shared_globals *) mapping_base; return SUCCESSFULLY_REATTACHED; @@ -226,7 +235,7 @@ static int create_segments(size_t requested_size, zend_shared_segment ***shared_ can be called before the child process is killed. In this case, the map will fail and we have to sleep some time (until the child releases the mapping object) and retry.*/ do { - memfile = OpenFileMapping(FILE_MAP_WRITE, 0, create_name_with_username(ACCEL_FILEMAP_NAME)); + memfile = OpenFileMapping(FILE_MAP_READ|FILE_MAP_WRITE|FILE_MAP_EXECUTE, 0, create_name_with_username(ACCEL_FILEMAP_NAME)); if (memfile == NULL) { err = GetLastError(); break; @@ -270,7 +279,7 @@ static int create_segments(size_t requested_size, zend_shared_segment ***shared_ shared_segment = (zend_shared_segment *)((char *)(*shared_segments_p) + sizeof(void *)); (*shared_segments_p)[0] = shared_segment; - memfile = CreateFileMapping(INVALID_HANDLE_VALUE, NULL, PAGE_READWRITE, size_high, size_low, + memfile = CreateFileMapping(INVALID_HANDLE_VALUE, NULL, PAGE_EXECUTE_READWRITE | SEC_COMMIT, size_high, size_low, create_name_with_username(ACCEL_FILEMAP_NAME)); if (memfile == NULL) { err = GetLastError(); @@ -300,7 +309,7 @@ static int create_segments(size_t requested_size, zend_shared_segment ***shared_ } do { - shared_segment->p = mapping_base = MapViewOfFileEx(memfile, FILE_MAP_ALL_ACCESS, 0, 0, 0, *wanted_mapping_base); + shared_segment->p = mapping_base = MapViewOfFileEx(memfile, FILE_MAP_ALL_ACCESS|FILE_MAP_EXECUTE, 0, 0, 0, *wanted_mapping_base); if (*wanted_mapping_base == NULL) { /* Auto address (NULL) is the last option on the array */ break; } @@ -314,8 +323,17 @@ static int create_segments(size_t requested_size, zend_shared_segment ***shared_ *error_in = "MapViewOfFile"; return ALLOC_FAILURE; } else { - char *mmap_base_file = get_mmap_base_file(); - FILE *fp = fopen(mmap_base_file, "w"); + char *mmap_base_file; + FILE *fp; + DWORD old; + + if (!VirtualProtect(mapping_base, requested_size, PAGE_READWRITE, &old)) { + err = GetLastError(); + zend_win_error_message(ACCEL_LOG_FATAL, "VirtualProtect() failed", err); + return ALLOC_FAILURE; + } + mmap_base_file = get_mmap_base_file(); + fp = fopen(mmap_base_file, "w"); if (!fp) { err = GetLastError(); zend_shared_alloc_unlock_win32(); diff --git a/ext/opcache/zend_shared_alloc.c b/ext/opcache/zend_shared_alloc.c index 177cbcc46fffe..b441392993c46 100644 --- a/ext/opcache/zend_shared_alloc.c +++ b/ext/opcache/zend_shared_alloc.c @@ -145,7 +145,7 @@ static int zend_shared_alloc_try(const zend_shared_memory_handler_entry *he, siz return ALLOC_FAILURE; } -int zend_shared_alloc_startup(size_t requested_size) +int zend_shared_alloc_startup(size_t requested_size, size_t reserved_size) { zend_shared_segment **tmp_shared_segments; size_t shared_segments_array_size; @@ -153,13 +153,13 @@ int zend_shared_alloc_startup(size_t requested_size) char *error_in = NULL; const zend_shared_memory_handler_entry *he; int res = ALLOC_FAILURE; - + int i; /* shared_free must be valid before we call zend_shared_alloc() * - make it temporarily point to a local variable */ smm_shared_globals = &tmp_shared_globals; - ZSMMG(shared_free) = requested_size; /* goes to tmp_shared_globals.shared_free */ + ZSMMG(shared_free) = requested_size - reserved_size; /* goes to tmp_shared_globals.shared_free */ #ifndef ZEND_WIN32 zend_shared_alloc_create_lock(ZCG(accel_directives).lockfile_path); @@ -220,10 +220,15 @@ int zend_shared_alloc_startup(size_t requested_size) } #endif + for (i = 0; i < ZSMMG(shared_segments_count); i++) { + ZSMMG(shared_segments)[i]->end = ZSMMG(shared_segments)[i]->size; + } + shared_segments_array_size = ZSMMG(shared_segments_count) * S_H(segment_type_size)(); /* move shared_segments and shared_free to shared memory */ ZCG(locked) = 1; /* no need to perform a real lock at this point */ + p_tmp_shared_globals = (zend_smm_shared_globals *) zend_shared_alloc(sizeof(zend_smm_shared_globals)); if (!p_tmp_shared_globals) { zend_accel_error(ACCEL_LOG_FATAL, "Insufficient shared memory!"); @@ -251,6 +256,18 @@ int zend_shared_alloc_startup(size_t requested_size) return ALLOC_FAILURE; } + if (reserved_size) { + i = ZSMMG(shared_segments_count) - 1; + if (ZSMMG(shared_segments)[i]->size - ZSMMG(shared_segments)[i]->pos >= reserved_size) { + ZSMMG(shared_segments)[i]->end = ZSMMG(shared_segments)[i]->size - reserved_size; + ZSMMG(reserved) = (char*)ZSMMG(shared_segments)[i]->p + ZSMMG(shared_segments)[i]->end; + ZSMMG(reserved_size) = reserved_size; + } else { + zend_accel_error(ACCEL_LOG_FATAL, "Insufficient shared memory!"); + return ALLOC_FAILURE; + } + } + ZCG(locked) = 0; return res; @@ -294,7 +311,7 @@ static size_t zend_shared_alloc_get_largest_free_block(void) size_t largest_block_size = 0; for (i = 0; i < ZSMMG(shared_segments_count); i++) { - size_t block_size = ZSMMG(shared_segments)[i]->size - ZSMMG(shared_segments)[i]->pos; + size_t block_size = ZSMMG(shared_segments)[i]->end - ZSMMG(shared_segments)[i]->pos; if (block_size>largest_block_size) { largest_block_size = block_size; @@ -327,7 +344,7 @@ void *zend_shared_alloc(size_t size) return NULL; } for (i = 0; i < ZSMMG(shared_segments_count); i++) { - if (ZSMMG(shared_segments)[i]->size - ZSMMG(shared_segments)[i]->pos >= block_size) { /* found a valid block */ + if (ZSMMG(shared_segments)[i]->end - ZSMMG(shared_segments)[i]->pos >= block_size) { /* found a valid block */ void *retval = (void *) (((char *) ZSMMG(shared_segments)[i]->p) + ZSMMG(shared_segments)[i]->pos); ZSMMG(shared_segments)[i]->pos += block_size; @@ -592,7 +609,7 @@ void zend_accel_shared_protect(int mode) } for (i = 0; i < ZSMMG(shared_segments_count); i++) { - mprotect(ZSMMG(shared_segments)[i]->p, ZSMMG(shared_segments)[i]->size, mode); + mprotect(ZSMMG(shared_segments)[i]->p, ZSMMG(shared_segments)[i]->end, mode); } #endif } @@ -607,7 +624,7 @@ int zend_accel_in_shm(void *ptr) for (i = 0; i < ZSMMG(shared_segments_count); i++) { if ((char*)ptr >= (char*)ZSMMG(shared_segments)[i]->p && - (char*)ptr < (char*)ZSMMG(shared_segments)[i]->p + ZSMMG(shared_segments)[i]->size) { + (char*)ptr < (char*)ZSMMG(shared_segments)[i]->p + ZSMMG(shared_segments)[i]->end) { return 1; } } diff --git a/ext/opcache/zend_shared_alloc.h b/ext/opcache/zend_shared_alloc.h index 2a77dfbef7dbe..1dbc88d42ec9c 100644 --- a/ext/opcache/zend_shared_alloc.h +++ b/ext/opcache/zend_shared_alloc.h @@ -75,6 +75,7 @@ typedef struct _zend_shared_segment { size_t size; + size_t end; size_t pos; /* position for simple stack allocator */ void *p; } zend_shared_segment; @@ -113,6 +114,9 @@ typedef struct _zend_smm_shared_globals { zend_shared_memory_state shared_memory_state; /* Pointer to the application's shared data structures */ void *app_shared_globals; + /* Reserved shared memory */ + void *reserved; + size_t reserved_size; } zend_smm_shared_globals; extern zend_smm_shared_globals *smm_shared_globals; @@ -121,10 +125,11 @@ extern zend_smm_shared_globals *smm_shared_globals; #define SHARED_ALLOC_REATTACHED (SUCCESS+1) -int zend_shared_alloc_startup(size_t requested_size); +int zend_shared_alloc_startup(size_t requested_size, size_t reserved_size); void zend_shared_alloc_shutdown(void); /* allocate shared memory block */ +void *zend_shared_alloc_pages(size_t requested_size); void *zend_shared_alloc(size_t size); /* copy into shared memory */ From 5ebafb65795a6b8da2421133c01d98f4b037f0e7 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 13 Mar 2019 12:24:06 +0300 Subject: [PATCH 565/569] Fixed ELF symbol type and binding checks --- ext/opcache/jit/zend_elf.c | 6 ++++-- ext/opcache/jit/zend_elf.h | 19 ++++++++++++++----- ext/opcache/jit/zend_jit_gdb.c | 4 ++-- 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/ext/opcache/jit/zend_elf.c b/ext/opcache/jit/zend_elf.c index 1ad1354f0364d..1bd7c6ea2c444 100644 --- a/ext/opcache/jit/zend_elf.c +++ b/ext/opcache/jit/zend_elf.c @@ -68,8 +68,10 @@ void zend_elf_load_symbols(void) && (str_tbl = (char*)zend_elf_read_sect(fd, §)) != NULL) { for (n = 0; n < count; n++) { if (syms[n].name - && (syms[n].info & ELFSYM_TYPE_FUNC) - && !(syms[n].info & ELFSYM_BIND_GLOBAL)) { + && (ELFSYM_TYPE(syms[n].info) == ELFSYM_TYPE_FUNC + /*|| ELFSYM_TYPE(syms[n].info) == ELFSYM_TYPE_DATA*/) + && (ELFSYM_BIND(syms[n].info) == ELFSYM_BIND_LOCAL + /*|| ELFSYM_BIND(syms[n].info) == ELFSYM_BIND_GLOBAL*/)) { zend_jit_disasm_add_symbol(str_tbl + syms[n].name, syms[n].value, syms[n].size); } } diff --git a/ext/opcache/jit/zend_elf.h b/ext/opcache/jit/zend_elf.h index 2c7e01ae12b69..5d7d27225a8fc 100644 --- a/ext/opcache/jit/zend_elf.h +++ b/ext/opcache/jit/zend_elf.h @@ -72,9 +72,10 @@ enum { ELFSECT_TYPE_DYNSYM = 11, }; -#define ELFSECT_FLAGS_WRITE 1 -#define ELFSECT_FLAGS_ALLOC 2 -#define ELFSECT_FLAGS_EXEC 4 +#define ELFSECT_FLAGS_WRITE (1 << 0) +#define ELFSECT_FLAGS_ALLOC (1 << 1) +#define ELFSECT_FLAGS_EXEC (1 << 2) +#define ELFSECT_FLAGS_TLS (1 << 10) typedef struct zend_elf_symbol { #ifdef ELF64 @@ -94,11 +95,19 @@ typedef struct zend_elf_symbol { #endif } zend_elf_symbol; +#define ELFSYM_BIND(info) ((info) >> 4) +#define ELFSYM_TYPE(info) ((info) & 0xf) +#define ELFSYM_INFO(bind, type) (((bind) << 4) | (type)) + enum { + ELFSYM_TYPE_DATA = 2, ELFSYM_TYPE_FUNC = 2, ELFSYM_TYPE_FILE = 4, - ELFSYM_BIND_LOCAL = 0 << 4, - ELFSYM_BIND_GLOBAL = 1 << 4, +}; + +enum { + ELFSYM_BIND_LOCAL = 0, + ELFSYM_BIND_GLOBAL = 1, }; void zend_elf_load_symbols(void); diff --git a/ext/opcache/jit/zend_jit_gdb.c b/ext/opcache/jit/zend_jit_gdb.c index a554bc20d9456..f2a600ff770d3 100644 --- a/ext/opcache/jit/zend_jit_gdb.c +++ b/ext/opcache/jit/zend_jit_gdb.c @@ -266,14 +266,14 @@ static void zend_gdbjit_symtab(zend_gdbjit_ctx *ctx) sym = &ctx->obj.sym[GDBJIT_SYM_FILE]; sym->name = zend_gdbjit_strz(ctx, "JIT code"); sym->sectidx = ELFSECT_IDX_ABS; - sym->info = ELFSYM_TYPE_FILE|ELFSYM_BIND_LOCAL; + sym->info = ELFSYM_INFO(ELFSYM_BIND_LOCAL, ELFSYM_TYPE_FILE); sym = &ctx->obj.sym[GDBJIT_SYM_FUNC]; sym->name = zend_gdbjit_strz(ctx, ctx->name); sym->sectidx = GDBJIT_SECT_text; sym->value = 0; sym->size = ctx->szmcode; - sym->info = ELFSYM_TYPE_FUNC|ELFSYM_BIND_GLOBAL; + sym->info = ELFSYM_INFO(ELFSYM_BIND_GLOBAL, ELFSYM_TYPE_FUNC); } #define SECTALIGN(p, a) \ From f8c9f47914dcbca7fd8042745c36b874219e419d Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 13 Mar 2019 12:54:57 +0300 Subject: [PATCH 566/569] ZTS detection --- ext/opcache/config.m4 | 4 ++++ ext/opcache/config.w32 | 3 +++ 2 files changed, 7 insertions(+) diff --git a/ext/opcache/config.m4 b/ext/opcache/config.m4 index 91d52a1478748..386dbf816c5bb 100644 --- a/ext/opcache/config.m4 +++ b/ext/opcache/config.m4 @@ -41,6 +41,10 @@ if test "$PHP_OPCACHE" != "no"; then fi rm -rf conftest* + if test "$enable_maintainer_zts" = "yes"; then + DASM_FLAGS="$DASM_FLAGS -D ZTS=1" + fi + AC_MSG_CHECKING(for opagent in default path) for i in /usr/local /usr; do if test -r $i/include/opagent.h; then diff --git a/ext/opcache/config.w32 b/ext/opcache/config.w32 index e6e3016f34d2b..965467fc58358 100644 --- a/ext/opcache/config.w32 +++ b/ext/opcache/config.w32 @@ -21,6 +21,9 @@ if (PHP_OPCACHE != "no") { if (PHP_OPCACHE_JIT == "yes") { if (CHECK_HEADER_ADD_INCLUDE("dynasm/dasm_x86.h", "CFLAGS_OPCACHE", PHP_OPCACHE + ";ext\\opcache\\jit")) { var dasm_flags = (X64 ? "-D X64=1" : "") + (X64 ? " -D X64WIN=1" : "") + " -D WIN=1"; + if (PHP_ZTS == "yes") { + dasm_flags += " -D ZTS=1"; + } DEFINE("DASM_FLAGS", dasm_flags); AC_DEFINE('HAVE_JIT', 1, 'Define to enable JIT'); From 801617a47250e3e708c12b1473592abf3e8e4d34 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 14 Mar 2019 18:45:39 +0300 Subject: [PATCH 567/569] JIT ZTS support (incomplete) --- TSRM/TSRM.c | 14 +++ TSRM/TSRM.h | 1 + ext/opcache/jit/zend_jit_x86.dasc | 169 +++++++++++++++++++++--------- 3 files changed, 135 insertions(+), 49 deletions(-) diff --git a/TSRM/TSRM.c b/TSRM/TSRM.c index 670c260678d42..72f26a8d394fd 100644 --- a/TSRM/TSRM.c +++ b/TSRM/TSRM.c @@ -866,6 +866,20 @@ TSRM_API void *tsrm_get_ls_cache(void) return tsrm_tls_get(); }/*}}}*/ +/* Returns offset of tsrm_ls_cache slot from Thread Control Block address */ +TSRM_API size_t tsrm_get_ls_cache_tcb_offset(void) +{/*{{{*/ +#if (defined(__i386__) || defined(__x86_64__)) && defined(__GNUC__) + size_t ret; + + asm ("mov $_tsrm_ls_cache@tpoff,%0" + : "=r" (ret)); + return ret; +#else + return 0; +#endif +}/*}}}*/ + TSRM_API uint8_t tsrm_is_main_thread(void) {/*{{{*/ return in_main_thread; diff --git a/TSRM/TSRM.h b/TSRM/TSRM.h index a14e31e1f00e0..f006c7490706f 100644 --- a/TSRM/TSRM.h +++ b/TSRM/TSRM.h @@ -154,6 +154,7 @@ TSRM_API void *tsrm_set_interpreter_context(void *new_ctx); TSRM_API void tsrm_free_interpreter_context(void *context); TSRM_API void *tsrm_get_ls_cache(void); +TSRM_API size_t tsrm_get_ls_cache_tcb_offset(void); TSRM_API uint8_t tsrm_is_main_thread(void); TSRM_API const char *tsrm_api_name(void); diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 86151b417e97c..b01e6f2a73b7f 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -123,6 +123,10 @@ const char* zend_reg_name[] = { static uint32_t zend_jit_x86_flags = 0; +#if ZTS +static size_t tsrm_ls_cache_tcb_offset; +#endif + |.type EX, zend_execute_data, FP |.type OP, zend_op |.type ZVAL, zval @@ -152,6 +156,25 @@ static void* dasm_labels[zend_lb_MAX]; | .endif |.endmacro +|.macro LOAD_TSRM_CACHE, reg +| .if X64 +| fs +| mov reg, aword [tsrm_ls_cache_tcb_offset] // mov reg, aword fs:tsrm_ls_cache_tcb_offset +| .else +| gs +| mov reg, aword [tsrm_ls_cache_tcb_offset] // mov reg, aword gs:tsrm_ls_cache_tcb_offset +| .endif +|.endmacro + +|.macro LOAD_ADDR_ZTS, reg, struct, field +| .if ZTS +| LOAD_TSRM_CACHE reg +| lea reg, aword [reg + (struct.._offset + offsetof(zend_..struct, field))] +| .else +| LOAD_ADDR reg, &struct.field +| .endif +|.endmacro + |.macro SAVE_OPLINE || if (GCC_GLOBAL_REGS) { | mov aword EX->opline, IP @@ -173,6 +196,21 @@ static void* dasm_labels[zend_lb_MAX]; || } |.endmacro +|.macro LOAD_IP_ADDR_ZTS, struct, field +| .if ZTS +|| if (GCC_GLOBAL_REGS) { +| LOAD_TSRM_CACHE IP +| mov IP, aword [IP + (struct.._offset + offsetof(zend_..struct, field))] +|| } else { +| LOAD_TSRM_CACHE RX +| mov RX, aword [RX + (struct.._offset + offsetof(zend_..struct, field))] +| mov aword EX->opline, RX +|| } +| .else +| LOAD_IP_ADDR &struct.field +| .endif +|.endmacro + |.macro GET_IP, reg || if (GCC_GLOBAL_REGS) { | mov reg, IP @@ -241,6 +279,16 @@ static void* dasm_labels[zend_lb_MAX]; | ADDR_OP1 push, addr, tmp_reg |.endmacro +|.macro PUSH_ADDR_ZTS, struct, field, tmp_reg +| .if ZTS +| LOAD_TSRM_CACHE tmp_reg +| lea tmp_reg, aword [tmp_reg + (struct.._offset + offsetof(zend_..struct, field))] +| push tmp_reg +| .else +| ADDR_OP1 push, &struct.field, tmp_reg +| .endif +|.endmacro + |.macro MEM_OP1, mem_ins, prefix, addr, tmp_reg | .if X64 || if (IS_32BIT(addr)) { @@ -280,6 +328,24 @@ static void* dasm_labels[zend_lb_MAX]; | .endif |.endmacro +|.macro MEM_OP2_1_ZTS, mem_ins, prefix, struct, field, op2, tmp_reg +| .if ZTS +| LOAD_TSRM_CACHE tmp_reg +| mem_ins prefix [tmp_reg + (struct.._offset + offsetof(zend_..struct, field))], op2 +| .else +| MEM_OP2_1 mem_ins, prefix, &struct.field, op2, tmp_reg +| .endif +|.endmacro + +|.macro MEM_OP2_2_ZTS, mem_ins, op1, prefix, struct, field, tmp_reg +| .if ZTS +| LOAD_TSRM_CACHE tmp_reg +| mem_ins op1, prefix [tmp_reg + (struct.._offset + offsetof(zend_..struct, field))] +| .else +| MEM_OP2_2 mem_ins, op1, prefix, &struct.field, tmp_reg +| .endif +|.endmacro + |.macro MEM_OP3_3, mem_ins, op1, op2, prefix, addr, tmp_reg | .if X64 || if (IS_32BIT(addr)) { @@ -1629,9 +1695,9 @@ static int zend_jit_interrupt_handler_stub(dasm_State **Dst) { |->interrupt_handler: | //EG(vm_interrupt) = 0; - | MEM_OP2_1 mov, byte, &EG(vm_interrupt), 0, r0 + | MEM_OP2_1_ZTS mov, byte, executor_globals, vm_interrupt, 0, r0 | //if (EG(timed_out)) { - | MEM_OP2_1 cmp, byte, &EG(timed_out), 0, r0 + | MEM_OP2_1_ZTS cmp, byte, executor_globals, timed_out, 0, r0 | je >1 | //zend_timeout(0); |.if X64 @@ -1656,7 +1722,7 @@ static int zend_jit_interrupt_handler_stub(dasm_State **Dst) |.endif | //ZEND_VM_ENTER(); | //execute_data = EG(current_execute_data); - | MEM_OP2_2 mov, FP, aword, &EG(current_execute_data), r0 + | MEM_OP2_2_ZTS mov, FP, aword, executor_globals, current_execute_data, r0 | LOAD_OPLINE } | //ZEND_VM_CONTINUE() @@ -1752,10 +1818,10 @@ static int zend_jit_leave_throw_stub(dasm_State **Dst) | cmp byte OP:IP->opcode, ZEND_HANDLE_EXCEPTION | je >5 | // EG(opline_before_exception) = opline; - | MEM_OP2_1 mov, aword, &EG(opline_before_exception), IP, r0 + | MEM_OP2_1_ZTS mov, aword, executor_globals, opline_before_exception, IP, r0 |5: | // opline = EG(exception_op); - | LOAD_IP_ADDR &EG(exception_op) + | LOAD_IP_ADDR_ZTS executor_globals, exception_op | // HANDLE_EXCEPTION() | jmp ->exception_handler } else { @@ -1763,10 +1829,10 @@ static int zend_jit_leave_throw_stub(dasm_State **Dst) | cmp byte OP:FCARG1a->opcode, ZEND_HANDLE_EXCEPTION | je >5 | // EG(opline_before_exception) = opline; - | MEM_OP2_1 mov, aword, &EG(opline_before_exception), FCARG1a, r0 + | MEM_OP2_1_ZTS mov, aword, executor_globals, opline_before_exception, FCARG1a, r0 |5: | // opline = EG(exception_op); - | LOAD_IP_ADDR &EG(exception_op) + | LOAD_IP_ADDR_ZTS executor_globals, exception_op | mov FP, aword T2 // restore FP | mov RX, aword T3 // restore IP | add r4, NR_SPAD // stack alignment @@ -2138,7 +2204,7 @@ static int zend_jit_hybrid_profile_jit_stub(dasm_State **Dst) | xor r1, r1 | test r2, 1 | jz >1 - | MEM_OP2_2 mov, r1, aword, &CG(map_ptr_base), r1 + | MEM_OP2_2_ZTS mov, r1, aword, compiler_globals, map_ptr_base, r1 | sub r1, 1 |1: | mov r2, aword [r1 + r2] @@ -2357,6 +2423,11 @@ static int zend_jit_setup(void) } |.endif +#if ZTS + tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset(); + ZEND_ASSERT(tsrm_ls_cache_tcb_offset != 0); +#endif + return SUCCESS; } @@ -2445,7 +2516,7 @@ static int zend_jit_check_timeout(dasm_State **Dst, const zend_op *opline) return 0; } #else - | MEM_OP2_1 cmp, byte, &EG(vm_interrupt), 0, r0 + | MEM_OP2_1_ZTS cmp, byte, executor_globals, vm_interrupt, 0, r0 | jne >1 |.cold_code |1: @@ -2455,14 +2526,14 @@ static int zend_jit_check_timeout(dasm_State **Dst, const zend_op *opline) return 1; #endif } - | MEM_OP2_1 cmp, byte, &EG(vm_interrupt), 0, r0 + | MEM_OP2_1_ZTS cmp, byte, executor_globals, vm_interrupt, 0, r0 | jne ->interrupt_handler return 1; } static int zend_jit_check_exception(dasm_State **Dst) { - | MEM_OP2_1 cmp, aword, &EG(exception), 0, r0 + | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0 | jne ->exception_handler return 1; } @@ -2470,7 +2541,7 @@ static int zend_jit_check_exception(dasm_State **Dst) static int zend_jit_check_exception_undef_result(dasm_State **Dst, const zend_op *opline) { if (opline->result_type & (IS_TMP_VAR|IS_VAR)) { - | MEM_OP2_1 cmp, aword, &EG(exception), 0, r0 + | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0 | mov r0, opline->result.var | jne ->exception_handler_undef return 1; @@ -4168,10 +4239,10 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o |2: | //retval = zend_hash_index_add_new(ht, hval, &EG(uninitialized_zval)); |.if X64 - | LOAD_ADDR CARG3, &EG(uninitialized_zval) + | LOAD_ADDR_ZTS CARG3, executor_globals, uninitialized_zval |.else | sub r4, 12 - | PUSH_ADDR &EG(uninitialized_zval), r0 + | PUSH_ADDR_ZTS executor_globals, uninitialized_zval, r0 |.endif | EXT_CALL zend_hash_index_add_new, r0 |.if not(X64) @@ -4662,7 +4733,7 @@ static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, zend_op_ |6: if (opline->op2_type == IS_UNUSED) { | // var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval)); - | LOAD_ADDR FCARG2a, &EG(uninitialized_zval) + | LOAD_ADDR_ZTS FCARG2a, executor_globals, uninitialized_zval | EXT_CALL zend_hash_next_index_insert, r0 | // if (UNEXPECTED(!var_ptr)) { | test r0, r0 @@ -4888,7 +4959,7 @@ static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, zend_ |6: if (opline->op2_type == IS_UNUSED) { | // var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval)); - | LOAD_ADDR FCARG2a, &EG(uninitialized_zval) + | LOAD_ADDR_ZTS FCARG2a, executor_globals, uninitialized_zval | EXT_CALL zend_hash_next_index_insert, r0 | // if (UNEXPECTED(!var_ptr)) { | test r0, r0 @@ -6004,7 +6075,7 @@ static int zend_jit_cmp(dasm_State **Dst, const zend_op *opline, int b, int *opn | IF_NOT_Z_TYPE FCARG2a, IS_UNDEF, >1 | mov FCARG1a, opline->op1.var | EXT_CALL zend_jit_undefined_op_helper, r0 - | LOAD_ADDR FCARG2a, &EG(uninitialized_zval) + | LOAD_ADDR_ZTS FCARG2a, executor_globals, uninitialized_zval |1: } if (opline->op2_type == IS_CV && (op2_info & MAY_BE_UNDEF)) { @@ -6014,10 +6085,10 @@ static int zend_jit_cmp(dasm_State **Dst, const zend_op *opline, int b, int *opn | EXT_CALL zend_jit_undefined_op_helper, r0 | mov FCARG2a, T1 // restore |.if X64 - | LOAD_ADDR CARG3, &EG(uninitialized_zval) + | LOAD_ADDR_ZTS CARG3, executor_globals, uninitialized_zval |.else | sub r4, 12 - | PUSH_ADDR &EG(uninitialized_zval), r0 + | PUSH_ADDR_ZTS executor_globals, uninitialized_zval, r0 |.endif | jmp >2 |1: @@ -6134,7 +6205,7 @@ static int zend_jit_identical(dasm_State **Dst, const zend_op *opline, int b, in if (zend_may_throw(opline, op_array, ssa)) { zend_jit_check_exception_undef_result(Dst, opline); } - | LOAD_ADDR FCARG1a, &EG(uninitialized_zval) + | LOAD_ADDR_ZTS FCARG1a, executor_globals, uninitialized_zval | jmp >1 |.code |1: @@ -6151,7 +6222,7 @@ static int zend_jit_identical(dasm_State **Dst, const zend_op *opline, int b, in zend_jit_check_exception_undef_result(Dst, opline); } | mov FCARG1a, aword T1 // restore - | LOAD_ADDR FCARG2a, &EG(uninitialized_zval) + | LOAD_ADDR_ZTS FCARG2a, executor_globals, uninitialized_zval | jmp >1 |.code |1: @@ -6168,7 +6239,7 @@ static int zend_jit_identical(dasm_State **Dst, const zend_op *opline, int b, in if (zend_may_throw(opline, op_array, ssa)) { zend_jit_check_exception_undef_result(Dst, opline); } - | LOAD_ADDR FCARG1a, &EG(uninitialized_zval) + | LOAD_ADDR_ZTS FCARG1a, executor_globals, uninitialized_zval | jmp >1 |.code |1: @@ -6188,7 +6259,7 @@ static int zend_jit_identical(dasm_State **Dst, const zend_op *opline, int b, in if (zend_may_throw(opline, op_array, ssa)) { zend_jit_check_exception_undef_result(Dst, opline); } - | LOAD_ADDR FCARG2a, &EG(uninitialized_zval) + | LOAD_ADDR_ZTS FCARG2a, executor_globals, uninitialized_zval | jmp >1 |.code |1: @@ -6754,7 +6825,7 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, int b, |3: } if (zend_may_throw(opline, op_array, ssa)) { - | MEM_OP2_1 cmp, aword, &EG(exception), 0, r1 + | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r1 | jne ->exception_handler_undef } @@ -6886,9 +6957,9 @@ static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, zen zend_jit_start_reuse_ip(); | // if (UNEXPECTED(used_stack > (size_t)(((char*)EG(vm_stack_end)) - (char*)call))) { - | MEM_OP2_2 mov, RX, aword, &EG(vm_stack_top), RX + | MEM_OP2_2_ZTS mov, RX, aword, executor_globals, vm_stack_top, RX | // Check Stack Overflow - | MEM_OP2_2 mov, r2, aword, &EG(vm_stack_end), r2 + | MEM_OP2_2_ZTS mov, r2, aword, executor_globals, vm_stack_end, r2 | sub r2, RX if (func) { | cmp r2, used_stack @@ -6917,9 +6988,9 @@ static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, zen |.code if (func) { - | MEM_OP2_1 add, aword, &EG(vm_stack_top), used_stack, r2 + | MEM_OP2_1_ZTS add, aword, executor_globals, vm_stack_top, used_stack, r2 } else { - | MEM_OP2_1 add, aword, &EG(vm_stack_top), FCARG1a, r2 + | MEM_OP2_1_ZTS add, aword, executor_globals, vm_stack_top, FCARG1a, r2 } | // zend_vm_init_call_frame(call, call_info, func, num_args, called_scope, object); | // ZEND_SET_CALL_INFO(call, 0, call_info); @@ -7258,7 +7329,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar | mov FCARG1a, RX } | EXT_CALL zend_jit_deprecated_or_abstract_helper, r0 - | MEM_OP2_1 cmp, aword, &EG(exception), 0, r0 + | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0 | jne ->exception_handler | mov r0, EX:RX->func // reload | jmp >1 @@ -7275,7 +7346,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar | mov FCARG1a, RX } | EXT_CALL zend_jit_deprecated_or_abstract_helper, r0 - | MEM_OP2_1 cmp, aword, &EG(exception), 0, r0 + | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0 | jne ->exception_handler | mov r0, EX:RX->func // reload } @@ -7327,7 +7398,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar | xor r1, r1 | test r2, 1 | jz >1 - | MEM_OP2_2 mov, r1, aword, &CG(map_ptr_base), r1 + | MEM_OP2_2_ZTS mov, r1, aword, compiler_globals, map_ptr_base, r1 | sub r1, 1 |1: | mov r2, aword [r1 + r2] @@ -7339,7 +7410,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar } | // EG(current_execute_data) = execute_data; - | MEM_OP2_1 mov, aword, &EG(current_execute_data), RX, r1 + | MEM_OP2_1_ZTS mov, aword, executor_globals, current_execute_data, RX, r1 | mov FP, RX | // opline = op_array->opcodes; @@ -7437,7 +7508,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar #ifdef CONTEXT_THREADED_JIT | call =>(num_args+ssa->cfg.blocks_count) - | MEM_OP2_1 cmp, aword, &EG(exception), 0, r0 + | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0 | jne ->exception_handler if (!func) { | jmp >9 @@ -7448,7 +7519,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar } else { #ifdef CONTEXT_THREADED_JIT | call aword [IP] - | MEM_OP2_1 cmp, aword, &EG(exception), 0, r0 + | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0 | jne ->exception_handler if (!func) { | jmp >9 @@ -7485,7 +7556,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar | mov FCARG1a, RX } | EXT_CALL zend_jit_deprecated_or_abstract_helper, r0 - | MEM_OP2_1 cmp, aword, &EG(exception), 0, r0 + | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0 | jne ->exception_handler | mov r0, EX:RX->func // reload | jmp >1 @@ -7502,7 +7573,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar | mov FCARG1a, RX } | EXT_CALL zend_jit_deprecated_or_abstract_helper, r0 - | MEM_OP2_1 cmp, aword, &EG(exception), 0, r0 + | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0 | jne ->exception_handler | mov r0, EX:RX->func // reload } @@ -7519,7 +7590,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar | SET_Z_TYPE_INFO FCARG2a, IS_NULL | // EG(current_execute_data) = execute_data; - | MEM_OP2_1 mov, aword, &EG(current_execute_data), RX, r1 + | MEM_OP2_1_ZTS mov, aword, executor_globals, current_execute_data, RX, r1 if (!func || (func->common.fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) { if (!func) { @@ -7552,7 +7623,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar } | // EG(current_execute_data) = execute_data; - | MEM_OP2_1 mov, aword, &EG(current_execute_data), FP, r0 + | MEM_OP2_1_ZTS mov, aword, executor_globals, current_execute_data, FP, r0 | // zend_vm_stack_free_args(call); if (func) { @@ -7576,7 +7647,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar | GET_Z_PTR r0, RX + offsetof(zend_execute_data, This) if (opline->op1.num & ZEND_CALL_CTOR) { | // if (UNEXPECTED(EG(exception) - | MEM_OP2_1 cmp, aword, &EG(exception), 0, r1 + | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r1 | je >1 | // GC_DELREF(object); | GC_DELREF r0 @@ -7601,7 +7672,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar | EXT_CALL zend_jit_free_call_frame, r0 | jmp >1 |.code - | MEM_OP2_1 mov, aword, &EG(vm_stack_top), RX, r0 + | MEM_OP2_1_ZTS mov, aword, executor_globals, vm_stack_top, RX, r0 |1: if (!RETURN_VALUE_USED(opline)) { @@ -7618,7 +7689,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_ar } | // if (UNEXPECTED(EG(exception) != NULL)) { - | MEM_OP2_1 cmp, aword, &EG(exception), 0, r0 + | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0 | jne >1 |.cold_code |1: @@ -8486,7 +8557,7 @@ static int zend_jit_leave_func(dasm_State **Dst, const zend_op *opline, zend_op_ if (op_array->scope || (op_array->fn_flags & ZEND_ACC_CLOSURE)) { | // EG(current_execute_data) = EX(prev_execute_data); | mov r0, EX->prev_execute_data - | MEM_OP2_1 mov, aword, &EG(current_execute_data), r0, r2 + | MEM_OP2_1_ZTS mov, aword, executor_globals, current_execute_data, r0, r2 if (op_array->scope) { | // if (call_info & ZEND_CALL_RELEASE_THIS) | test FCARG1d, ZEND_CALL_RELEASE_THIS @@ -8501,7 +8572,7 @@ static int zend_jit_leave_func(dasm_State **Dst, const zend_op *opline, zend_op_ | mov r0, EX->This.value.obj if (op_array->scope && op_array->scope->constructor == (zend_function*)op_array) { | // if (UNEXPECTED(EG(exception) != NULL) - | MEM_OP2_1 cmp, aword, &EG(exception), 0, r1 + | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r1 | jne >6 |.cold_code |6: @@ -8542,19 +8613,19 @@ static int zend_jit_leave_func(dasm_State **Dst, const zend_op *opline, zend_op_ } |4: | // EG(vm_stack_top) = (zval*)execute_data; - | MEM_OP2_1 mov, aword, &EG(vm_stack_top), FP, r0 + | MEM_OP2_1_ZTS mov, aword, executor_globals, vm_stack_top, FP, r0 | // execute_data = EX(prev_execute_data); | mov FP, EX->prev_execute_data } else { | // EG(vm_stack_top) = (zval*)execute_data; - | MEM_OP2_1 mov, aword, &EG(vm_stack_top), FP, r0 + | MEM_OP2_1_ZTS mov, aword, executor_globals, vm_stack_top, FP, r0 | // execute_data = EX(prev_execute_data); | mov FP, EX->prev_execute_data | // EG(current_execute_data) = execute_data - | MEM_OP2_1 mov, aword, &EG(current_execute_data), FP, r0 + | MEM_OP2_1_ZTS mov, aword, executor_globals, current_execute_data, FP, r0 } | // if (EG(exception)) - | MEM_OP2_1 cmp, aword, &EG(exception), 0, r0 + | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0 | LOAD_OPLINE | jne ->leave_throw_handler | // opline = EX(opline) + 1 @@ -9048,7 +9119,7 @@ static int zend_jit_bind_global(dasm_State **Dst, const zend_op *opline, zend_op | mov r0, aword [r0 + opline->extended_value] | sub r0, 1 //if (EXPECTED(idx < EG(symbol_table).nNumUsed)) - | MEM_OP2_2 cmp, eax, dword, &EG(symbol_table).nNumUsed, r1 + | MEM_OP2_2_ZTS cmp, eax, dword, executor_globals, symbol_table.nNumUsed, r1 | jae >9 //Bucket *p = EG(symbol_table).arData + idx; |.if X64 @@ -9056,7 +9127,7 @@ static int zend_jit_bind_global(dasm_State **Dst, const zend_op *opline, zend_op |.else | imul r0, sizeof(Bucket) |.endif - | MEM_OP2_2 add, r0, aword, &EG(symbol_table).arData, r1 + | MEM_OP2_2_ZTS add, r0, aword, executor_globals, symbol_table.arData, r1 | IF_Z_TYPE r0, IS_UNDEF, >9 // (EXPECTED(p->key == Z_STR_P(varname)) | ADDR_OP2_2 cmp, aword [r0 + offsetof(Bucket, key)], Z_PTR_P(varname), r1 From 1fc22f3d3aaeb7d467487fb8e524b5f9ead2e46d Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 15 Mar 2019 00:57:45 +0300 Subject: [PATCH 568/569] JIT ZTS support (only Linux) --- TSRM/TSRM.c | 10 +++- ext/opcache/config.m4 | 3 +- ext/opcache/jit/zend_jit_x86.dasc | 81 +++++++++++++++++++++++++++++-- 3 files changed, 86 insertions(+), 8 deletions(-) diff --git a/TSRM/TSRM.c b/TSRM/TSRM.c index 72f26a8d394fd..dac6577e9d0c4 100644 --- a/TSRM/TSRM.c +++ b/TSRM/TSRM.c @@ -869,10 +869,16 @@ TSRM_API void *tsrm_get_ls_cache(void) /* Returns offset of tsrm_ls_cache slot from Thread Control Block address */ TSRM_API size_t tsrm_get_ls_cache_tcb_offset(void) {/*{{{*/ -#if (defined(__i386__) || defined(__x86_64__)) && defined(__GNUC__) +#if defined(__x86_64__) && defined(__GNUC__) size_t ret; - asm ("mov $_tsrm_ls_cache@tpoff,%0" + asm ("movq _tsrm_ls_cache@gottpoff(%%rip),%0" + : "=r" (ret)); + return ret; +#elif defined(__i386__) && defined(__GNUC__) + size_t ret; + + asm ("leal _tsrm_ls_cache@ntpoff,%0" : "=r" (ret)); return ret; #else diff --git a/ext/opcache/config.m4 b/ext/opcache/config.m4 index 386dbf816c5bb..b2fc09696091e 100644 --- a/ext/opcache/config.m4 +++ b/ext/opcache/config.m4 @@ -35,7 +35,6 @@ if test "$PHP_OPCACHE" != "no"; then case `/usr/bin/file conftest.o` in *64-bit*) DASM_FLAGS="-D X64=1" - PHP_SUBST(DASM_FLAGS) ;; esac fi @@ -45,6 +44,8 @@ if test "$PHP_OPCACHE" != "no"; then DASM_FLAGS="$DASM_FLAGS -D ZTS=1" fi + PHP_SUBST(DASM_FLAGS) + AC_MSG_CHECKING(for opagent in default path) for i in /usr/local /usr; do if test -r $i/include/opagent.h; then diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index b01e6f2a73b7f..5c26efb3fcf52 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -124,7 +124,9 @@ const char* zend_reg_name[] = { static uint32_t zend_jit_x86_flags = 0; #if ZTS -static size_t tsrm_ls_cache_tcb_offset; +static size_t tsrm_ls_cache_tcb_offset = 0; +static size_t tsrm_tls_index; +static size_t tsrm_tls_offset; #endif |.type EX, zend_execute_data, FP @@ -157,12 +159,34 @@ static void* dasm_labels[zend_lb_MAX]; |.endmacro |.macro LOAD_TSRM_CACHE, reg -| .if X64 +| .if X64WIN +| gs +| mov reg, aword [0x58] +| mov reg, aword [reg + tsrm_tls_index] +| mov reg, aword [reg + tsrm_tls_offset] +| .elif WIN | fs -| mov reg, aword [tsrm_ls_cache_tcb_offset] // mov reg, aword fs:tsrm_ls_cache_tcb_offset +| mov reg, aword [0x2c] +| mov reg, aword [reg + tsrm_tls_index] +| mov reg, aword [reg + tsrm_tls_offset] +| .elif X64 +| fs +|| if (tsrm_ls_cache_tcb_offset) { +| mov reg, aword [tsrm_ls_cache_tcb_offset] +|| } else { +| mov reg, [0x8] +| mov reg, aword [reg + tsrm_tls_index] +| mov reg, aword [reg + tsrm_tls_offset] +|| } | .else | gs -| mov reg, aword [tsrm_ls_cache_tcb_offset] // mov reg, aword gs:tsrm_ls_cache_tcb_offset +|| if (tsrm_ls_cache_tcb_offset) { +| mov reg, aword [tsrm_ls_cache_tcb_offset] +|| } else { +| mov reg, [0x4] +| mov reg, aword [reg + tsrm_tls_index] +| mov reg, aword [reg + tsrm_tls_offset] +|| } | .endif |.endmacro @@ -2424,8 +2448,55 @@ static int zend_jit_setup(void) |.endif #if ZTS +# ifdef _WIN64 + // TODO: ??? + tsrm_tls_index = _tls_index * sizeof(void*); + // TODO: how to get this magic value? + tsrm_tls_offset = 0x110; +# elif ZEND_WIN32 + // TODO: ??? + tsrm_tls_index = _tls_index * sizeof(void*); + // TODO: how to get this magic value? + tsrm_tls_offset = 0x110; +# elif defined(__GNUC__) && defined(__x86_64__) tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset(); - ZEND_ASSERT(tsrm_ls_cache_tcb_offset != 0); + if (tsrm_ls_cache_tcb_offset == 0) { + size_t *ti; + + __asm__( + "leaq _tsrm_ls_cache@tlsgd(%%rip), %0\n" + : "=a" (ti)); + tsrm_tls_offset = ti[1]; + tsrm_tls_index = ti[0] * 16; + } +# elif defined(__GNUC__) && defined(__i386__) + tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset(); + if (tsrm_ls_cache_tcb_offset == 0) { +#if 1 + size_t ret; + + asm ("leal _tsrm_ls_cache@ntpoff,%0\n" + : "=a" (ret)); + tsrm_ls_cache_tcb_offset = ret; +#else + size_t *ti, _ebx, _ecx, _edx; + + __asm__( + "call 1f\n" + ".subsection 1\n" + "1:\tmovl (%%esp), %%ebx\n\t" + "ret\n" + ".previous\n\t" + "addl $_GLOBAL_OFFSET_TABLE_, %%ebx\n\t" + "leal _tsrm_ls_cache@tlsldm(%%ebx), %0\n\t" + "call ___tls_get_addr@plt\n\t" + "leal _tsrm_ls_cache@tlsldm(%%ebx), %0\n" + : "=a" (ti), "=&b" (_ebx), "=&c" (_ecx), "=&d" (_edx)); + tsrm_tls_offset = ti[1]; + tsrm_tls_index = ti[0] * 8; +#endif + } +# endif #endif return SUCCESS; From 0a1d7183158bd326db0040e808913aedba006472 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 15 Mar 2019 12:21:57 +0300 Subject: [PATCH 569/569] JIT ZTS support (Linux and Windows) --- ext/opcache/jit/zend_jit_x86.dasc | 58 +++++++++++++++++++++++++++---- 1 file changed, 51 insertions(+), 7 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 5c26efb3fcf52..e7c7b47ca91c6 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -2235,7 +2235,7 @@ static int zend_jit_hybrid_profile_jit_stub(dasm_State **Dst) #else # error "Unknown ZEND_MAP_PTR_KIND" #endif - | inc aword [r2 + zend_jit_profile_counter_rid * sizeof(void)] + | inc aword [r2 + zend_jit_profile_counter_rid * sizeof(void*)] | // handler = (const void*)ZEND_FUNC_INFO(op_array); | mov r0, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])] | // return ((zend_vm_opcode_handler_t)handler)(); @@ -2433,6 +2433,12 @@ static const zend_jit_stub zend_jit_stubs[] = { JIT_STUB(double_one), }; +#if ZTS && defined(ZEND_WIN32) +extern uint32_t _tls_index; +extern char *_tls_start; +extern char *_tls_end; +#endif + static int zend_jit_setup(void) { |.if SSE @@ -2449,15 +2455,53 @@ static int zend_jit_setup(void) #if ZTS # ifdef _WIN64 - // TODO: ??? tsrm_tls_index = _tls_index * sizeof(void*); - // TODO: how to get this magic value? - tsrm_tls_offset = 0x110; + + /* To find offset of "_tsrm_ls_cache" in TLS segment we perform a linear scan of local TLS memory */ + /* Probably, it might be better solution */ + do { + void ***tls_mem = ((void***)__readgsqword(0x58))[_tls_index]; + void *val = _tsrm_ls_cache; + size_t offset = 0; + size_t size = (char*)&_tls_end - (char*)&_tls_start; + + while (offset < size) { + if (*tls_mem == val) { + tsrm_tls_offset = offset; + break; + } + tls_mem++; + offset += sizeof(void*); + } + if (offset >= size) { + // TODO: error message ??? + return FAILURE; + } + } while(0); # elif ZEND_WIN32 - // TODO: ??? tsrm_tls_index = _tls_index * sizeof(void*); - // TODO: how to get this magic value? - tsrm_tls_offset = 0x110; + + /* To find offset of "_tsrm_ls_cache" in TLS segment we perform a linear scan of local TLS memory */ + /* Probably, it might be better solution */ + do { + void ***tls_mem = ((void***)__readfsdword(0x2c))[_tls_index]; + void *val = _tsrm_ls_cache; + size_t offset = 0; + size_t size = (char*)&_tls_end - (char*)&_tls_start; + + while (offset < size) { + if (*tls_mem == val) { + tsrm_tls_offset = offset; + break; + } + tls_mem++; + offset += sizeof(void*); + } + if (offset >= size) { + // TODO: error message ??? + return FAILURE; + } + } while(0); # elif defined(__GNUC__) && defined(__x86_64__) tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset(); if (tsrm_ls_cache_tcb_offset == 0) {