Skip to content

Improved sinking #36

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Brugarolas wants to merge 2 commits into
base: v3.0-work
Choose a base branch
from
Open

Improved sinking #36

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c956050
Improve sinking
XmiliaH Jan 5, 2021
a8d99fe
Small fix
XmiliaH Jan 7, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 7 additions & 3 deletions src/jit/dump.lua
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -387,11 +387,15 @@ local function dump_snap(tr)
end

-- Return a register name or stack slot for a rid/sp location.
local function ridsp_name(ridsp, ins)
local function ridsp_name(ridsp, ins, op)
if not disass then disass = require("jit.dis_"..jit.arch) end
local rid, slot = band(ridsp, 0xff), shr(ridsp, 8)
if rid == 253 or rid == 254 then
return (slot == 0 or slot == 255) and " {sink" or format(" {%04d", ins-slot)
if op == "TNEW " or op == "TDUP " or op == "CNEW " then
return (slot == 0) and " {sink" or format(" {ri%02d", slot)
else
return (slot == 0 or slot == 255) and " {sink" or format(" {%04d", ins-slot)
end
end
if ridsp > 255 then return format("[%x]", slot*4) end
if rid < 128 then return disass.regname(rid) end
Expand Down Expand Up @@ -476,7 +480,7 @@ local function dump_ir(tr, dumpsnap, dumpreg)
(dumpreg or op ~= "RENAME") then
local rid = band(ridsp, 255)
if dumpreg then
out:write(format("%04d %-6s", ins, ridsp_name(ridsp, ins)))
out:write(format("%04d %-6s", ins, ridsp_name(ridsp, ins, op)))
else
out:write(format("%04d ", ins))
end
Expand Down
6 changes: 4 additions & 2 deletions src/lj_asm.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -302,7 +302,7 @@ static void ra_dprintf(ASMState *as, const char *fmt, ...)
#define ra_weak(as, r) rset_set(as->weakset, (r))
#define ra_noweak(as, r) rset_clear(as->weakset, (r))

#define ra_used(ir) (ra_hasreg((ir)->r) || ra_hasspill((ir)->s))
#define ra_used(ir) (ra_hasreg((ir)->r) || ((ir)->r != RID_SUNK && (ir)->r != RID_SINK && ra_hasspill((ir)->s)))

/* Setup register allocator. */
static void ra_setup(ASMState *as)
Expand Down Expand Up @@ -884,7 +884,9 @@ static int asm_sunk_store(ASMState *as, IRIns *ira, IRIns *irs)
}
return 0;
} else {
return (ira + irs->s == irs); /* Quick check. */
if (ira + irs->s != irs) return 0;
return irs->o == IR_ASTORE || irs->o == IR_HSTORE ||
irs->o == IR_FSTORE || irs->o == IR_XSTORE;
}
}

Expand Down
81 changes: 70 additions & 11 deletions src/lj_opt_sink.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
#include "lj_jit.h"
#include "lj_iropt.h"
#include "lj_target.h"
#include "lj_dispatch.h"

/* Some local macros to save typing. Undef'd at the end. */
#define IR(ref) (&J->cur.ir[(ref)])
Expand Down Expand Up @@ -52,6 +53,8 @@ static int sink_checkphi(jit_State *J, IRIns *ira, IRRef ref)
IRIns *ir = IR(ref);
if (irt_isphi(ir->t) || (ir->o == IR_CONV && ir->op2 == IRCONV_NUM_INT &&
irt_isphi(IR(ir->op1)->t))) {
if ((ira->prev & 0x1FFF) == 0x1FFF)
return 0; /* This would cause an overflow, just force the allocation to not be sunken. */
ira->prev++;
return 1; /* Sinkable PHI. */
}
Expand All @@ -61,6 +64,14 @@ static int sink_checkphi(jit_State *J, IRIns *ira, IRRef ref)
return 1; /* Constant (non-PHI). */
}

/* Set prev of all instructions to 0. */
static void sink_prepare(jit_State *J) {
IRIns *ir, *irlast = IR(J->cur.nins-1);
for (ir = irlast ; ir->o != IR_BASE; ir--) {
ir->prev = 0;
}
}

/* Mark non-sinkable allocations using single-pass backward propagation.
**
** Roots for the marking process are:
Expand All @@ -71,13 +82,20 @@ static int sink_checkphi(jit_State *J, IRIns *ira, IRRef ref)
** - Stores with non-constant keys.
** - All stored values.
*/
static void sink_mark_ins(jit_State *J)
static int sink_mark_ins(jit_State *J, int lightsink)
{
int remark = 0;
int heavysinks = 0;
IRIns *ir, *irlast = IR(J->cur.nins-1);
for (ir = irlast ; ; ir--) {
switch (ir->o) {
case IR_BASE:
return; /* Finished. */
if (!remark)
return heavysinks;
ir = irlast + 1;
remark = 0;
heavysinks = 0;
break;
case IR_ALOAD: case IR_HLOAD: case IR_XLOAD: case IR_TBAR: case IR_ALEN:
irt_setmark(IR(ir->op1)->t); /* Mark ref for remaining loads. */
break;
Expand All @@ -87,9 +105,16 @@ static void sink_mark_ins(jit_State *J)
break;
case IR_ASTORE: case IR_HSTORE: case IR_FSTORE: case IR_XSTORE: {
IRIns *ira = sink_checkalloc(J, ir);
if (!ira || (irt_isphi(ira->t) && !sink_checkphi(J, ira, ir->op2)))
IRIns *irv = IR(ir->op2);
if (!ira || (irt_isphi(ira->t) && !sink_checkphi(J, ira, ir->op2)) || (irt_ismarked(ira->t))) {
irt_setmark(IR(ir->op1)->t); /* Mark ineligible ref. */
irt_setmark(IR(ir->op2)->t); /* Mark stored value. */
irt_setmark(irv->t); /* Mark stored value. */
} else if (lightsink || (irv->o != IR_TNEW && irv->o != IR_TDUP && irv->o != IR_CNEW)) {
irt_setmark(irv->t);
} else {
ira->prev |= 0x2000; /* For this allocation is a store that assumes it is sinkable. */
irv->prev |= 0x4000; /* The sunken allocation is required for a other sunken allocation. It requires a global index. */
}
break;
}
#if LJ_HASFFI
Expand All @@ -112,7 +137,8 @@ static void sink_mark_ins(jit_State *J)
break;
case IR_PHI: {
IRIns *irl = IR(ir->op1), *irr = IR(ir->op2);
irl->prev = irr->prev = 0; /* Clear PHI value counts. */
irl->prev &= 0xC000;
irr->prev &= 0xC000; /* Clear PHI value counts. */
if (irl->o == irr->o &&
(irl->o == IR_TNEW || irl->o == IR_TDUP ||
(LJ_HASFFI && (irl->o == IR_CNEW || irl->o == IR_CNEWI))))
Expand All @@ -121,6 +147,21 @@ static void sink_mark_ins(jit_State *J)
irt_setmark(irr->t);
break;
}
case IR_TNEW: case IR_TDUP: case IR_CNEW:
if ((ir->prev & 0x2000) && irt_ismarked(ir->t)) {
ir->prev &= ~0x2000;
remark = 1; /* There is an store that assumed that this allocation can be sunken, but it can't. We need to redo the whool process so that this store can mark it's value. */
}
if (!irt_ismarked(ir->t) && (ir->prev & 0x4000)) {
ir->prev |= 0x8000;
heavysinks++;
} else {
ir->prev &= ~0x8000;
}
ir->prev &= ~0x4000;
if (!irt_isphi(ir->t))
ir->prev &= ~0x2000;
/* fallthrough */
default:
if (irt_ismarked(ir->t) || irt_isguard(ir->t)) { /* Propagate mark. */
if (ir->op1 >= REF_FIRST) irt_setmark(IR(ir->op1)->t);
Expand All @@ -144,26 +185,34 @@ static void sink_mark_snap(jit_State *J, SnapShot *snap)
}

/* Iteratively remark PHI refs with differing marks or PHI value counts. */
static void sink_remark_phi(jit_State *J)
static int sink_remark_phi(jit_State *J)
{
IRIns *ir;
int remark;
int require_remark = 0;
do {
remark = 0;
for (ir = IR(J->cur.nins-1); ir->o == IR_PHI; ir--) {
IRIns *irl = IR(ir->op1), *irr = IR(ir->op2);
if (!((irl->t.irt ^ irr->t.irt) & IRT_MARK) && irl->prev == irr->prev)
if (!((irl->t.irt ^ irr->t.irt) & IRT_MARK) && (irl->prev & 0x1FFF) == (irr->prev & 0x1FFF))
continue;
remark |= (~(irl->t.irt & irr->t.irt) & IRT_MARK);
if ((IR(ir->op1)->prev & 0x2000) || (IR(ir->op2)->prev & 0x2000)) {
IR(ir->op1)->prev &= ~0x2000;
IR(ir->op2)->prev &= ~0x2000;
require_remark |= (~(irl->t.irt & irr->t.irt) & IRT_MARK);
}
irt_setmark(IR(ir->op1)->t);
irt_setmark(IR(ir->op2)->t);
}
} while (remark);
return require_remark;
}

/* Sweep instructions and tag sunken allocations and stores. */
static void sink_sweep_ins(jit_State *J)
{
int index = 0;
IRIns *ir, *irbase = IR(REF_BASE);
for (ir = IR(J->cur.nins-1) ; ir >= irbase; ir--) {
switch (ir->o) {
Expand All @@ -190,8 +239,14 @@ static void sink_sweep_ins(jit_State *J)
#endif
case IR_TNEW: case IR_TDUP:
if (!irt_ismarked(ir->t)) {
if (ir->prev & 0x8000) {
index++; /* A sunken store requires this for unsinking. */
lj_assertJ(index <= 0xFF, "Too many heavy sinks");
ir->prev = REGSP(RID_SINK, index);
} else {
ir->prev = REGSP(RID_SINK, 0);
}
ir->t.irt &= ~IRT_GUARD;
ir->prev = REGSP(RID_SINK, 0);
J->cur.sinktags = 1; /* Signal present SINK tags to assembler. */
} else {
irt_clearmark(ir->t);
Expand Down Expand Up @@ -236,11 +291,15 @@ void lj_opt_sink(jit_State *J)
if ((J->flags & need) == need &&
(J->chain[IR_TNEW] || J->chain[IR_TDUP] ||
(LJ_HASFFI && (J->chain[IR_CNEW] || J->chain[IR_CNEWI])))) {
sink_prepare(J);
if (!J->loopref)
sink_mark_snap(J, &J->cur.snap[J->cur.nsnap-1]);
sink_mark_ins(J);
if (J->loopref)
sink_remark_phi(J);
int heavysinks;
int dolightsink = 0;
do {
heavysinks = sink_mark_ins(J, dolightsink);
dolightsink |= heavysinks >= 0xFF;
} while ((J->loopref && sink_remark_phi(J)) || heavysinks >= 0xFF);
sink_sweep_ins(J);
}
}
Expand Down
Loading