Skip to content

Commit e675564

Browse files
wenyonghwenyongh
authored
Fix fast jit issues (#1169)
Implement bitwise 64-bit operations in codegen Fix and refine shift IRs Zero local variables Remove ref-type/bulk-memory macros Implement set aux stack Refine clear mem registers
1 parent eec5450 commit e675564

File tree

11 files changed

+349
-206
lines changed

11 files changed

+349
-206
lines changed

‎core/iwasm/fast-jit/cg/x86-64/jit_codegen_x86_64.cpp‎

Lines changed: 182 additions & 36 deletions
Large diffs are not rendered by default.

‎core/iwasm/fast-jit/fe/jit_emit_control.c‎

Lines changed: 45 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -241,7 +241,7 @@ push_jit_block_to_stack_and_pass_params(JitCompContext *cc, JitBlock *block,
241241
block->incoming_insn_for_else_bb = insn;
242242
}
243243
else {
244-
if (!jit_block_add_incoming_insn(block, insn)) {
244+
if (!jit_block_add_incoming_insn(block, insn, 2)) {
245245
jit_set_last_error(cc, "add incoming insn failed");
246246
goto fail;
247247
}
@@ -449,17 +449,9 @@ handle_op_end(JitCompContext *cc, uint8 **p_frame_ip, bool is_block_polymorphic)
449449
incoming_insn = block->incoming_insns_for_end_bb;
450450
while (incoming_insn) {
451451
insn = incoming_insn->insn;
452-
if (insn->opcode == JIT_OP_JMP) {
453-
*(jit_insn_opnd(insn, 0)) =
454-
jit_basic_block_label(block->basic_block_end);
455-
}
456-
else if (insn->opcode == JIT_OP_BNE) {
457-
*(jit_insn_opnd(insn, 2)) =
458-
jit_basic_block_label(block->basic_block_end);
459-
}
460-
else {
461-
bh_assert(0);
462-
}
452+
bh_assert(insn->opcode == JIT_OP_JMP || insn->opcode == JIT_OP_BNE);
453+
*(jit_insn_opnd(insn, incoming_insn->opnd_idx)) =
454+
jit_basic_block_label(block->basic_block_end);
463455
incoming_insn = incoming_insn->next;
464456
}
465457

@@ -534,7 +526,7 @@ handle_op_else(JitCompContext *cc, uint8 **p_frame_ip,
534526
jit_set_last_error(cc, "generate jmp insn failed");
535527
return false;
536528
}
537-
if (!jit_block_add_incoming_insn(block, insn)) {
529+
if (!jit_block_add_incoming_insn(block, insn, 0)) {
538530
jit_set_last_error(cc, "add incoming insn failed");
539531
return false;
540532
}
@@ -821,7 +813,7 @@ handle_op_br(JitCompContext *cc, uint32 br_depth, uint8 **p_frame_ip)
821813
jit_set_last_error(cc, "generate jmp insn failed");
822814
goto fail;
823815
}
824-
if (!jit_block_add_incoming_insn(block_dst, insn)) {
816+
if (!jit_block_add_incoming_insn(block_dst, insn, 0)) {
825817
jit_set_last_error(cc, "add incoming insn failed");
826818
goto fail;
827819
}
@@ -847,6 +839,8 @@ jit_compile_op_br_if(JitCompContext *cc, uint32 br_depth, uint8 **p_frame_ip)
847839
JitBlock *block_dst;
848840
JitReg cond;
849841
JitBasicBlock *cur_basic_block, *if_basic_block = NULL;
842+
JitValueSlot *frame_sp_src;
843+
JitInsn *insn;
850844

851845
if (!(block_dst = get_target_block(cc, br_depth))) {
852846
return false;
@@ -861,6 +855,41 @@ jit_compile_op_br_if(JitCompContext *cc, uint32 br_depth, uint8 **p_frame_ip)
861855
/* Clear frame values */
862856
clear_values(jit_frame);
863857

858+
if (block_dst->label_type == LABEL_TYPE_LOOP) {
859+
frame_sp_src =
860+
jit_frame->sp
861+
- wasm_get_cell_num(block_dst->param_types, block_dst->param_count);
862+
}
863+
else {
864+
frame_sp_src = jit_frame->sp
865+
- wasm_get_cell_num(block_dst->result_types,
866+
block_dst->result_count);
867+
}
868+
869+
if (block_dst->frame_sp_begin == frame_sp_src) {
870+
if (block_dst->label_type == LABEL_TYPE_LOOP) {
871+
if (!GEN_INSN(CMP, cc->cmp_reg, cond, NEW_CONST(I32, 0))
872+
|| !GEN_INSN(
873+
BNE, cc->cmp_reg,
874+
jit_basic_block_label(block_dst->basic_block_entry), 0)) {
875+
jit_set_last_error(cc, "generate bne insn failed");
876+
goto fail;
877+
}
878+
}
879+
else {
880+
if (!GEN_INSN(CMP, cc->cmp_reg, cond, NEW_CONST(I32, 0))
881+
|| !(insn = GEN_INSN(BNE, cc->cmp_reg, 0, 0))) {
882+
jit_set_last_error(cc, "generate bne insn failed");
883+
goto fail;
884+
}
885+
if (!jit_block_add_incoming_insn(block_dst, insn, 1)) {
886+
jit_set_last_error(cc, "add incoming insn failed");
887+
goto fail;
888+
}
889+
}
890+
return true;
891+
}
892+
864893
CREATE_BASIC_BLOCK(if_basic_block);
865894
if (!GEN_INSN(CMP, cc->cmp_reg, cond, NEW_CONST(I32, 0))
866895
|| !GEN_INSN(BNE, cc->cmp_reg, jit_basic_block_label(if_basic_block),
@@ -912,6 +941,8 @@ jit_compile_op_br_table(JitCompContext *cc, uint32 *br_depths, uint32 br_count,
912941
for (i = 0, opnd = jit_insn_opndls(insn); i < br_count + 1; i++) {
913942
JitBasicBlock *basic_block = NULL;
914943

944+
/* TODO: refine the code */
945+
915946
CREATE_BASIC_BLOCK(basic_block);
916947
SET_BB_BEGIN_BCIP(basic_block, *p_frame_ip - 1);
917948

‎core/iwasm/fast-jit/fe/jit_emit_function.c‎

Lines changed: 14 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,9 @@ post_return(JitCompContext *cc, const WASMType *func_type)
112112
}
113113
}
114114

115+
/* Update the committed_sp as the callee has updated the frame sp */
116+
cc->jit_frame->committed_sp = cc->jit_frame->sp;
117+
115118
return true;
116119
fail:
117120
return false;
@@ -127,6 +130,10 @@ jit_compile_op_call(JitCompContext *cc, uint32 func_idx, bool tail_call)
127130
JitFrame *jit_frame = cc->jit_frame;
128131
JitReg result = 0, native_ret;
129132
JitReg func_ptrs, jitted_code = 0;
133+
#if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64)
134+
JitReg eax_hreg = jit_codegen_get_hreg_by_name("eax");
135+
JitReg rax_hreg = jit_codegen_get_hreg_by_name("rax");
136+
#endif
130137
JitInsn *insn;
131138
uint32 jitted_func_idx;
132139

@@ -156,7 +163,7 @@ jit_compile_op_call(JitCompContext *cc, uint32 func_idx, bool tail_call)
156163
if (func_idx < wasm_module->import_function_count) {
157164
#if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64)
158165
/* Set native_ret to x86::eax */
159-
native_ret = jit_codegen_get_hreg_by_name("eax");
166+
native_ret = eax_hreg;
160167
#else
161168
native_ret = jit_cc_new_reg_I32(cc);
162169
#endif
@@ -184,16 +191,14 @@ jit_compile_op_call(JitCompContext *cc, uint32 func_idx, bool tail_call)
184191
case VALUE_TYPE_FUNCREF:
185192
#endif
186193
#if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64)
187-
/* Set result to x86::eax, 1 is hard reg index of eax */
188-
result = jit_reg_new(JIT_REG_KIND_I32, 1);
194+
result = eax_hreg;
189195
#else
190196
result = jit_cc_new_reg_I32(cc);
191197
#endif
192198
break;
193199
case VALUE_TYPE_I64:
194200
#if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64)
195-
/* Set result to x86::rax, 1 is hard reg index of rax */
196-
result = jit_reg_new(JIT_REG_KIND_I64, 1);
201+
result = rax_hreg;
197202
#else
198203
result = jit_cc_new_reg_I64(cc);
199204
#endif
@@ -219,7 +224,8 @@ jit_compile_op_call(JitCompContext *cc, uint32 func_idx, bool tail_call)
219224

220225
/* Clear part of memory regs and table regs as their values
221226
may be changed in the function call */
222-
clear_memory_regs(jit_frame);
227+
if (cc->cur_wasm_module->possible_memory_grow)
228+
clear_memory_regs(jit_frame);
223229
clear_table_regs(jit_frame);
224230

225231
/* Ignore tail call currently */
@@ -364,7 +370,8 @@ jit_compile_op_call_indirect(JitCompContext *cc, uint32 type_idx,
364370

365371
/* Clear part of memory regs and table regs as their values
366372
may be changed in the function call */
367-
clear_memory_regs(cc->jit_frame);
373+
if (cc->cur_wasm_module->possible_memory_grow)
374+
clear_memory_regs(cc->jit_frame);
368375
clear_table_regs(cc->jit_frame);
369376
return true;
370377
fail:

‎core/iwasm/fast-jit/fe/jit_emit_numberic.c‎

Lines changed: 45 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -182,7 +182,7 @@ compile_op_ibinopt_const(JitCompContext *cc, JitReg left, JitReg right,
182182

183183
if (jit_reg_is_const(left) || jit_reg_is_const(right)) {
184184
res = handle_one_const(cc, left, right, is_i32);
185-
if (!res)
185+
if (res)
186186
goto shortcut;
187187
}
188188

@@ -787,25 +787,44 @@ jit_compile_op_i64_bitwise(JitCompContext *cc, IntBitwise bitwise_op)
787787

788788
DEF_UNI_INT_CONST_OPS(shl)
789789
{
790-
if (IS_CONST_ZERO(right)) {
790+
if (IS_CONST_ZERO(right) || IS_CONST_ZERO(left)) {
791791
return left;
792792
}
793+
794+
if (jit_reg_is_const(right)) {
795+
JitReg res = is_i32 ? jit_cc_new_reg_I32(cc) : jit_cc_new_reg_I64(cc);
796+
GEN_INSN(SHL, res, left, right);
797+
return res;
798+
}
793799
return 0;
794800
}
795801

796802
DEF_UNI_INT_CONST_OPS(shrs)
797803
{
798-
if (IS_CONST_ZERO(right)) {
804+
if (IS_CONST_ZERO(right) || IS_CONST_ZERO(left)
805+
|| IS_CONST_ALL_ONE(left, is_i32)) {
799806
return left;
800807
}
808+
809+
if (jit_reg_is_const(right)) {
810+
JitReg res = is_i32 ? jit_cc_new_reg_I32(cc) : jit_cc_new_reg_I64(cc);
811+
GEN_INSN(SHRS, res, left, right);
812+
return res;
813+
}
801814
return 0;
802815
}
803816

804817
DEF_UNI_INT_CONST_OPS(shru)
805818
{
806-
if (IS_CONST_ZERO(right)) {
819+
if (IS_CONST_ZERO(right) || IS_CONST_ZERO(left)) {
807820
return left;
808821
}
822+
823+
if (jit_reg_is_const(right)) {
824+
JitReg res = is_i32 ? jit_cc_new_reg_I32(cc) : jit_cc_new_reg_I64(cc);
825+
GEN_INSN(SHRU, res, left, right);
826+
return res;
827+
}
809828
return 0;
810829
}
811830

@@ -958,11 +977,15 @@ compile_int_shru(JitCompContext *cc, JitReg left, JitReg right, bool is_i32)
958977

959978
DEF_UNI_INT_CONST_OPS(rotl)
960979
{
961-
if (IS_CONST_ZERO(right))
980+
if (IS_CONST_ZERO(right) || IS_CONST_ZERO(left)
981+
|| IS_CONST_ALL_ONE(left, is_i32))
962982
return left;
963983

964-
if (IS_CONST_ZERO(left))
965-
return right;
984+
if (jit_reg_is_const(right)) {
985+
JitReg res = is_i32 ? jit_cc_new_reg_I32(cc) : jit_cc_new_reg_I64(cc);
986+
GEN_INSN(ROTL, res, left, right);
987+
return res;
988+
}
966989

967990
return 0;
968991
}
@@ -986,7 +1009,7 @@ do_i64_const_rotl(int64 lhs, int64 rhs)
9861009
static JitReg
9871010
compile_int_rotl(JitCompContext *cc, JitReg left, JitReg right, bool is_i32)
9881011
{
989-
JitReg res, tmp, shl_res, shr_res;
1012+
JitReg res;
9901013
#if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64)
9911014
JitReg ecx_hreg = jit_codegen_get_hreg_by_name("ecx");
9921015
JitReg rcx_hreg = jit_codegen_get_hreg_by_name("rcx");
@@ -1000,34 +1023,13 @@ compile_int_rotl(JitCompContext *cc, JitReg left, JitReg right, bool is_i32)
10001023

10011024
left = mov_left_to_reg(cc, is_i32, left);
10021025

1003-
if (is_i32) {
1004-
tmp = jit_cc_new_reg_I32(cc);
1005-
shl_res = jit_cc_new_reg_I32(cc);
1006-
shr_res = jit_cc_new_reg_I32(cc);
1007-
res = jit_cc_new_reg_I32(cc);
1008-
}
1009-
else {
1010-
tmp = jit_cc_new_reg_I64(cc);
1011-
shl_res = jit_cc_new_reg_I64(cc);
1012-
shr_res = jit_cc_new_reg_I64(cc);
1013-
res = jit_cc_new_reg_I64(cc);
1014-
}
1015-
1016-
/* 32/64 - rhs */
1017-
GEN_INSN(SUB, tmp, is_i32 ? NEW_CONST(I32, 32) : NEW_CONST(I64, 64), right);
1026+
res = is_i32 ? jit_cc_new_reg_I32(cc) : jit_cc_new_reg_I64(cc);
10181027
#if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64)
10191028
GEN_INSN(MOV, is_i32 ? ecx_hreg : rcx_hreg, right);
1020-
GEN_INSN(SHL, shl_res, left, is_i32 ? ecx_hreg : rcx_hreg);
1021-
1022-
GEN_INSN(MOV, is_i32 ? ecx_hreg : rcx_hreg, tmp);
1023-
GEN_INSN(SHRU, shr_res, left, is_i32 ? ecx_hreg : rcx_hreg);
1024-
1025-
GEN_INSN(OR, res, shl_res, shr_res);
1029+
GEN_INSN(ROTL, res, left, is_i32 ? ecx_hreg : rcx_hreg);
10261030
GEN_INSN(MOV, ecx_hreg, ecx_hreg);
10271031
#else
1028-
GEN_INSN(SHL, shl_res, left, right);
1029-
GEN_INSN(SHRU, shr_res, left, tmp);
1030-
GEN_INSN(OR, res, shl_res, shr_res);
1032+
GEN_INSN(ROTL, res, left, right);
10311033
#endif
10321034

10331035
shortcut:
@@ -1036,11 +1038,15 @@ compile_int_rotl(JitCompContext *cc, JitReg left, JitReg right, bool is_i32)
10361038

10371039
DEF_UNI_INT_CONST_OPS(rotr)
10381040
{
1039-
if (IS_CONST_ZERO(right))
1041+
if (IS_CONST_ZERO(right) || IS_CONST_ZERO(left)
1042+
|| IS_CONST_ALL_ONE(left, is_i32))
10401043
return left;
10411044

1042-
if (IS_CONST_ZERO(left))
1043-
return right;
1045+
if (jit_reg_is_const(right)) {
1046+
JitReg res = is_i32 ? jit_cc_new_reg_I32(cc) : jit_cc_new_reg_I64(cc);
1047+
GEN_INSN(ROTR, res, left, right);
1048+
return res;
1049+
}
10441050

10451051
return 0;
10461052
}
@@ -1064,7 +1070,7 @@ do_i64_const_rotr(int64 lhs, int64 rhs)
10641070
static JitReg
10651071
compile_int_rotr(JitCompContext *cc, JitReg left, JitReg right, bool is_i32)
10661072
{
1067-
JitReg res, tmp, shr_res, shl_res;
1073+
JitReg res;
10681074
#if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64)
10691075
JitReg ecx_hreg = jit_codegen_get_hreg_by_name("ecx");
10701076
JitReg rcx_hreg = jit_codegen_get_hreg_by_name("rcx");
@@ -1078,34 +1084,13 @@ compile_int_rotr(JitCompContext *cc, JitReg left, JitReg right, bool is_i32)
10781084

10791085
left = mov_left_to_reg(cc, is_i32, left);
10801086

1081-
if (is_i32) {
1082-
tmp = jit_cc_new_reg_I32(cc);
1083-
shr_res = jit_cc_new_reg_I32(cc);
1084-
shl_res = jit_cc_new_reg_I32(cc);
1085-
res = jit_cc_new_reg_I32(cc);
1086-
}
1087-
else {
1088-
tmp = jit_cc_new_reg_I64(cc);
1089-
shr_res = jit_cc_new_reg_I64(cc);
1090-
shl_res = jit_cc_new_reg_I64(cc);
1091-
res = jit_cc_new_reg_I64(cc);
1092-
}
1093-
1094-
/* 32/64 - rhs */
1095-
GEN_INSN(SUB, tmp, is_i32 ? NEW_CONST(I32, 32) : NEW_CONST(I64, 64), right);
1087+
res = is_i32 ? jit_cc_new_reg_I32(cc) : jit_cc_new_reg_I64(cc);
10961088
#if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64)
10971089
GEN_INSN(MOV, is_i32 ? ecx_hreg : rcx_hreg, right);
1098-
GEN_INSN(SHRU, shl_res, left, is_i32 ? ecx_hreg : rcx_hreg);
1099-
1100-
GEN_INSN(MOV, is_i32 ? ecx_hreg : rcx_hreg, tmp);
1101-
GEN_INSN(SHL, shr_res, left, is_i32 ? ecx_hreg : rcx_hreg);
1102-
1103-
GEN_INSN(OR, res, shl_res, shr_res);
1090+
GEN_INSN(ROTR, res, left, is_i32 ? ecx_hreg : rcx_hreg);
11041091
GEN_INSN(MOV, ecx_hreg, ecx_hreg);
11051092
#else
1106-
GEN_INSN(SHRU, shr_res, left, right);
1107-
GEN_INSN(SHL, shl_res, left, tmp);
1108-
GEN_INSN(OR, res, shr_res, shl_res);
1093+
GEN_INSN(ROTR, res, left, right);
11091094
#endif
11101095

11111096
shortcut:

‎core/iwasm/fast-jit/fe/jit_emit_parametric.c‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -104,7 +104,8 @@ jit_compile_op_select(JitCompContext *cc, bool is_select_32)
104104
else
105105
selected = jit_cc_new_reg_I64(cc);
106106

107-
GEN_INSN(SELECTNE, selected, cond, val1, val2);
107+
GEN_INSN(CMP, cc->cmp_reg, cond, NEW_CONST(I32, 0));
108+
GEN_INSN(SELECTNE, selected, cc->cmp_reg, val1, val2);
108109
PUSH(selected, val1_type);
109110
return true;
110111
fail:

0 commit comments

Comments
 (0)