Skip to content

Commit 092ac24

Browse files
rth7680pm215
rth7680
authored and
pm215
committed
tcg: Add tcg_gen_atomic_{xchg,fetch_and,fetch_or}_i128
Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Message-id: 20250815122653.701782-5-richard.henderson@linaro.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org>
1 parent 33aefd1 commit 092ac24

File tree

5 files changed

+125
-3
lines changed

5 files changed

+125
-3
lines changed

‎accel/tcg/atomic_common.c.inc‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -122,5 +122,14 @@ GEN_ATOMIC_HELPERS(umax_fetch)
122122

123123
GEN_ATOMIC_HELPERS(xchg)
124124

125+
#if HAVE_CMPXCHG128
126+
ATOMIC_HELPER(xchgo_be, Int128)
127+
ATOMIC_HELPER(xchgo_le, Int128)
128+
ATOMIC_HELPER(fetch_ando_be, Int128)
129+
ATOMIC_HELPER(fetch_ando_le, Int128)
130+
ATOMIC_HELPER(fetch_oro_be, Int128)
131+
ATOMIC_HELPER(fetch_oro_le, Int128)
132+
#endif
133+
125134
#undef ATOMIC_HELPER
126135
#undef GEN_ATOMIC_HELPERS

‎accel/tcg/tcg-runtime.h‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,18 @@ DEF_HELPER_FLAGS_5(atomic_cmpxchgo_be, TCG_CALL_NO_WG,
6363
i128, env, i64, i128, i128, i32)
6464
DEF_HELPER_FLAGS_5(atomic_cmpxchgo_le, TCG_CALL_NO_WG,
6565
i128, env, i64, i128, i128, i32)
66+
DEF_HELPER_FLAGS_4(atomic_xchgo_be, TCG_CALL_NO_WG,
67+
i128, env, i64, i128, i32)
68+
DEF_HELPER_FLAGS_4(atomic_xchgo_le, TCG_CALL_NO_WG,
69+
i128, env, i64, i128, i32)
70+
DEF_HELPER_FLAGS_4(atomic_fetch_ando_be, TCG_CALL_NO_WG,
71+
i128, env, i64, i128, i32)
72+
DEF_HELPER_FLAGS_4(atomic_fetch_ando_le, TCG_CALL_NO_WG,
73+
i128, env, i64, i128, i32)
74+
DEF_HELPER_FLAGS_4(atomic_fetch_oro_be, TCG_CALL_NO_WG,
75+
i128, env, i64, i128, i32)
76+
DEF_HELPER_FLAGS_4(atomic_fetch_oro_le, TCG_CALL_NO_WG,
77+
i128, env, i64, i128, i32)
6678
#endif
6779

6880
DEF_HELPER_FLAGS_5(nonatomic_cmpxchgo, TCG_CALL_NO_WG,

‎include/tcg/tcg-op-common.h‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -344,6 +344,8 @@ void tcg_gen_atomic_xchg_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32,
344344
TCGArg, MemOp, TCGType);
345345
void tcg_gen_atomic_xchg_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64,
346346
TCGArg, MemOp, TCGType);
347+
void tcg_gen_atomic_xchg_i128_chk(TCGv_i128, TCGTemp *, TCGv_i128,
348+
TCGArg, MemOp, TCGType);
347349

348350
void tcg_gen_atomic_fetch_add_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32,
349351
TCGArg, MemOp, TCGType);
@@ -411,6 +413,11 @@ void tcg_gen_atomic_umax_fetch_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32,
411413
void tcg_gen_atomic_umax_fetch_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64,
412414
TCGArg, MemOp, TCGType);
413415

416+
void tcg_gen_atomic_fetch_and_i128_chk(TCGv_i128, TCGTemp *, TCGv_i128,
417+
TCGArg, MemOp, TCGType);
418+
void tcg_gen_atomic_fetch_or_i128_chk(TCGv_i128, TCGTemp *, TCGv_i128,
419+
TCGArg, MemOp, TCGType);
420+
414421
/* Vector ops */
415422

416423
void tcg_gen_mov_vec(TCGv_vec, TCGv_vec);

‎include/tcg/tcg-op.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -134,13 +134,16 @@ DEF_ATOMIC3(tcg_gen_nonatomic_cmpxchg, i128)
134134

135135
DEF_ATOMIC2(tcg_gen_atomic_xchg, i32)
136136
DEF_ATOMIC2(tcg_gen_atomic_xchg, i64)
137+
DEF_ATOMIC2(tcg_gen_atomic_xchg, i128)
137138

138139
DEF_ATOMIC2(tcg_gen_atomic_fetch_add, i32)
139140
DEF_ATOMIC2(tcg_gen_atomic_fetch_add, i64)
140141
DEF_ATOMIC2(tcg_gen_atomic_fetch_and, i32)
141142
DEF_ATOMIC2(tcg_gen_atomic_fetch_and, i64)
143+
DEF_ATOMIC2(tcg_gen_atomic_fetch_and, i128)
142144
DEF_ATOMIC2(tcg_gen_atomic_fetch_or, i32)
143145
DEF_ATOMIC2(tcg_gen_atomic_fetch_or, i64)
146+
DEF_ATOMIC2(tcg_gen_atomic_fetch_or, i128)
144147
DEF_ATOMIC2(tcg_gen_atomic_fetch_xor, i32)
145148
DEF_ATOMIC2(tcg_gen_atomic_fetch_xor, i64)
146149
DEF_ATOMIC2(tcg_gen_atomic_fetch_smin, i32)

‎tcg/tcg-op-ldst.c‎

Lines changed: 94 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -801,6 +801,8 @@ typedef void (*gen_atomic_op_i32)(TCGv_i32, TCGv_env, TCGv_i64,
801801
TCGv_i32, TCGv_i32);
802802
typedef void (*gen_atomic_op_i64)(TCGv_i64, TCGv_env, TCGv_i64,
803803
TCGv_i64, TCGv_i32);
804+
typedef void (*gen_atomic_op_i128)(TCGv_i128, TCGv_env, TCGv_i64,
805+
TCGv_i128, TCGv_i32);
804806

805807
#ifdef CONFIG_ATOMIC64
806808
# define WITH_ATOMIC64(X) X,
@@ -1201,6 +1203,94 @@ static void do_atomic_op_i64(TCGv_i64 ret, TCGTemp *addr, TCGv_i64 val,
12011203
}
12021204
}
12031205

1206+
static void do_nonatomic_op_i128(TCGv_i128 ret, TCGTemp *addr, TCGv_i128 val,
1207+
TCGArg idx, MemOp memop, bool new_val,
1208+
void (*gen)(TCGv_i64, TCGv_i64, TCGv_i64))
1209+
{
1210+
TCGv_i128 t = tcg_temp_ebb_new_i128();
1211+
TCGv_i128 r = tcg_temp_ebb_new_i128();
1212+
1213+
tcg_gen_qemu_ld_i128_int(r, addr, idx, memop);
1214+
gen(TCGV128_LOW(t), TCGV128_LOW(r), TCGV128_LOW(val));
1215+
gen(TCGV128_HIGH(t), TCGV128_HIGH(r), TCGV128_HIGH(val));
1216+
tcg_gen_qemu_st_i128_int(t, addr, idx, memop);
1217+
1218+
tcg_gen_mov_i128(ret, r);
1219+
tcg_temp_free_i128(t);
1220+
tcg_temp_free_i128(r);
1221+
}
1222+
1223+
static void do_atomic_op_i128(TCGv_i128 ret, TCGTemp *addr, TCGv_i128 val,
1224+
TCGArg idx, MemOp memop, void * const table[])
1225+
{
1226+
gen_atomic_op_i128 gen = table[memop & (MO_SIZE | MO_BSWAP)];
1227+
1228+
if (gen) {
1229+
MemOpIdx oi = make_memop_idx(memop & ~MO_SIGN, idx);
1230+
TCGv_i64 a64 = maybe_extend_addr64(addr);
1231+
gen(ret, tcg_env, a64, val, tcg_constant_i32(oi));
1232+
maybe_free_addr64(a64);
1233+
return;
1234+
}
1235+
1236+
gen_helper_exit_atomic(tcg_env);
1237+
/* Produce a result */
1238+
tcg_gen_movi_i64(TCGV128_LOW(ret), 0);
1239+
tcg_gen_movi_i64(TCGV128_HIGH(ret), 0);
1240+
}
1241+
1242+
#define GEN_ATOMIC_HELPER128(NAME, OP, NEW) \
1243+
static void * const table_##NAME[(MO_SIZE | MO_BSWAP) + 1] = { \
1244+
[MO_8] = gen_helper_atomic_##NAME##b, \
1245+
[MO_16 | MO_LE] = gen_helper_atomic_##NAME##w_le, \
1246+
[MO_16 | MO_BE] = gen_helper_atomic_##NAME##w_be, \
1247+
[MO_32 | MO_LE] = gen_helper_atomic_##NAME##l_le, \
1248+
[MO_32 | MO_BE] = gen_helper_atomic_##NAME##l_be, \
1249+
WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_##NAME##q_le) \
1250+
WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_##NAME##q_be) \
1251+
WITH_ATOMIC128([MO_128 | MO_LE] = gen_helper_atomic_##NAME##o_le) \
1252+
WITH_ATOMIC128([MO_128 | MO_BE] = gen_helper_atomic_##NAME##o_be) \
1253+
}; \
1254+
void tcg_gen_atomic_##NAME##_i32_chk(TCGv_i32 ret, TCGTemp *addr, \
1255+
TCGv_i32 val, TCGArg idx, \
1256+
MemOp memop, TCGType addr_type) \
1257+
{ \
1258+
tcg_debug_assert(addr_type == tcg_ctx->addr_type); \
1259+
tcg_debug_assert((memop & MO_SIZE) <= MO_32); \
1260+
if (tcg_ctx->gen_tb->cflags & CF_PARALLEL) { \
1261+
do_atomic_op_i32(ret, addr, val, idx, memop, table_##NAME); \
1262+
} else { \
1263+
do_nonatomic_op_i32(ret, addr, val, idx, memop, NEW, \
1264+
tcg_gen_##OP##_i32); \
1265+
} \
1266+
} \
1267+
void tcg_gen_atomic_##NAME##_i64_chk(TCGv_i64 ret, TCGTemp *addr, \
1268+
TCGv_i64 val, TCGArg idx, \
1269+
MemOp memop, TCGType addr_type) \
1270+
{ \
1271+
tcg_debug_assert(addr_type == tcg_ctx->addr_type); \
1272+
tcg_debug_assert((memop & MO_SIZE) <= MO_64); \
1273+
if (tcg_ctx->gen_tb->cflags & CF_PARALLEL) { \
1274+
do_atomic_op_i64(ret, addr, val, idx, memop, table_##NAME); \
1275+
} else { \
1276+
do_nonatomic_op_i64(ret, addr, val, idx, memop, NEW, \
1277+
tcg_gen_##OP##_i64); \
1278+
} \
1279+
} \
1280+
void tcg_gen_atomic_##NAME##_i128_chk(TCGv_i128 ret, TCGTemp *addr, \
1281+
TCGv_i128 val, TCGArg idx, \
1282+
MemOp memop, TCGType addr_type) \
1283+
{ \
1284+
tcg_debug_assert(addr_type == tcg_ctx->addr_type); \
1285+
tcg_debug_assert((memop & MO_SIZE) == MO_128); \
1286+
if (tcg_ctx->gen_tb->cflags & CF_PARALLEL) { \
1287+
do_atomic_op_i128(ret, addr, val, idx, memop, table_##NAME); \
1288+
} else { \
1289+
do_nonatomic_op_i128(ret, addr, val, idx, memop, NEW, \
1290+
tcg_gen_##OP##_i64); \
1291+
} \
1292+
}
1293+
12041294
#define GEN_ATOMIC_HELPER(NAME, OP, NEW) \
12051295
static void * const table_##NAME[(MO_SIZE | MO_BSWAP) + 1] = { \
12061296
[MO_8] = gen_helper_atomic_##NAME##b, \
@@ -1239,8 +1329,8 @@ void tcg_gen_atomic_##NAME##_i64_chk(TCGv_i64 ret, TCGTemp *addr, \
12391329
}
12401330

12411331
GEN_ATOMIC_HELPER(fetch_add, add, 0)
1242-
GEN_ATOMIC_HELPER(fetch_and, and, 0)
1243-
GEN_ATOMIC_HELPER(fetch_or, or, 0)
1332+
GEN_ATOMIC_HELPER128(fetch_and, and, 0)
1333+
GEN_ATOMIC_HELPER128(fetch_or, or, 0)
12441334
GEN_ATOMIC_HELPER(fetch_xor, xor, 0)
12451335
GEN_ATOMIC_HELPER(fetch_smin, smin, 0)
12461336
GEN_ATOMIC_HELPER(fetch_umin, umin, 0)
@@ -1266,6 +1356,7 @@ static void tcg_gen_mov2_i64(TCGv_i64 r, TCGv_i64 a, TCGv_i64 b)
12661356
tcg_gen_mov_i64(r, b);
12671357
}
12681358

1269-
GEN_ATOMIC_HELPER(xchg, mov2, 0)
1359+
GEN_ATOMIC_HELPER128(xchg, mov2, 0)
12701360

12711361
#undef GEN_ATOMIC_HELPER
1362+
#undef GEN_ATOMIC_HELPER128

0 commit comments

Comments
 (0)