Skip to content

Commit d61ce5a

Browse files
vtjnashKristofferC
vtjnash
authored and
KristofferC
committed
non-power-of-two atomic handling (#41369)
(cherry picked from commit 252ae69)
1 parent 8b6a7ea commit d61ce5a

File tree

4 files changed

+241
-141
lines changed

4 files changed

+241
-141
lines changed

‎src/cgutils.cpp‎

Lines changed: 63 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -1479,12 +1479,19 @@ static jl_cgval_t typed_load(jl_codectx_t &ctx, Value *ptr, Value *idx_0based, j
14791479
if (type_is_ghost(elty))
14801480
return ghostValue(jltype);
14811481
AllocaInst *intcast = NULL;
1482-
if (!isboxed && Order != AtomicOrdering::NotAtomic && !elty->isIntOrPtrTy() && !elty->isFloatingPointTy()) {
1482+
if (!isboxed && Order != AtomicOrdering::NotAtomic && !elty->isIntOrPtrTy()) {
14831483
const DataLayout &DL = jl_data_layout;
14841484
unsigned nb = DL.getTypeSizeInBits(elty);
14851485
intcast = ctx.builder.CreateAlloca(elty);
14861486
elty = Type::getIntNTy(jl_LLVMContext, nb);
14871487
}
1488+
Type *realelty = elty;
1489+
if (Order != AtomicOrdering::NotAtomic && isa<IntegerType>(elty)) {
1490+
unsigned nb = cast<IntegerType>(elty)->getBitWidth();
1491+
unsigned nb2 = PowerOf2Ceil(nb);
1492+
if (nb != nb2)
1493+
elty = Type::getIntNTy(jl_LLVMContext, nb2);
1494+
}
14881495
Type *ptrty = PointerType::get(elty, ptr->getType()->getPointerAddressSpace());
14891496
Value *data;
14901497
if (ptr->getType() != ptrty)
@@ -1493,7 +1500,7 @@ static jl_cgval_t typed_load(jl_codectx_t &ctx, Value *ptr, Value *idx_0based, j
14931500
data = ptr;
14941501
if (idx_0based)
14951502
data = ctx.builder.CreateInBoundsGEP(elty, data, idx_0based);
1496-
Instruction *load;
1503+
Value *instr;
14971504
// TODO: can only lazy load if we can create a gc root for ptr for the lifetime of elt
14981505
//if (elty->isAggregateType() && tbaa == tbaa_immut && !alignment) { // can lazy load on demand, no copy needed
14991506
// elt = data;
@@ -1503,20 +1510,23 @@ static jl_cgval_t typed_load(jl_codectx_t &ctx, Value *ptr, Value *idx_0based, j
15031510
alignment = sizeof(void*);
15041511
else if (!alignment)
15051512
alignment = julia_alignment(jltype);
1506-
load = ctx.builder.CreateAlignedLoad(data, Align(alignment), false);
1507-
cast<LoadInst>(load)->setOrdering(Order);
1513+
LoadInst *load = ctx.builder.CreateAlignedLoad(data, Align(alignment), false);
1514+
load->setOrdering(Order);
15081515
if (aliasscope)
15091516
load->setMetadata("alias.scope", aliasscope);
15101517
if (isboxed)
1511-
load = maybe_mark_load_dereferenceable(load, true, jltype);
1518+
maybe_mark_load_dereferenceable(load, true, jltype);
15121519
if (tbaa)
1513-
load = tbaa_decorate(tbaa, load);
1520+
tbaa_decorate(tbaa, load);
1521+
instr = load;
1522+
if (elty != realelty)
1523+
instr = ctx.builder.CreateTrunc(instr, realelty);
15141524
if (intcast) {
1515-
ctx.builder.CreateStore(load, ctx.builder.CreateBitCast(intcast, load->getType()->getPointerTo()));
1516-
load = ctx.builder.CreateLoad(intcast);
1525+
ctx.builder.CreateStore(instr, ctx.builder.CreateBitCast(intcast, instr->getType()->getPointerTo()));
1526+
instr = ctx.builder.CreateLoad(intcast);
15171527
}
15181528
if (maybe_null_if_boxed) {
1519-
Value *first_ptr = isboxed ? load : extract_first_ptr(ctx, load);
1529+
Value *first_ptr = isboxed ? instr : extract_first_ptr(ctx, instr);
15201530
if (first_ptr)
15211531
null_pointer_check(ctx, first_ptr, nullcheck);
15221532
}
@@ -1526,9 +1536,9 @@ static jl_cgval_t typed_load(jl_codectx_t &ctx, Value *ptr, Value *idx_0based, j
15261536
//load->setMetadata(LLVMContext::MD_range, MDNode::get(jl_LLVMContext, {
15271537
// ConstantAsMetadata::get(ConstantInt::get(T_int8, 0)),
15281538
// ConstantAsMetadata::get(ConstantInt::get(T_int8, 2)) }));
1529-
load = ctx.builder.Insert(CastInst::Create(Instruction::Trunc, load, T_int1));
1539+
instr = ctx.builder.CreateTrunc(instr, T_int1);
15301540
}
1531-
return mark_julia_type(ctx, load, isboxed, jltype);
1541+
return mark_julia_type(ctx, instr, isboxed, jltype);
15321542
}
15331543

15341544
static jl_cgval_t typed_store(jl_codectx_t &ctx,
@@ -1544,18 +1554,27 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx,
15441554
if (type_is_ghost(elty))
15451555
return oldval;
15461556
Value *intcast = nullptr;
1547-
if (!isboxed && Order != AtomicOrdering::NotAtomic && !elty->isIntOrPtrTy() && !elty->isFloatingPointTy()) {
1557+
if (!isboxed && Order != AtomicOrdering::NotAtomic && !elty->isIntOrPtrTy()) {
15481558
const DataLayout &DL = jl_data_layout;
15491559
unsigned nb = DL.getTypeSizeInBits(elty);
15501560
if (!issetfield)
15511561
intcast = ctx.builder.CreateAlloca(elty);
15521562
elty = Type::getIntNTy(jl_LLVMContext, nb);
15531563
}
1564+
Type *realelty = elty;
1565+
if (Order != AtomicOrdering::NotAtomic && isa<IntegerType>(elty)) {
1566+
unsigned nb = cast<IntegerType>(elty)->getBitWidth();
1567+
unsigned nb2 = PowerOf2Ceil(nb);
1568+
if (nb != nb2)
1569+
elty = Type::getIntNTy(jl_LLVMContext, nb2);
1570+
}
15541571
Value *r;
15551572
if (!isboxed)
1556-
r = emit_unbox(ctx, elty, rhs, jltype);
1573+
r = emit_unbox(ctx, realelty, rhs, jltype);
15571574
else
15581575
r = boxed(ctx, rhs);
1576+
if (realelty != elty)
1577+
r = ctx.builder.CreateZExt(r, elty);
15591578
Type *ptrty = PointerType::get(elty, ptr->getType()->getPointerAddressSpace());
15601579
if (ptr->getType() != ptrty)
15611580
ptr = ctx.builder.CreateBitCast(ptr, ptrty);
@@ -1578,18 +1597,19 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx,
15781597
instr->setMetadata("noalias", aliasscope);
15791598
if (tbaa)
15801599
tbaa_decorate(tbaa, instr);
1581-
}
1582-
if (isreplacefield) {
1583-
oldval = mark_julia_type(ctx, instr, isboxed, jltype);
1584-
Value *first_ptr = nullptr;
1585-
if (maybe_null_if_boxed)
1586-
first_ptr = isboxed ? instr : extract_first_ptr(ctx, instr);
1587-
Success = emit_nullcheck_guard(ctx, first_ptr, [&] {
1588-
return emit_f_is(ctx, oldval, cmp);
1589-
});
1590-
BasicBlock *BB = BasicBlock::Create(jl_LLVMContext, "xchg", ctx.f);
1591-
ctx.builder.CreateCondBr(Success, BB, DoneBB);
1592-
ctx.builder.SetInsertPoint(BB);
1600+
assert(realelty == elty);
1601+
if (isreplacefield) {
1602+
oldval = mark_julia_type(ctx, instr, isboxed, jltype);
1603+
Value *first_ptr = nullptr;
1604+
if (maybe_null_if_boxed)
1605+
first_ptr = isboxed ? instr : extract_first_ptr(ctx, instr);
1606+
Success = emit_nullcheck_guard(ctx, first_ptr, [&] {
1607+
return emit_f_is(ctx, oldval, cmp);
1608+
});
1609+
BasicBlock *BB = BasicBlock::Create(jl_LLVMContext, "xchg", ctx.f);
1610+
ctx.builder.CreateCondBr(Success, BB, DoneBB);
1611+
ctx.builder.SetInsertPoint(BB);
1612+
}
15931613
}
15941614
StoreInst *store = ctx.builder.CreateAlignedStore(r, ptr, Align(alignment));
15951615
store->setOrdering(Order);
@@ -1628,7 +1648,9 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx,
16281648
Current->addIncoming(instr, SkipBB);
16291649
ctx.builder.SetInsertPoint(BB);
16301650
}
1631-
Compare = emit_unbox(ctx, elty, cmp, jltype);
1651+
Compare = emit_unbox(ctx, realelty, cmp, jltype);
1652+
if (realelty != elty)
1653+
Compare = ctx.builder.CreateZExt(Compare, elty);
16321654
}
16331655
else if (cmp.isboxed) {
16341656
Compare = boxed(ctx, cmp);
@@ -1676,21 +1698,26 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx,
16761698
if (tbaa)
16771699
tbaa_decorate(tbaa, store);
16781700
instr = ctx.builder.Insert(ExtractValueInst::Create(store, 0));
1679-
Success = ctx.builder.CreateExtractValue(store, 1);
1701+
Success = ctx.builder.Insert(ExtractValueInst::Create(store, 1));
16801702
Value *Done = Success;
16811703
if (needloop) {
16821704
if (isreplacefield) {
1705+
Value *realinstr = instr;
1706+
if (realelty != elty)
1707+
realinstr = ctx.builder.CreateTrunc(instr, realelty);
16831708
if (intcast) {
1684-
ctx.builder.CreateStore(instr, ctx.builder.CreateBitCast(intcast, instr->getType()->getPointerTo()));
1709+
ctx.builder.CreateStore(realinstr, ctx.builder.CreateBitCast(intcast, realinstr->getType()->getPointerTo()));
16851710
oldval = mark_julia_slot(intcast, jltype, NULL, tbaa_stack);
1711+
if (maybe_null_if_boxed)
1712+
realinstr = ctx.builder.CreateLoad(intcast);
16861713
}
16871714
else {
1688-
oldval = mark_julia_type(ctx, instr, isboxed, jltype);
1715+
oldval = mark_julia_type(ctx, realinstr, isboxed, jltype);
16891716
}
16901717
Done = emit_guarded_test(ctx, ctx.builder.CreateNot(Success), false, [&] {
16911718
Value *first_ptr = nullptr;
16921719
if (maybe_null_if_boxed)
1693-
first_ptr = isboxed ? instr : extract_first_ptr(ctx, instr);
1720+
first_ptr = isboxed ? realinstr : extract_first_ptr(ctx, realinstr);
16941721
return emit_nullcheck_guard(ctx, first_ptr, [&] {
16951722
return emit_f_is(ctx, oldval, cmp);
16961723
});
@@ -1747,6 +1774,8 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx,
17471774
}
17481775
}
17491776
if (!issetfield) {
1777+
if (realelty != elty)
1778+
instr = ctx.builder.Insert(CastInst::Create(Instruction::Trunc, instr, realelty));
17501779
if (intcast) {
17511780
ctx.builder.CreateStore(instr, ctx.builder.CreateBitCast(intcast, instr->getType()->getPointerTo()));
17521781
instr = ctx.builder.CreateLoad(intcast);
@@ -2053,6 +2082,9 @@ static jl_cgval_t emit_getfield_knownidx(jl_codectx_t &ctx, const jl_cgval_t &st
20532082
emit_atomic_error(ctx, "getfield: atomic field cannot be accessed non-atomically");
20542083
return jl_cgval_t(); // unreachable
20552084
}
2085+
if (order == jl_memory_order_unspecified) {
2086+
order = isatomic ? jl_memory_order_unordered : jl_memory_order_notatomic;
2087+
}
20562088
if (jfty == jl_bottom_type) {
20572089
raise_exception(ctx, literal_pointer_val(ctx, jl_undefref_exception));
20582090
return jl_cgval_t(); // unreachable
@@ -2126,7 +2158,7 @@ static jl_cgval_t emit_getfield_knownidx(jl_codectx_t &ctx, const jl_cgval_t &st
21262158
if (needlock)
21272159
emit_lockstate_value(ctx, strct, true);
21282160
jl_cgval_t ret = typed_load(ctx, addr, NULL, jfty, tbaa, nullptr, false,
2129-
needlock || order <= jl_memory_order_notatomic ? AtomicOrdering::NotAtomic : get_llvm_atomic_order(order), // TODO: we should use unordered for anything with CountTrackedPointers(elty).count > 0
2161+
needlock ? AtomicOrdering::NotAtomic : get_llvm_atomic_order(order), // TODO: we should use unordered for anything with CountTrackedPointers(elty).count > 0
21302162
maybe_null, align, nullcheck);
21312163
if (needlock)
21322164
emit_lockstate_value(ctx, strct, false);

0 commit comments

Comments
 (0)