diff --git a/src/binaryen-c.cpp b/src/binaryen-c.cpp index 0136bc999a1..b93eaa103e7 100644 --- a/src/binaryen-c.cpp +++ b/src/binaryen-c.cpp @@ -2633,12 +2633,14 @@ void BinaryenMemoryGrowSetDelta(BinaryenExpressionRef expr, bool BinaryenLoadIsAtomic(BinaryenExpressionRef expr) { auto* expression = (Expression*)expr; assert(expression->is()); - return static_cast(expression)->isAtomic; + return static_cast(expression)->isAtomic(); } + void BinaryenLoadSetAtomic(BinaryenExpressionRef expr, bool isAtomic) { auto* expression = (Expression*)expr; assert(expression->is()); - static_cast(expression)->isAtomic = isAtomic != 0; + static_cast(expression)->order = + isAtomic ? MemoryOrder::SeqCst : MemoryOrder::Unordered; } bool BinaryenLoadIsSigned(BinaryenExpressionRef expr) { auto* expression = (Expression*)expr; diff --git a/src/ir/cost.h b/src/ir/cost.h index 175cf722e2e..0c27e694e5c 100644 --- a/src/ir/cost.h +++ b/src/ir/cost.h @@ -101,7 +101,7 @@ struct CostAnalyzer : public OverriddenVisitor { CostType visitGlobalGet(GlobalGet* curr) { return 1; } CostType visitGlobalSet(GlobalSet* curr) { return 2 + visit(curr->value); } CostType visitLoad(Load* curr) { - return 1 + visit(curr->ptr) + AtomicCost * curr->isAtomic; + return 1 + visit(curr->ptr) + AtomicCost * curr->isAtomic(); } CostType visitStore(Store* curr) { return 2 + visit(curr->ptr) + visit(curr->value) + diff --git a/src/ir/effects.h b/src/ir/effects.h index cbab700b94e..1fce1b3ebc5 100644 --- a/src/ir/effects.h +++ b/src/ir/effects.h @@ -598,7 +598,7 @@ class EffectAnalyzer { } void visitLoad(Load* curr) { parent.readsMemory = true; - parent.isAtomic |= curr->isAtomic; + parent.isAtomic |= curr->isAtomic(); parent.implicitTrap = true; } void visitStore(Store* curr) { diff --git a/src/ir/load-utils.h b/src/ir/load-utils.h index 011bdcbdf08..43f03873602 100644 --- a/src/ir/load-utils.h +++ b/src/ir/load-utils.h @@ -33,7 +33,7 @@ inline bool isSignRelevant(Load* load) { } // check if a load can be signed (which some opts want to do) -inline bool canBeSigned(Load* load) { return !load->isAtomic; } +inline bool canBeSigned(Load* load) { return !load->isAtomic(); } } // namespace wasm::LoadUtils diff --git a/src/ir/properties.h b/src/ir/properties.h index 8b2938378aa..f65e2375eff 100644 --- a/src/ir/properties.h +++ b/src/ir/properties.h @@ -502,7 +502,7 @@ inline MemoryOrder getMemoryOrder(Expression* curr) { return set->order; } if (auto* load = curr->dynCast()) { - return load->isAtomic ? MemoryOrder::SeqCst : MemoryOrder::Unordered; + return load->order; } if (auto* store = curr->dynCast()) { return store->isAtomic ? MemoryOrder::SeqCst : MemoryOrder::Unordered; diff --git a/src/passes/I64ToI32Lowering.cpp b/src/passes/I64ToI32Lowering.cpp index 0cc6b2bd35b..96d4cd5b9bb 100644 --- a/src/passes/I64ToI32Lowering.cpp +++ b/src/passes/I64ToI32Lowering.cpp @@ -414,7 +414,7 @@ struct I64ToI32Lowering : public WalkerPass> { if (curr->type != Type::i64) { return; } - assert(!curr->isAtomic && "64-bit atomic load not implemented"); + assert(!curr->isAtomic() && "64-bit atomic load not implemented"); TempVar lowBits = getTemp(); TempVar highBits = getTemp(); TempVar ptrTemp = getTemp(); diff --git a/src/passes/OptimizeInstructions.cpp b/src/passes/OptimizeInstructions.cpp index 3af16ee9f56..9ef6ccd4716 100644 --- a/src/passes/OptimizeInstructions.cpp +++ b/src/passes/OptimizeInstructions.cpp @@ -1049,7 +1049,7 @@ struct OptimizeInstructions // extend operation. bool willBeSigned = curr->op == ExtendSInt32 && load->bytes == 4; if (!(curr->op == ExtendUInt32 && load->bytes <= 2 && load->signed_) && - !(willBeSigned && load->isAtomic)) { + !(willBeSigned && load->isAtomic())) { if (willBeSigned) { load->signed_ = true; } @@ -1092,7 +1092,7 @@ struct OptimizeInstructions // i32.reinterpret_f32(f32.load(x)) => i32.load(x) // i64.reinterpret_f64(f64.load(x)) => i64.load(x) if (auto* load = curr->value->dynCast()) { - if (!load->isAtomic && load->bytes == curr->type.getByteSize()) { + if (!load->isAtomic() && load->bytes == curr->type.getByteSize()) { load->type = curr->type; return replaceCurrent(load); } diff --git a/src/passes/PickLoadSigns.cpp b/src/passes/PickLoadSigns.cpp index 693fd4a68e6..f04f8451ca0 100644 --- a/src/passes/PickLoadSigns.cpp +++ b/src/passes/PickLoadSigns.cpp @@ -120,7 +120,7 @@ struct PickLoadSigns : public WalkerPass> { continue; } // Atomic operations are always unsigned, never signed. - if (load->isAtomic) { + if (load->isAtomic()) { continue; } // we can pick the optimal one. our hope is to remove 2 items per diff --git a/src/passes/Print.cpp b/src/passes/Print.cpp index 96fd3c3051f..c83cccbe5dd 100644 --- a/src/passes/Print.cpp +++ b/src/passes/Print.cpp @@ -549,7 +549,7 @@ struct PrintExpressionContents } void visitLoad(Load* curr) { prepareColor(o) << forceConcrete(curr->type, curr->align); - if (curr->isAtomic) { + if (curr->isAtomic()) { o << ".atomic"; } o << ".load"; diff --git a/src/passes/SafeHeap.cpp b/src/passes/SafeHeap.cpp index c8a2a1122f8..12e516f66f5 100644 --- a/src/passes/SafeHeap.cpp +++ b/src/passes/SafeHeap.cpp @@ -43,7 +43,7 @@ static Name getLoadName(Load* curr) { if (LoadUtils::isSignRelevant(curr) && !curr->signed_) { ret += "U_"; } - if (curr->isAtomic) { + if (curr->isAtomic()) { ret += "A"; } else { ret += std::to_string(curr->align); @@ -233,7 +233,8 @@ struct SafeHeap : public Pass { continue; } for (auto isAtomic : {true, false}) { - load.isAtomic = isAtomic; + load.order = + isAtomic ? MemoryOrder::SeqCst : MemoryOrder::Unordered; if (isAtomic && !isPossibleAtomicOperation( align, bytes, module->memories[0]->shared, type)) { @@ -321,7 +322,7 @@ struct SafeHeap : public Pass { *load = style; // basically the same as the template we are given! load->ptr = builder.makeLocalGet(2, addressType); Expression* last = load; - if (load->isAtomic && load->signed_) { + if (load->isAtomic() && load->signed_) { // atomic loads cannot be signed, manually sign it last = Bits::makeSignExt(load, load->bytes, *module); load->signed_ = false; diff --git a/src/tools/fuzzing/fuzzing.cpp b/src/tools/fuzzing/fuzzing.cpp index bbc324479ac..ef863fe016d 100644 --- a/src/tools/fuzzing/fuzzing.cpp +++ b/src/tools/fuzzing/fuzzing.cpp @@ -3239,7 +3239,7 @@ Expression* TranslateToFuzzReader::makeLoad(Type type) { // make it atomic auto* load = ret->cast(); wasm.memories[0]->shared = true; - load->isAtomic = true; + load->order = MemoryOrder::SeqCst; load->signed_ = false; load->align = load->bytes; return load; diff --git a/src/wasm-builder.h b/src/wasm-builder.h index d14e716f12b..4cdeeb0bffd 100644 --- a/src/wasm-builder.h +++ b/src/wasm-builder.h @@ -373,7 +373,6 @@ class Builder { Type type, Name memory) { auto* ret = wasm.allocator.alloc(); - ret->isAtomic = false; ret->bytes = bytes; ret->signed_ = signed_; ret->offset = offset; @@ -381,13 +380,14 @@ class Builder { ret->ptr = ptr; ret->type = type; ret->memory = memory; + ret->order = MemoryOrder::Unordered; ret->finalize(); return ret; } Load* makeAtomicLoad( unsigned bytes, Address offset, Expression* ptr, Type type, Name memory) { Load* load = makeLoad(bytes, false, offset, bytes, ptr, type, memory); - load->isAtomic = true; + load->order = MemoryOrder::SeqCst; return load; } AtomicWait* makeAtomicWait(Expression* ptr, diff --git a/src/wasm-delegations-fields.def b/src/wasm-delegations-fields.def index 376648311f3..c5df66a3439 100644 --- a/src/wasm-delegations-fields.def +++ b/src/wasm-delegations-fields.def @@ -354,7 +354,7 @@ DELEGATE_FIELD_INT(Load, bytes) DELEGATE_FIELD_INT(Load, signed_) DELEGATE_FIELD_ADDRESS(Load, offset) DELEGATE_FIELD_ADDRESS(Load, align) -DELEGATE_FIELD_INT(Load, isAtomic) +DELEGATE_FIELD_INT(Load, order) DELEGATE_FIELD_NAME_KIND(Load, memory, ModuleItemKind::Memory) DELEGATE_FIELD_CASE_END(Load) diff --git a/src/wasm-interpreter.h b/src/wasm-interpreter.h index d46dde4d102..3c0c77b0cec 100644 --- a/src/wasm-interpreter.h +++ b/src/wasm-interpreter.h @@ -3862,7 +3862,7 @@ class ModuleRunnerBase : public ExpressionRunner { auto memorySize = info.instance->getMemorySize(info.name); auto addr = info.instance->getFinalAddress(curr, flow.getSingleValue(), memorySize); - if (curr->isAtomic) { + if (curr->isAtomic()) { info.instance->checkAtomicAddress(addr, curr->bytes, memorySize); } auto ret = info.interface()->load(curr, addr, info.name); @@ -3995,7 +3995,7 @@ class ModuleRunnerBase : public ExpressionRunner { load.signed_ = false; load.offset = curr->offset; load.align = curr->align; - load.isAtomic = false; + load.order = MemoryOrder::Unordered; load.ptr = curr->ptr; Literal (Literal::*splat)() const = nullptr; switch (curr->op) { @@ -5023,7 +5023,7 @@ class ModuleRunnerBase : public ExpressionRunner { // always an unsigned extension. load.signed_ = false; load.align = bytes; - load.isAtomic = true; // understatement + load.order = MemoryOrder::SeqCst; load.ptr = &ptr; load.type = type; load.memory = memoryName; diff --git a/src/wasm.h b/src/wasm.h index c7677d01412..3e08a99e3d2 100644 --- a/src/wasm.h +++ b/src/wasm.h @@ -993,9 +993,11 @@ class Load : public SpecificExpression { bool signed_ = false; Address offset; Address align; - bool isAtomic; Expression* ptr; Name memory; + MemoryOrder order; + + bool isAtomic() const { return order != MemoryOrder::Unordered; } // type must be set during creation, cannot be inferred @@ -2664,6 +2666,7 @@ std::ostream& operator<<(std::ostream& o, wasm::ModuleExpression pair); std::ostream& operator<<(std::ostream& o, wasm::ShallowExpression expression); std::ostream& operator<<(std::ostream& o, wasm::ModuleType pair); std::ostream& operator<<(std::ostream& o, wasm::ModuleHeapType pair); +std::ostream& operator<<(std::ostream& os, wasm::MemoryOrder mo); } // namespace std diff --git a/src/wasm/wasm-stack.cpp b/src/wasm/wasm-stack.cpp index f1d26774680..1ec8ba7e619 100644 --- a/src/wasm/wasm-stack.cpp +++ b/src/wasm/wasm-stack.cpp @@ -248,7 +248,7 @@ void BinaryInstWriter::visitGlobalSet(GlobalSet* curr) { } void BinaryInstWriter::visitLoad(Load* curr) { - if (!curr->isAtomic) { + if (!curr->isAtomic()) { switch (curr->type.getBasic()) { case Type::i32: { switch (curr->bytes) { diff --git a/src/wasm/wasm-validator.cpp b/src/wasm/wasm-validator.cpp index 62efe49f007..59f755a19ef 100644 --- a/src/wasm/wasm-validator.cpp +++ b/src/wasm/wasm-validator.cpp @@ -1095,7 +1095,7 @@ void FunctionValidator::visitGlobalSet(GlobalSet* curr) { void FunctionValidator::visitLoad(Load* curr) { auto* memory = getModule()->getMemoryOrNull(curr->memory); shouldBeTrue(!!memory, curr, "memory.load memory must exist"); - if (curr->isAtomic) { + if (curr->isAtomic()) { shouldBeTrue(getModule()->features.hasAtomics(), curr, "Atomic operations require threads [--enable-threads]"); @@ -1111,13 +1111,14 @@ void FunctionValidator::visitLoad(Load* curr) { } validateMemBytes(curr->bytes, curr->type, curr); validateOffset(curr->offset, memory, curr); - validateAlignment(curr->align, curr->type, curr->bytes, curr->isAtomic, curr); + validateAlignment( + curr->align, curr->type, curr->bytes, curr->isAtomic(), curr); shouldBeEqualOrFirstIsUnreachable( curr->ptr->type, memory->addressType, curr, "load pointer type must match memory index type"); - if (curr->isAtomic) { + if (curr->isAtomic()) { shouldBeFalse(curr->signed_, curr, "atomic loads must be unsigned"); shouldBeIntOrUnreachable( curr->type, curr, "atomic loads must be of integers"); diff --git a/src/wasm/wasm.cpp b/src/wasm/wasm.cpp index 44c1daea622..111f4c9d21d 100644 --- a/src/wasm/wasm.cpp +++ b/src/wasm/wasm.cpp @@ -2015,3 +2015,22 @@ void Module::clearDebugInfo() { } } // namespace wasm + +namespace std { + +std::ostream& operator<<(std::ostream& os, wasm::MemoryOrder mo) { + switch (mo) { + case wasm::MemoryOrder::Unordered: + os << "Unordered"; + break; + case wasm::MemoryOrder::SeqCst: + os << "SeqCst"; + break; + case wasm::MemoryOrder::AcqRel: + os << "AcqRel"; + break; + } + return os; +} + +} // namespace std diff --git a/src/wasm2js.h b/src/wasm2js.h index d6e0e4d6560..5725b3b0d76 100644 --- a/src/wasm2js.h +++ b/src/wasm2js.h @@ -1468,7 +1468,7 @@ Ref Wasm2JSBuilder::processExpression(Expression* curr, Fatal() << "Unhandled type in load: " << curr->type; } } - if (curr->isAtomic) { + if (curr->isAtomic()) { Ref call = ValueBuilder::makeCall( ValueBuilder::makeDot(ValueBuilder::makeName(ATOMICS), LOAD)); ValueBuilder::appendToCall(call, ret[1]);