@@ -334,9 +334,9 @@ static Value downcastSelectAndUpcast(OpBuilder &builder, Location loc,
334334// /
335335// / Result:
336336// / linearizedMemref = |2|2|3|3| : <4xi2> (<1xi8>)
337- static void atomicStore (OpBuilder &builder, Location loc,
338- MemRefValue linearizedMemref, Value storeIdx,
339- VectorValue valueToStore, Value mask) {
337+ static void atomicRMWStore (OpBuilder &builder, Location loc,
338+ MemRefValue linearizedMemref, Value storeIdx,
339+ VectorValue valueToStore, Value mask) {
340340 assert (valueToStore.getType ().getRank () == 1 && " expected 1-D vector" );
341341
342342 // Create an atomic load-modify-write region using
@@ -364,10 +364,11 @@ static void atomicStore(OpBuilder &builder, Location loc,
364364}
365365
366366// / Generate a non-atomic read-modify-write sequence for subbyte storing.
367- // / It has similar logic to `atomicStore`, but without atomicity.
368- static void rmwStore (OpBuilder &builder, Location loc,
369- MemRefValue linearizedMemref, Value linearizedIndex,
370- VectorValue valueToStore, Value mask) {
367+ // / It has similar logic to `atomicRMWStore`, but without atomicity.
368+ static void nonAtomicRMWStore (OpBuilder &builder, Location loc,
369+ MemRefValue linearizedMemref,
370+ Value linearizedIndex, VectorValue valueToStore,
371+ Value mask) {
371372 assert (valueToStore.getType ().getRank () == 1 && " expected 1-D vector" );
372373
373374 auto oneElemVecType =
@@ -580,8 +581,10 @@ struct ConvertVectorStore final : OpConversionPattern<vector::StoreOp> {
580581 extractSliceIntoByte (rewriter, loc, valueToStore, 0 ,
581582 frontSubWidthStoreElem, *foldedNumFrontPadElems);
582583
583- subEmulatedWidthStore (rewriter, loc, memrefBase, currentDestIndex,
584- cast<VectorValue>(value), frontMask.getResult ());
584+ auto storeFunc = useAtomicWrites_ ? atomicRMWStore : nonAtomicRMWStore;
585+
586+ storeFunc (rewriter, loc, memrefBase, currentDestIndex,
587+ cast<VectorValue>(value), frontMask.getResult ());
585588 }
586589
587590 if (currentSourceIndex >= origElements) {
@@ -645,20 +648,6 @@ struct ConvertVectorStore final : OpConversionPattern<vector::StoreOp> {
645648 return success ();
646649 }
647650
648- // / Store a subbyte-sized value to memory, with a mask. Depending on the
649- // / configuration, it could be an atomic store or a non-atomic RMW sequence.
650- template <typename ... Args>
651- void subEmulatedWidthStore (Args &&...args) const {
652- static_assert (
653- std::is_same_v<decltype (atomicStore), decltype (rmwStore)> &&
654- " `atomicStore` and `rmwStore` must have same signature, as per "
655- " the design to keep the code clean, which one to call is "
656- " determined by the `useAtomicWrites` flag." );
657- std::function<decltype (atomicStore)> storeFunc =
658- useAtomicWrites_ ? atomicStore : rmwStore;
659- storeFunc (std::forward<Args>(args)...);
660- }
661-
662651private:
663652 const bool useAtomicWrites_;
664653};
0 commit comments