Skip to content

Commit 17698eb

Browse files
bruteforceboybruteforceboy
committed
Merge remote-tracking branch 'origin/main' into std-vector-op
2 parents 9939a0f + b647f4b commit 17698eb

16 files changed

+800
-9
lines changed

‎clang/lib/CIR/CodeGen/CIRGenBuilder.h

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -895,6 +895,34 @@ class CIRGenBuilderTy : public cir::CIRBaseBuilderTy {
895895
return CIRBaseBuilderTy::createStore(loc, flag, dst);
896896
}
897897

898+
/// Create a call to a Masked Load intrinsic.
899+
/// \p loc - expression location
900+
/// \p ty - vector type to load
901+
/// \p ptr - base pointer for the load
902+
/// \p alignment - alignment of the source location
903+
/// \p mask - vector of booleans which indicates what vector lanes should
904+
/// be accessed in memory
905+
/// \p passThru - pass-through value that is used to fill the masked-off
906+
/// lanes
907+
/// of the result
908+
mlir::Value createMaskedLoad(mlir::Location loc, mlir::Type ty,
909+
mlir::Value ptr, llvm::Align alignment,
910+
mlir::Value mask, mlir::Value passThru) {
911+
912+
assert(mlir::isa<cir::VectorType>(ty) && "Type should be vector");
913+
assert(mask && "Mask should not be all-ones (null)");
914+
915+
if (!passThru)
916+
passThru = this->getConstant(loc, cir::PoisonAttr::get(ty));
917+
918+
mlir::Value ops[] = {ptr, this->getUInt32(int32_t(alignment.value()), loc),
919+
mask, passThru};
920+
921+
return create<cir::LLVMIntrinsicCallOp>(loc, getStringAttr("masked.load"),
922+
ty, ops)
923+
.getResult();
924+
}
925+
898926
/// Create a call to a masked store intrinsic.
899927
/// \p loc - expression location
900928
/// \p val - data to be stored

‎clang/lib/CIR/CodeGen/CIRGenBuiltinX86.cpp

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,19 @@ static mlir::Value emitX86MaskedStore(CIRGenFunction &cgf,
105105
maskVec);
106106
}
107107

108+
static mlir::Value emitX86MaskedLoad(CIRGenFunction &cgf,
109+
ArrayRef<mlir::Value> ops,
110+
llvm::Align alignment,
111+
mlir::Location loc) {
112+
mlir::Type ty = ops[1].getType();
113+
mlir::Value ptr = ops[0];
114+
mlir::Value maskVec =
115+
getMaskVecValue(cgf, ops[2], cast<cir::VectorType>(ty).getSize(), loc);
116+
117+
return cgf.getBuilder().createMaskedLoad(loc, ty, ptr, alignment, maskVec,
118+
ops[1]);
119+
}
120+
108121
static mlir::Value emitX86SExtMask(CIRGenFunction &cgf, mlir::Value op,
109122
mlir::Type dstTy, mlir::Location loc) {
110123
unsigned numberOfElements = cast<cir::VectorType>(dstTy).getSize();
@@ -586,13 +599,15 @@ mlir::Value CIRGenFunction::emitX86BuiltinExpr(unsigned BuiltinID,
586599
case X86::BI__builtin_ia32_loaddqudi128_mask:
587600
case X86::BI__builtin_ia32_loaddqudi256_mask:
588601
case X86::BI__builtin_ia32_loaddqudi512_mask:
589-
llvm_unreachable("vfmaddsubph256_round_mask3 NYI");
602+
return emitX86MaskedLoad(*this, Ops, llvm::Align(1),
603+
getLoc(E->getExprLoc()));
590604

591605
case X86::BI__builtin_ia32_loadsbf16128_mask:
592606
case X86::BI__builtin_ia32_loadsh128_mask:
593607
case X86::BI__builtin_ia32_loadss128_mask:
594608
case X86::BI__builtin_ia32_loadsd128_mask:
595-
llvm_unreachable("vfmaddsubph256_round_mask3 NYI");
609+
return emitX86MaskedLoad(*this, Ops, llvm::Align(1),
610+
getLoc(E->getExprLoc()));
596611

597612
case X86::BI__builtin_ia32_loadaps128_mask:
598613
case X86::BI__builtin_ia32_loadaps256_mask:
@@ -606,7 +621,10 @@ mlir::Value CIRGenFunction::emitX86BuiltinExpr(unsigned BuiltinID,
606621
case X86::BI__builtin_ia32_movdqa64load128_mask:
607622
case X86::BI__builtin_ia32_movdqa64load256_mask:
608623
case X86::BI__builtin_ia32_movdqa64load512_mask:
609-
llvm_unreachable("vfmaddsubph256_round_mask3 NYI");
624+
return emitX86MaskedLoad(
625+
*this, Ops,
626+
getContext().getTypeAlignInChars(E->getArg(1)->getType()).getAsAlign(),
627+
getLoc(E->getExprLoc()));
610628

611629
case X86::BI__builtin_ia32_expandloaddf128_mask:
612630
case X86::BI__builtin_ia32_expandloaddf256_mask:

‎clang/lib/CIR/CodeGen/CIRGenCXXABI.h

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -182,6 +182,11 @@ class CIRGenCXXABI {
182182
virtual void registerGlobalDtor(CIRGenFunction &CGF, const VarDecl *D,
183183
cir::FuncOp dtor, mlir::Value Addr) = 0;
184184

185+
virtual void emitVirtualObjectDelete(CIRGenFunction &CGF,
186+
const CXXDeleteExpr *DE, Address Ptr,
187+
QualType ElementType,
188+
const CXXDestructorDecl *Dtor) = 0;
189+
185190
virtual size_t getSrcArgforCopyCtor(const CXXConstructorDecl *,
186191
FunctionArgList &Args) const = 0;
187192

@@ -212,6 +217,15 @@ class CIRGenCXXABI {
212217
virtual void emitVTableDefinitions(CIRGenVTables &CGVT,
213218
const CXXRecordDecl *RD) = 0;
214219

220+
using DeleteOrMemberCallExpr =
221+
llvm::PointerUnion<const CXXDeleteExpr *, const CXXMemberCallExpr *>;
222+
223+
virtual mlir::Value emitVirtualDestructorCall(CIRGenFunction &CGF,
224+
const CXXDestructorDecl *Dtor,
225+
CXXDtorType DtorType,
226+
Address This,
227+
DeleteOrMemberCallExpr E) = 0;
228+
215229
/// Emit any tables needed to implement virtual inheritance. For Itanium,
216230
/// this emits virtual table tables.
217231
virtual void emitVirtualInheritanceTables(const CXXRecordDecl *RD) = 0;

‎clang/lib/CIR/CodeGen/CIRGenExprCXX.cpp

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1227,7 +1227,8 @@ static bool EmitObjectDelete(CIRGenFunction &CGF, const CXXDeleteExpr *DE,
12271227
}
12281228
}
12291229
if (UseVirtualCall) {
1230-
llvm_unreachable("NYI");
1230+
CGF.CGM.getCXXABI().emitVirtualObjectDelete(CGF, DE, Ptr, ElementType,
1231+
Dtor);
12311232
return false;
12321233
}
12331234
}
@@ -1241,7 +1242,9 @@ static bool EmitObjectDelete(CIRGenFunction &CGF, const CXXDeleteExpr *DE,
12411242
NormalAndEHCleanup, Ptr.getPointer(), OperatorDelete, ElementType);
12421243

12431244
if (Dtor) {
1244-
llvm_unreachable("NYI");
1245+
CGF.emitCXXDestructorCall(Dtor, Dtor_Complete,
1246+
/*ForVirtualBase=*/false,
1247+
/*Delegating=*/false, Ptr, ElementType);
12451248
} else if (auto Lifetime = ElementType.getObjCLifetime()) {
12461249
switch (Lifetime) {
12471250
case Qualifiers::OCL_None:

‎clang/lib/CIR/CodeGen/CIRGenExprComplex.cpp

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -431,8 +431,14 @@ mlir::Value ComplexExprEmitter::emitCast(CastKind CK, Expr *Op,
431431
case CK_LValueBitCast:
432432
llvm_unreachable("NYI");
433433

434-
case CK_LValueToRValueBitCast:
435-
llvm_unreachable("NYI");
434+
case CK_LValueToRValueBitCast: {
435+
LValue SourceLVal = CGF.emitLValue(Op);
436+
Address Addr = SourceLVal.getAddress().withElementType(
437+
Builder, CGF.convertTypeForMem(DestTy));
438+
LValue DestLV = CGF.makeAddrLValue(Addr, DestTy);
439+
DestLV.setTBAAInfo(TBAAAccessInfo::getMayAliasInfo());
440+
return emitLoadOfLValue(DestLV, Op->getExprLoc());
441+
}
436442

437443
case CK_BitCast:
438444
case CK_BaseToDerived:

‎clang/lib/CIR/CodeGen/CIRGenItaniumCXXABI.cpp

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -187,6 +187,9 @@ class CIRGenItaniumCXXABI : public CIRGenCXXABI {
187187
QualType ThisTy) override;
188188
void registerGlobalDtor(CIRGenFunction &CGF, const VarDecl *D,
189189
cir::FuncOp dtor, mlir::Value Addr) override;
190+
void emitVirtualObjectDelete(CIRGenFunction &CGF, const CXXDeleteExpr *DE,
191+
Address Ptr, QualType ElementType,
192+
const CXXDestructorDecl *Dtor) override;
190193
virtual void emitRethrow(CIRGenFunction &CGF, bool isNoReturn) override;
191194
virtual void emitThrow(CIRGenFunction &CGF, const CXXThrowExpr *E) override;
192195
CatchTypeInfo
@@ -205,6 +208,10 @@ class CIRGenItaniumCXXABI : public CIRGenCXXABI {
205208
CIRGenCallee getVirtualFunctionPointer(CIRGenFunction &CGF, GlobalDecl GD,
206209
Address This, mlir::Type Ty,
207210
SourceLocation Loc) override;
211+
mlir::Value emitVirtualDestructorCall(CIRGenFunction &CGF,
212+
const CXXDestructorDecl *Dtor,
213+
CXXDtorType DtorType, Address This,
214+
DeleteOrMemberCallExpr E) override;
208215
mlir::Value getVTableAddressPoint(BaseSubobject Base,
209216
const CXXRecordDecl *VTableClass) override;
210217
mlir::Value getVTableAddressPointInStructorWithVTT(
@@ -2186,6 +2193,32 @@ void CIRGenItaniumCXXABI::emitVTableDefinitions(CIRGenVTables &CGVT,
21862193
llvm_unreachable("NYI");
21872194
}
21882195

2196+
mlir::Value CIRGenItaniumCXXABI::emitVirtualDestructorCall(
2197+
CIRGenFunction &CGF, const CXXDestructorDecl *dtor, CXXDtorType dtorType,
2198+
Address thisAddr, DeleteOrMemberCallExpr expr) {
2199+
auto *callExpr = dyn_cast<const CXXMemberCallExpr *>(expr);
2200+
auto *delExpr = dyn_cast<const CXXDeleteExpr *>(expr);
2201+
assert((callExpr != nullptr) ^ (delExpr != nullptr));
2202+
assert(callExpr == nullptr || callExpr->arg_begin() == callExpr->arg_end());
2203+
assert(dtorType == Dtor_Deleting || dtorType == Dtor_Complete);
2204+
2205+
GlobalDecl globalDecl(dtor, dtorType);
2206+
const CIRGenFunctionInfo *fnInfo =
2207+
&CGM.getTypes().arrangeCXXStructorDeclaration(globalDecl);
2208+
auto fnTy = CGF.CGM.getTypes().GetFunctionType(*fnInfo);
2209+
auto callee = CIRGenCallee::forVirtual(callExpr, globalDecl, thisAddr, fnTy);
2210+
2211+
QualType thisTy;
2212+
if (callExpr)
2213+
thisTy = callExpr->getObjectType();
2214+
else
2215+
thisTy = delExpr->getDestroyedType();
2216+
2217+
CGF.emitCXXDestructorCall(globalDecl, callee, thisAddr.emitRawPointer(),
2218+
thisTy, nullptr, QualType(), nullptr);
2219+
return nullptr;
2220+
}
2221+
21892222
void CIRGenItaniumCXXABI::emitVirtualInheritanceTables(
21902223
const CXXRecordDecl *RD) {
21912224
CIRGenVTables &VTables = CGM.getVTables();
@@ -2716,6 +2749,22 @@ bool CIRGenItaniumCXXABI::isZeroInitializable(const MemberPointerType *MPT) {
27162749
return MPT->isMemberFunctionPointer();
27172750
}
27182751

2752+
/// The Itanium ABI always places an offset to the complete object
2753+
/// at entry -2 in the vtable.
2754+
void CIRGenItaniumCXXABI::emitVirtualObjectDelete(
2755+
CIRGenFunction &CGF, const CXXDeleteExpr *delExpr, Address ptr,
2756+
QualType elementType, const CXXDestructorDecl *dtor) {
2757+
bool useGlobalDelete = delExpr->isGlobalDelete();
2758+
if (useGlobalDelete)
2759+
llvm_unreachable("NYI");
2760+
2761+
CXXDtorType dtorType = useGlobalDelete ? Dtor_Complete : Dtor_Deleting;
2762+
emitVirtualDestructorCall(CGF, dtor, dtorType, ptr, delExpr);
2763+
2764+
if (useGlobalDelete)
2765+
llvm_unreachable("NYI");
2766+
}
2767+
27192768
/************************** Array allocation cookies **************************/
27202769

27212770
CharUnits CIRGenItaniumCXXABI::getArrayCookieSizeImpl(QualType ElementType) {

‎clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2063,8 +2063,14 @@ mlir::LogicalResult CIRToLLVMVecCmpOpLowering::matchAndRewrite(
20632063
} else {
20642064
return op.emitError() << "unsupported type for VecCmpOp: " << elementType;
20652065
}
2066-
rewriter.replaceOpWithNewOp<mlir::LLVM::SExtOp>(
2067-
op, typeConverter->convertType(op.getType()), bitResult);
2066+
2067+
// Check if the types are the same before generating SExtOp
2068+
auto targetType = typeConverter->convertType(op.getType());
2069+
if (bitResult.getType() == targetType)
2070+
rewriter.replaceOp(op, bitResult);
2071+
else
2072+
rewriter.replaceOpWithNewOp<mlir::LLVM::SExtOp>(op, targetType, bitResult);
2073+
20682074
return mlir::success();
20692075
}
20702076

‎clang/test/CIR/CodeGen/X86/avx10_2bf16-builtins.c

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,3 +13,22 @@ void test_mm_mask_store_sbh(void *__P, __mmask8 __U, __m128bh __A) {
1313
// LLVM: call void @llvm.masked.store.v8bf16.p0(<8 x bfloat> %{{.*}}, ptr %{{.*}}, i32 1, <8 x i1> %{{.*}})
1414
_mm_mask_store_sbh(__P, __U, __A);
1515
}
16+
17+
__m128bh test_mm_load_sbh(void const *A) {
18+
// CIR-LABEL: _mm_load_sbh
19+
// CIR: %{{.*}} = cir.llvm.intrinsic "masked.load" %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}} : (!cir.ptr<!cir.vector<!cir.bf16 x 8>>, !u32i, !cir.vector<!cir.int<s, 1> x 8>, !cir.vector<!cir.bf16 x 8>) -> !cir.vector<!cir.bf16 x 8>
20+
21+
// LLVM-LABEL: @test_mm_load_sbh
22+
// NOTE: OG represents the mask using a bitcast from splat (i8 1), see IR-differences #1767
23+
// LLVM: %{{.*}} = call <8 x bfloat> @llvm.masked.load.v8bf16.p0(ptr %{{.*}}, i32 1, <8 x i1> <i1 true, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false>, <8 x bfloat> %{{.*}})
24+
return _mm_load_sbh(A);
25+
}
26+
27+
__m128bh test_mm_mask_load_sbh(__m128bh __A, __mmask8 __U, const void *__W) {
28+
// CIR-LABEL: _mm_mask_load_sbh
29+
// CIR: %{{.*}} = cir.llvm.intrinsic "masked.load" %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}} : (!cir.ptr<!cir.vector<!cir.bf16 x 8>>, !u32i, !cir.vector<!cir.int<s, 1> x 8>, !cir.vector<!cir.bf16 x 8>) -> !cir.vector<!cir.bf16 x 8>
30+
31+
// LLVM-LABEL: @test_mm_mask_load_sbh
32+
// LLVM: %{{.*}} = call <8 x bfloat> @llvm.masked.load.v8bf16.p0(ptr %{{.*}}, i32 1, <8 x i1> %{{.*}}, <8 x bfloat> %{{.*}})
33+
return _mm_mask_load_sbh(__A, __U, __W);
34+
}

‎clang/test/CIR/CodeGen/X86/avx512bw-builtins.c

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,3 +37,39 @@ __m512i test_mm512_movm_epi16(__mmask32 __A) {
3737
// LLVM: %{{.*}} = sext <32 x i1> %{{.*}} to <32 x i16>
3838
return _mm512_movm_epi16(__A);
3939
}
40+
41+
__m512i test_mm512_mask_loadu_epi8(__m512i __W, __mmask64 __U, void const *__P) {
42+
// CIR-LABEL: _mm512_mask_loadu_epi8
43+
// CIR: %{{.*}} = cir.llvm.intrinsic "masked.load" %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}} : (!cir.ptr<!cir.vector<{{!s8i|!u8i}} x 64>>, !u32i, !cir.vector<!cir.int<s, 1> x 64>, !cir.vector<{{!s8i|!u8i}} x 64>) -> !cir.vector<{{!s8i|!u8i}} x 64>
44+
45+
// LLVM-LABEL: @test_mm512_mask_loadu_epi8
46+
// LLVM: @llvm.masked.load.v64i8.p0(ptr %{{.*}}, i32 1, <64 x i1> %{{.*}}, <64 x i8> %{{.*}})
47+
return _mm512_mask_loadu_epi8(__W, __U, __P);
48+
}
49+
50+
__m512i test_mm512_mask_loadu_epi16(__m512i __W, __mmask32 __U, void const *__P) {
51+
// CIR-LABEL: _mm512_mask_loadu_epi16
52+
// CIR: %{{.*}} = cir.llvm.intrinsic "masked.load" %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}} : (!cir.ptr<!cir.vector<!s16i x 32>>, !u32i, !cir.vector<!cir.int<s, 1> x 32>, !cir.vector<!s16i x 32>) -> !cir.vector<!s16i x 32>
53+
54+
// LLVM-LABEL: @test_mm512_mask_loadu_epi16
55+
// LLVM: @llvm.masked.load.v32i16.p0(ptr %{{.*}}, i32 1, <32 x i1> %{{.*}}, <32 x i16> %{{.*}})
56+
return _mm512_mask_loadu_epi16(__W, __U, __P);
57+
}
58+
59+
__m512i test_mm512_maskz_loadu_epi16(__mmask32 __U, void const *__P) {
60+
// CIR-LABEL: _mm512_maskz_loadu_epi16
61+
// CIR: %{{.*}} = cir.llvm.intrinsic "masked.load" %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}} : (!cir.ptr<!cir.vector<!s16i x 32>>, !u32i, !cir.vector<!cir.int<s, 1> x 32>, !cir.vector<!s16i x 32>) -> !cir.vector<!s16i x 32>
62+
63+
// LLVM-LABEL: @test_mm512_maskz_loadu_epi16
64+
// LLVM: @llvm.masked.load.v32i16.p0(ptr %{{.*}}, i32 1, <32 x i1> %{{.*}}, <32 x i16> %{{.*}})
65+
return _mm512_maskz_loadu_epi16(__U, __P);
66+
}
67+
68+
__m512i test_mm512_maskz_loadu_epi8(__mmask64 __U, void const *__P) {
69+
// CIR-LABEL: _mm512_maskz_loadu_epi8
70+
// CIR: cir.llvm.intrinsic "masked.load" %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}} : (!cir.ptr<!cir.vector<{{!s8i|!u8i}} x 64>>, !u32i, !cir.vector<!cir.int<s, 1> x 64>, !cir.vector<{{!s8i|!u8i}} x 64>) -> !cir.vector<{{!s8i|!u8i}} x 64>
71+
72+
// LLVM-LABEL: @test_mm512_maskz_loadu_epi8
73+
// LLVM: @llvm.masked.load.v64i8.p0(ptr %{{.*}}, i32 1, <64 x i1> %{{.*}}, <64 x i8> %{{.*}})
74+
return _mm512_maskz_loadu_epi8(__U, __P);
75+
}

0 commit comments

Comments
 (0)