Skip to content

Commit 3ff3c4e

Browse files
HendrikHuebnerHendrikHuebner
authored
[CIR] Upstream X86 builtin clflush, fence and pause (#167401)
This PR upstreams the intrinsics `_mm_prefetch`, `_mm_(l|m)fenche`, `_mm_pause` and `_mm_clflush` from the incubator repository.
1 parent 36848a3 commit 3ff3c4e

File tree

8 files changed

+217
-1
lines changed

8 files changed

+217
-1
lines changed

‎clang/include/clang/CIR/Dialect/IR/CIROps.td‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -413,6 +413,18 @@ def CIR_ConstantOp : CIR_Op<"const", [
413413

414414
template <typename T>
415415
T getValueAttr() { return mlir::dyn_cast<T>(getValue()); }
416+
417+
llvm::APInt getIntValue() {
418+
if (const auto intAttr = getValueAttr<cir::IntAttr>())
419+
return intAttr.getValue();
420+
llvm_unreachable("Expected an IntAttr in ConstantOp");
421+
}
422+
423+
bool getBoolValue() {
424+
if (const auto boolAttr = getValueAttr<cir::BoolAttr>())
425+
return boolAttr.getValue();
426+
llvm_unreachable("Expected a BoolAttr in ConstantOp");
427+
}
416428
}];
417429

418430
let hasFolder = 1;
@@ -2579,6 +2591,39 @@ def CIR_FuncOp : CIR_Op<"func", [
25792591
}];
25802592
}
25812593

2594+
//===----------------------------------------------------------------------===//
2595+
// LLVMIntrinsicCallOp
2596+
//===----------------------------------------------------------------------===//
2597+
2598+
def CIR_LLVMIntrinsicCallOp : CIR_Op<"call_llvm_intrinsic"> {
2599+
let summary = "Call to llvm intrinsic functions that is not defined in CIR";
2600+
let description = [{
2601+
`cir.call_llvm_intrinsic` operation represents a call-like expression which has
2602+
return type and arguments that maps directly to a llvm intrinsic.
2603+
It only records intrinsic `intrinsic_name`.
2604+
}];
2605+
2606+
let results = (outs Optional<CIR_AnyType>:$result);
2607+
let arguments = (ins
2608+
StrAttr:$intrinsic_name, Variadic<CIR_AnyType>:$arg_ops);
2609+
2610+
let skipDefaultBuilders = 1;
2611+
2612+
let assemblyFormat = [{
2613+
$intrinsic_name $arg_ops `:` functional-type($arg_ops, $result) attr-dict
2614+
}];
2615+
2616+
let builders = [
2617+
OpBuilder<(ins "mlir::StringAttr":$intrinsic_name, "mlir::Type":$resType,
2618+
CArg<"mlir::ValueRange", "{}">:$operands), [{
2619+
$_state.addAttribute("intrinsic_name", intrinsic_name);
2620+
$_state.addOperands(operands);
2621+
if (resType)
2622+
$_state.addTypes(resType);
2623+
}]>,
2624+
];
2625+
}
2626+
25822627
//===----------------------------------------------------------------------===//
25832628
// CallOp
25842629
//===----------------------------------------------------------------------===//

‎clang/include/clang/CIR/MissingFeatures.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -274,6 +274,7 @@ struct MissingFeatures {
274274
static bool innermostEHScope() { return false; }
275275
static bool insertBuiltinUnpredictable() { return false; }
276276
static bool instrumentation() { return false; }
277+
static bool intrinsicElementTypeSupport() { return false; }
277278
static bool intrinsics() { return false; }
278279
static bool isMemcpyEquivalentSpecialMember() { return false; }
279280
static bool isTrivialCtorOrDtor() { return false; }

‎clang/lib/CIR/CodeGen/CIRGenBuiltinX86.cpp‎

Lines changed: 36 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,19 @@
2121
using namespace clang;
2222
using namespace clang::CIRGen;
2323

24+
template <typename... Operands>
25+
static mlir::Value emitIntrinsicCallOp(CIRGenFunction &cgf, const CallExpr *e,
26+
const std::string &str,
27+
const mlir::Type &resTy,
28+
Operands &&...op) {
29+
CIRGenBuilderTy &builder = cgf.getBuilder();
30+
mlir::Location location = cgf.getLoc(e->getExprLoc());
31+
return cir::LLVMIntrinsicCallOp::create(builder, location,
32+
builder.getStringAttr(str), resTy,
33+
std::forward<Operands>(op)...)
34+
.getResult();
35+
}
36+
2437
mlir::Value CIRGenFunction::emitX86BuiltinExpr(unsigned builtinID,
2538
const CallExpr *e) {
2639
if (builtinID == Builtin::BI__builtin_cpu_is) {
@@ -43,15 +56,37 @@ mlir::Value CIRGenFunction::emitX86BuiltinExpr(unsigned builtinID,
4356
// Find out if any arguments are required to be integer constant expressions.
4457
assert(!cir::MissingFeatures::handleBuiltinICEArguments());
4558

59+
// The operands of the builtin call
60+
llvm::SmallVector<mlir::Value> ops;
61+
62+
// `ICEArguments` is a bitmap indicating whether the argument at the i-th bit
63+
// is required to be a constant integer expression.
64+
unsigned iceArguments = 0;
65+
ASTContext::GetBuiltinTypeError error;
66+
getContext().GetBuiltinType(builtinID, error, &iceArguments);
67+
assert(error == ASTContext::GE_None && "Error while getting builtin type.");
68+
69+
for (auto [idx, arg] : llvm::enumerate(e->arguments())) {
70+
ops.push_back(emitScalarOrConstFoldImmArg(iceArguments, idx, arg));
71+
}
72+
73+
CIRGenBuilderTy &builder = getBuilder();
74+
mlir::Type voidTy = builder.getVoidTy();
75+
4676
switch (builtinID) {
4777
default:
4878
return {};
49-
case X86::BI_mm_prefetch:
5079
case X86::BI_mm_clflush:
80+
return emitIntrinsicCallOp(*this, e, "x86.sse2.clflush", voidTy, ops[0]);
5181
case X86::BI_mm_lfence:
82+
return emitIntrinsicCallOp(*this, e, "x86.sse2.lfence", voidTy);
5283
case X86::BI_mm_pause:
84+
return emitIntrinsicCallOp(*this, e, "x86.sse2.pause", voidTy);
5385
case X86::BI_mm_mfence:
86+
return emitIntrinsicCallOp(*this, e, "x86.sse2.mfence", voidTy);
5487
case X86::BI_mm_sfence:
88+
return emitIntrinsicCallOp(*this, e, "x86.sse.sfence", voidTy);
89+
case X86::BI_mm_prefetch:
5590
case X86::BI__rdtsc:
5691
case X86::BI__builtin_ia32_rdtscp:
5792
case X86::BI__builtin_ia32_lzcnt_u16:

‎clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1442,6 +1442,28 @@ mlir::Value CIRGenFunction::emitPromotedScalarExpr(const Expr *e,
14421442
return ScalarExprEmitter(*this, builder).Visit(const_cast<Expr *>(e));
14431443
}
14441444

1445+
mlir::Value CIRGenFunction::emitScalarOrConstFoldImmArg(unsigned iceArguments,
1446+
unsigned index,
1447+
const Expr *arg) {
1448+
mlir::Value result{};
1449+
1450+
// The bit at the specified index indicates whether the argument is required
1451+
// to be a constant integer expression.
1452+
bool isArgRequiredToBeConstant = (iceArguments & (1 << index));
1453+
1454+
if (!isArgRequiredToBeConstant) {
1455+
result = emitScalarExpr(arg);
1456+
} else {
1457+
// If this is required to be a constant, constant fold it so that we
1458+
// know that the generated intrinsic gets a ConstantInt.
1459+
std::optional<llvm::APSInt> iceOpt =
1460+
arg->getIntegerConstantExpr(getContext());
1461+
assert(iceOpt && "Expected argument to be a constant");
1462+
result = builder.getConstInt(getLoc(arg->getSourceRange()), *iceOpt);
1463+
}
1464+
return result;
1465+
}
1466+
14451467
[[maybe_unused]] static bool mustVisitNullValue(const Expr *e) {
14461468
// If a null pointer expression's type is the C++0x nullptr_t and
14471469
// the expression is not a simple literal, it must be evaluated

‎clang/lib/CIR/CodeGen/CIRGenFunction.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1546,6 +1546,9 @@ class CIRGenFunction : public CIRGenTypeCache {
15461546
mlir::Value emitScalarExpr(const clang::Expr *e,
15471547
bool ignoreResultAssign = false);
15481548

1549+
mlir::Value emitScalarOrConstFoldImmArg(unsigned iceArguments, unsigned index,
1550+
const Expr *arg);
1551+
15491552
mlir::Value emitScalarPrePostIncDec(const UnaryOperator *e, LValue lv,
15501553
cir::UnaryOpKind kind, bool isPre);
15511554

‎clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -320,6 +320,30 @@ static mlir::LLVM::CallIntrinsicOp replaceOpWithCallLLVMIntrinsicOp(
320320
return callIntrinOp;
321321
}
322322

323+
mlir::LogicalResult CIRToLLVMLLVMIntrinsicCallOpLowering::matchAndRewrite(
324+
cir::LLVMIntrinsicCallOp op, OpAdaptor adaptor,
325+
mlir::ConversionPatternRewriter &rewriter) const {
326+
mlir::Type llvmResTy =
327+
getTypeConverter()->convertType(op->getResultTypes()[0]);
328+
if (!llvmResTy)
329+
return op.emitError("expected LLVM result type");
330+
StringRef name = op.getIntrinsicName();
331+
332+
// Some LLVM intrinsics require ElementType attribute to be attached to
333+
// the argument of pointer type. That prevents us from generating LLVM IR
334+
// because from LLVM dialect, we have LLVM IR like the below which fails
335+
// LLVM IR verification.
336+
// %3 = call i64 @llvm.aarch64.ldxr.p0(ptr %2)
337+
// The expected LLVM IR should be like
338+
// %3 = call i64 @llvm.aarch64.ldxr.p0(ptr elementtype(i32) %2)
339+
// TODO(cir): MLIR LLVM dialect should handle this part as CIR has no way
340+
// to set LLVM IR attribute.
341+
assert(!cir::MissingFeatures::intrinsicElementTypeSupport());
342+
replaceOpWithCallLLVMIntrinsicOp(rewriter, op, "llvm." + name, llvmResTy,
343+
adaptor.getOperands());
344+
return mlir::success();
345+
}
346+
323347
/// IntAttr visitor.
324348
mlir::Value CIRAttrToValue::visitCirAttr(cir::IntAttr intAttr) {
325349
mlir::Location loc = parentOp->getLoc();

‎clang/test/CIR/CodeGen/X86/sse-builtins.c‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-unknown-linux -target-feature +sse -fclangir -emit-cir -o %t.cir -Wall -Werror
2+
// RUN: FileCheck --check-prefix=CIR --input-file=%t.cir %s
3+
// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-unknown-linux -target-feature +sse -fclangir -emit-llvm -o %t.ll -Wall -Werror
4+
// RUN: FileCheck --check-prefixes=LLVM --input-file=%t.ll %s
5+
6+
// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-unknown-linux -target-feature +sse -fno-signed-char -fclangir -emit-cir -o %t.cir -Wall -Werror
7+
// RUN: FileCheck --check-prefix=CIR --input-file=%t.cir %s
8+
// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-unknown-linux -target-feature +sse -fclangir -emit-llvm -o %t.ll -Wall -Werror
9+
// RUN: FileCheck --check-prefixes=LLVM --input-file=%t.ll %s
10+
11+
// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-unknown-linux -target-feature +sse -emit-llvm -o - -Wall -Werror | FileCheck %s -check-prefix=OGCG
12+
// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-unknown-linux -target-feature +sse -emit-llvm -o - -Wall -Werror | FileCheck %s -check-prefix=OGCG
13+
14+
// This test mimics clang/test/CodeGen/X86/sse-builtins.c, which eventually
15+
// CIR shall be able to support fully.
16+
17+
#include <immintrin.h>
18+
19+
20+
void test_mm_sfence(void) {
21+
// CIR-LABEL: test_mm_sfence
22+
// LLVM-LABEL: test_mm_sfence
23+
// OGCG-LABEL: test_mm_sfence
24+
_mm_sfence();
25+
// CIR: {{%.*}} = cir.call_llvm_intrinsic "x86.sse.sfence" : () -> !void
26+
// LLVM: call void @llvm.x86.sse.sfence()
27+
// OGCG: call void @llvm.x86.sse.sfence()
28+
}

‎clang/test/CIR/CodeGen/X86/sse2-builtins.c‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-unknown-linux -target-feature +sse2 -fclangir -emit-cir -o %t.cir -Wall -Werror
2+
// RUN: FileCheck --check-prefixes=CIR --input-file=%t.cir %s
3+
// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-unknown-linux -target-feature +sse2 -fno-signed-char -fclangir -emit-cir -o %t.cir -Wall -Werror
4+
// RUN: FileCheck --check-prefixes=CIR --input-file=%t.cir %s
5+
6+
// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-unknown-linux -target-feature +sse2 -fclangir -emit-llvm -o %t.ll -Wall -Werror
7+
// RUN: FileCheck --check-prefixes=LLVM --input-file=%t.ll %s
8+
// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-unknown-linux -target-feature +sse2 -fno-signed-char -fclangir -emit-llvm -o %t.ll -Wall -Werror
9+
// RUN: FileCheck --check-prefixes=LLVM --input-file=%t.ll %s
10+
11+
// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-unknown-linux -target-feature +sse -emit-llvm -o - -Wall -Werror | FileCheck %s -check-prefix=OGCG
12+
// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-unknown-linux -target-feature +sse -emit-llvm -o - -Wall -Werror | FileCheck %s -check-prefix=OGCG
13+
14+
// This test mimics clang/test/CodeGen/X86/sse2-builtins.c, which eventually
15+
// CIR shall be able to support fully.
16+
17+
#include <immintrin.h>
18+
19+
20+
void test_mm_clflush(void* A) {
21+
// CIR-LABEL: test_mm_clflush
22+
// LLVM-LABEL: test_mm_clflush
23+
// OGCG-LABEL: test_mm_clflush
24+
_mm_clflush(A);
25+
// CIR: {{%.*}} = cir.call_llvm_intrinsic "x86.sse2.clflush" {{%.*}} : (!cir.ptr<!void>) -> !void
26+
// LLVM: call void @llvm.x86.sse2.clflush(ptr {{%.*}})
27+
// OGCG: call void @llvm.x86.sse2.clflush(ptr {{%.*}})
28+
}
29+
30+
void test_mm_lfence(void) {
31+
// CIR-LABEL: test_mm_lfence
32+
// LLVM-LABEL: test_mm_lfence
33+
// OGCG-LABEL: test_mm_lfence
34+
_mm_lfence();
35+
// CIR: {{%.*}} = cir.call_llvm_intrinsic "x86.sse2.lfence" : () -> !void
36+
// LLVM: call void @llvm.x86.sse2.lfence()
37+
// OGCG: call void @llvm.x86.sse2.lfence()
38+
}
39+
40+
void test_mm_mfence(void) {
41+
// CIR-LABEL: test_mm_mfence
42+
// LLVM-LABEL: test_mm_mfence
43+
// OGCG-LABEL: test_mm_mfence
44+
_mm_mfence();
45+
// CIR: {{%.*}} = cir.call_llvm_intrinsic "x86.sse2.mfence" : () -> !void
46+
// LLVM: call void @llvm.x86.sse2.mfence()
47+
// OGCG: call void @llvm.x86.sse2.mfence()
48+
}
49+
50+
void test_mm_pause(void) {
51+
// CIR-LABEL: test_mm_pause
52+
// LLVM-LABEL: test_mm_pause
53+
// OGCG-LABEL: test_mm_pause
54+
_mm_pause();
55+
// CIR: {{%.*}} = cir.call_llvm_intrinsic "x86.sse2.pause" : () -> !void
56+
// LLVM: call void @llvm.x86.sse2.pause()
57+
// OGCG: call void @llvm.x86.sse2.pause()
58+
}

0 commit comments

Comments
 (0)