diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp index 61011d55227e7..e9e476b10ff34 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp @@ -3076,12 +3076,13 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) { // TODO: apply range metadata for range check patterns? } - // Separate storage assumptions apply to the underlying allocations, not any - // particular pointer within them. When evaluating the hints for AA purposes - // we getUnderlyingObject them; by precomputing the answers here we can - // avoid having to do so repeatedly there. for (unsigned Idx = 0; Idx < II->getNumOperandBundles(); Idx++) { OperandBundleUse OBU = II->getOperandBundleAt(Idx); + + // Separate storage assumptions apply to the underlying allocations, not + // any particular pointer within them. When evaluating the hints for AA + // purposes we getUnderlyingObject them; by precomputing the answers here + // we can avoid having to do so repeatedly there. if (OBU.getTagName() == "separate_storage") { assert(OBU.Inputs.size() == 2); auto MaybeSimplifyHint = [&](const Use &U) { @@ -3095,6 +3096,28 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) { MaybeSimplifyHint(OBU.Inputs[0]); MaybeSimplifyHint(OBU.Inputs[1]); } + + // Try to fold alignment assumption into a load's !align metadata, if the + // assumption is valid in the load's context. + if (OBU.getTagName() == "align" && OBU.Inputs.size() == 2) { + RetainedKnowledge RK = getKnowledgeFromBundle( + *cast(II), II->bundle_op_info_begin()[Idx]); + if (!RK || RK.AttrKind != Attribute::Alignment || + !isPowerOf2_64(RK.ArgValue)) + continue; + + auto *LI = dyn_cast(OBU.Inputs[0]); + if (!LI || + !isValidAssumeForContext(II, LI, &DT, /*AllowEphemerals=*/true)) + continue; + + LI->setMetadata( + LLVMContext::MD_align, + MDNode::get(II->getContext(), ValueAsMetadata::getConstant( + Builder.getInt64(RK.ArgValue)))); + auto *New = CallBase::removeOperandBundle(II, OBU.getTagID()); + return New; + } } // Convert nonnull assume like: diff --git a/llvm/lib/Transforms/Scalar/EarlyCSE.cpp b/llvm/lib/Transforms/Scalar/EarlyCSE.cpp index cf11f5bc885a7..b6a6806707091 100644 --- a/llvm/lib/Transforms/Scalar/EarlyCSE.cpp +++ b/llvm/lib/Transforms/Scalar/EarlyCSE.cpp @@ -31,6 +31,7 @@ #include "llvm/IR/Constants.h" #include "llvm/IR/Dominators.h" #include "llvm/IR/Function.h" +#include "llvm/IR/IRBuilder.h" #include "llvm/IR/InstrTypes.h" #include "llvm/IR/Instruction.h" #include "llvm/IR/Instructions.h" @@ -1588,6 +1589,14 @@ bool EarlyCSE::processNode(DomTreeNode *Node) { if (InVal.IsLoad) if (auto *I = dyn_cast(Op)) combineMetadataForCSE(I, &Inst, false); + if (auto *AlignMD = Inst.getMetadata(LLVMContext::MD_align)) { + auto *A = mdconst::extract(AlignMD->getOperand(0)); + if (Op->getPointerAlignment(SQ.DL).value() % A->getZExtValue() != 0) { + IRBuilder B(&Inst); + B.CreateAlignmentAssumption(SQ.DL, Op, A); + } + } + if (!Inst.use_empty()) Inst.replaceAllUsesWith(Op); salvageKnowledge(&Inst, &AC); diff --git a/llvm/test/Transforms/InstCombine/assume-align.ll b/llvm/test/Transforms/InstCombine/assume-align.ll index 2b8ca5d25fd1a..7020e5920f328 100644 --- a/llvm/test/Transforms/InstCombine/assume-align.ll +++ b/llvm/test/Transforms/InstCombine/assume-align.ll @@ -123,11 +123,9 @@ define i8 @assume_align_non_pow2(ptr %p) { ret i8 %v } -; TODO: Can fold alignment assumption into !align metadata on load. define ptr @fold_assume_align_pow2_of_loaded_pointer_into_align_metadata(ptr %p) { ; CHECK-LABEL: @fold_assume_align_pow2_of_loaded_pointer_into_align_metadata( -; CHECK-NEXT: [[P2:%.*]] = load ptr, ptr [[P:%.*]], align 8 -; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[P2]], i64 8) ] +; CHECK-NEXT: [[P2:%.*]] = load ptr, ptr [[P:%.*]], align 8, !align [[META0:![0-9]+]] ; CHECK-NEXT: ret ptr [[P2]] ; %p2 = load ptr, ptr %p @@ -135,6 +133,16 @@ define ptr @fold_assume_align_pow2_of_loaded_pointer_into_align_metadata(ptr %p) ret ptr %p2 } +define ptr @fold_assume_align_i32_pow2_of_loaded_pointer_into_align_metadata(ptr %p) { +; CHECK-LABEL: @fold_assume_align_i32_pow2_of_loaded_pointer_into_align_metadata( +; CHECK-NEXT: [[P2:%.*]] = load ptr, ptr [[P:%.*]], align 8, !align [[META0]] +; CHECK-NEXT: ret ptr [[P2]] +; + %p2 = load ptr, ptr %p + call void @llvm.assume(i1 true) [ "align"(ptr %p2, i32 8) ] + ret ptr %p2 +} + define ptr @dont_fold_assume_align_pow2_of_loaded_pointer_into_align_metadata_due_to_call(ptr %p) { ; CHECK-LABEL: @dont_fold_assume_align_pow2_of_loaded_pointer_into_align_metadata_due_to_call( ; CHECK-NEXT: [[P2:%.*]] = load ptr, ptr [[P:%.*]], align 8 @@ -171,3 +179,19 @@ define ptr @dont_fold_assume_align_zero_of_loaded_pointer_into_align_metadata(pt call void @llvm.assume(i1 true) [ "align"(ptr %p2, i64 0) ] ret ptr %p2 } + +; !align must have a constant integer alignment. +define ptr @dont_fold_assume_align_not_constant_of_loaded_pointer_into_align_metadata(ptr %p, i64 %align) { +; CHECK-LABEL: @dont_fold_assume_align_not_constant_of_loaded_pointer_into_align_metadata( +; CHECK-NEXT: [[P2:%.*]] = load ptr, ptr [[P:%.*]], align 8, !align [[META1:![0-9]+]] +; CHECK-NEXT: ret ptr [[P2]] +; + %p2 = load ptr, ptr %p + call void @llvm.assume(i1 true) [ "align"(ptr %p2, i64 %align) ] + ret ptr %p2 +} + +;. +; CHECK: [[META0]] = !{i64 8} +; CHECK: [[META1]] = !{i64 1} +;.