diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp index 7d6dbd51a404d..676e64b8982ad 100644 --- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp +++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp @@ -42,13 +42,12 @@ #include "llvm/IR/DiagnosticInfo.h" #include "llvm/IR/Dominators.h" #include "llvm/IR/Function.h" -#include "llvm/IR/GetElementPtrTypeIterator.h" #include "llvm/IR/InstrTypes.h" #include "llvm/IR/Instruction.h" #include "llvm/IR/Instructions.h" +#include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/Operator.h" #include "llvm/IR/PassManager.h" -#include "llvm/IR/PatternMatch.h" #include "llvm/IR/Type.h" #include "llvm/IR/Value.h" #include "llvm/IR/ValueHandle.h" @@ -66,7 +65,6 @@ #include using namespace llvm; -using namespace llvm::PatternMatch; #define DEBUG_TYPE "loop-accesses" @@ -2815,50 +2813,25 @@ bool LoopAccessInfo::isInvariant(Value *V) const { return SE->isLoopInvariant(S, TheLoop); } -/// Find the operand of the GEP that should be checked for consecutive -/// stores. This ignores trailing indices that have no effect on the final -/// pointer. -static unsigned getGEPInductionOperand(const GetElementPtrInst *Gep) { - const DataLayout &DL = Gep->getDataLayout(); - unsigned LastOperand = Gep->getNumOperands() - 1; - TypeSize GEPAllocSize = DL.getTypeAllocSize(Gep->getResultElementType()); - - // Walk backwards and try to peel off zeros. - while (LastOperand > 1 && match(Gep->getOperand(LastOperand), m_Zero())) { - // Find the type we're currently indexing into. - gep_type_iterator GEPTI = gep_type_begin(Gep); - std::advance(GEPTI, LastOperand - 2); - - // If it's a type with the same allocation size as the result of the GEP we - // can peel off the zero index. - TypeSize ElemSize = GEPTI.isStruct() - ? DL.getTypeAllocSize(GEPTI.getIndexedType()) - : GEPTI.getSequentialElementStride(DL); - if (ElemSize != GEPAllocSize) - break; - --LastOperand; - } - - return LastOperand; -} - -/// If the argument is a GEP, then returns the operand identified by -/// getGEPInductionOperand. However, if there is some other non-loop-invariant -/// operand, it returns that instead. -static Value *stripGetElementPtr(Value *Ptr, ScalarEvolution *SE, Loop *Lp) { +/// If \p Ptr is a GEP, which has a loop-variant operand, return that operand. +/// Otherwise, return \p Ptr. +static Value *getLoopVariantGEPOperand(Value *Ptr, ScalarEvolution *SE, + Loop *Lp) { auto *GEP = dyn_cast(Ptr); if (!GEP) return Ptr; - unsigned InductionOperand = getGEPInductionOperand(GEP); - - // Check that all of the gep indices are uniform except for our induction - // operand. - for (unsigned I = 0, E = GEP->getNumOperands(); I != E; ++I) - if (I != InductionOperand && - !SE->isLoopInvariant(SE->getSCEV(GEP->getOperand(I)), Lp)) - return Ptr; - return GEP->getOperand(InductionOperand); + Value *V = Ptr; + for (const Use &U : GEP->operands()) { + if (!SE->isLoopInvariant(SE->getSCEV(U), Lp)) { + if (V == Ptr) + V = U; + else + // There must be exactly one loop-variant operand. + return Ptr; + } + } + return V; } /// Get the stride of a pointer access in a loop. Looks for symbolic @@ -2873,7 +2846,7 @@ static const SCEV *getStrideFromPointer(Value *Ptr, ScalarEvolution *SE, Loop *L // pointer, otherwise, we are analyzing the index. Value *OrigPtr = Ptr; - Ptr = stripGetElementPtr(Ptr, SE, Lp); + Ptr = getLoopVariantGEPOperand(Ptr, SE, Lp); const SCEV *V = SE->getSCEV(Ptr); if (Ptr != OrigPtr) diff --git a/llvm/test/Analysis/LoopAccessAnalysis/symbolic-stride.ll b/llvm/test/Analysis/LoopAccessAnalysis/symbolic-stride.ll index 525995156481c..8603417081067 100644 --- a/llvm/test/Analysis/LoopAccessAnalysis/symbolic-stride.ll +++ b/llvm/test/Analysis/LoopAccessAnalysis/symbolic-stride.ll @@ -140,6 +140,53 @@ exit: ret void } +; Test with multiple GEP indices +define void @single_stride_array(ptr noalias %A, ptr noalias %B, i64 %N, i64 %stride) { +; CHECK-LABEL: 'single_stride_array' +; CHECK-NEXT: loop: +; CHECK-NEXT: Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop +; CHECK-NEXT: Backward loop carried data dependence. +; CHECK-NEXT: Dependences: +; CHECK-NEXT: Backward: +; CHECK-NEXT: %load = load [2 x i32], ptr %gep.A, align 4 -> +; CHECK-NEXT: store [2 x i32] %ins, ptr %gep.A.next, align 4 +; CHECK-EMPTY: +; CHECK-NEXT: Run-time memory checks: +; CHECK-NEXT: Grouped accesses: +; CHECK-EMPTY: +; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop. +; CHECK-NEXT: SCEV assumptions: +; CHECK-NEXT: Equal predicate: %stride == 1 +; CHECK-EMPTY: +; CHECK-NEXT: Expressions re-written: +; CHECK-NEXT: [PSE] %gep.A = getelementptr inbounds [2 x i32], ptr %A, i64 %mul, i64 1: +; CHECK-NEXT: {(4 + %A),+,(8 * %stride)}<%loop> +; CHECK-NEXT: --> {(4 + %A),+,8}<%loop> +; +entry: + br label %loop + +loop: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] + %mul = mul i64 %iv, %stride + %gep.A = getelementptr inbounds [2 x i32], ptr %A, i64 %mul, i64 1 + %load = load [2 x i32], ptr %gep.A, align 4 + %gep.B = getelementptr inbounds [2 x i32], ptr %B, i64 %iv + %load_1 = load [2 x i32], ptr %gep.B, align 4 + %v1 = extractvalue [2 x i32] %load, 0 + %v2 = extractvalue [2 x i32] %load_1, 0 + %add = add i32 %v1, %v2 + %ins = insertvalue [2 x i32] poison, i32 %add, 0 + %iv.next = add nuw nsw i64 %iv, 1 + %gep.A.next = getelementptr inbounds [2 x i32], ptr %A, i64 %iv.next + store [2 x i32] %ins, ptr %gep.A.next, align 4 + %exitcond = icmp eq i64 %iv.next, %N + br i1 %exitcond, label %exit, label %loop + +exit: + ret void +} + define void @single_stride_castexpr(i32 %offset, ptr %src, ptr %dst, i1 %cond) { ; CHECK-LABEL: 'single_stride_castexpr' ; CHECK-NEXT: inner.loop: