Skip to content

Commit deedbf6

Browse files
yukatan1701yukatan1701
committed
[TSAR, Memory] Enhance EstimateMemory and Delinearization passes.
1 parent 118907e commit deedbf6

File tree

2 files changed

+176
-148
lines changed

2 files changed

+176
-148
lines changed

‎lib/Analysis/Memory/Delinearization.cpp‎

Lines changed: 168 additions & 148 deletions
Original file line numberDiff line numberDiff line change
@@ -177,7 +177,9 @@ void delinearizationLog(const DelinearizeInfo &Info, ScalarEvolution &SE,
177177
}
178178
OS << " accesses:\n";
179179
for (auto &Range : *ArrayInfo) {
180-
OS << " address: ";
180+
OS << " "
181+
<< "[" << (Range.isValid() ? "valid" : "invalid") << "] "
182+
<< "address: ";
181183
Range.Ptr->print(OS, true);
182184
OS << "\n";
183185
for (auto *S : Range.Subscripts) {
@@ -222,9 +224,9 @@ void DelinearizationPass::cleanSubscripts(Array &ArrayInfo) {
222224
<< (ArrayInfo.isAddressOfVariable() ? "address of " : "")
223225
<< "base " << ArrayInfo.getBase()->getName() << "\n");
224226
auto addExtraConstZero =
225-
[this, &ArrayInfo](const Array::Range &Range, unsigned ExtraZeroCount,
226-
const llvm::Type *DereferenceTy, unsigned Idx,
227-
SmallVectorImpl<const SCEV *> &NewSubscripts) -> bool {
227+
[this](const Array::Range &Range, const llvm::Type *DereferenceTy,
228+
unsigned Idx,
229+
SmallVectorImpl<const SCEV *> &NewSubscripts) -> bool {
228230
auto *ATy{dyn_cast<ArrayType>(DereferenceTy)};
229231
if (!ATy)
230232
return false;
@@ -234,18 +236,11 @@ void DelinearizationPass::cleanSubscripts(Array &ArrayInfo) {
234236
if (ATy->getElementType() == Range.Types[Idx]) {
235237
NewSubscripts.push_back(Range.Subscripts[Idx]);
236238
++Idx;
237-
} else if (ExtraZeroCount > 0) {
238-
--ExtraZeroCount;
239-
NewSubscripts.push_back(mSE->getZero(mIndexTy));
240239
} else {
241-
return false;
240+
NewSubscripts.push_back(mSE->getZero(mIndexTy));
242241
}
243242
}
244-
if (!ATy)
245-
return Idx == IdxE;
246-
for (unsigned I = 0; I < ExtraZeroCount; ++I)
247-
NewSubscripts.push_back(mSE->getZero(mIndexTy));
248-
return true;
243+
return Idx == IdxE;
249244
};
250245
auto LastConstDim = ArrayInfo.getNumberOfDims();
251246
for (LastConstDim; LastConstDim > 0; --LastConstDim)
@@ -255,30 +250,31 @@ void DelinearizationPass::cleanSubscripts(Array &ArrayInfo) {
255250
for (auto &Range : ArrayInfo) {
256251
LLVM_DEBUG(dbgs() << "[DELINEARIZE]: process access ";
257252
Range.Ptr->print(dbgs()); dbgs() << "\n");
258-
if (auto ExtraZeroCount{
259-
Range.is(Array::Range::NeedExtraZero)
260-
? ArrayInfo.getNumberOfDims() - Range.Subscripts.size()
261-
: 0};
262-
ExtraZeroCount > 0) {
263-
SmallVector<const SCEV *, 4> NewSubscripts;
264-
auto *BaseType{ArrayInfo.getBase()->getType()};
265-
if (auto *GV{dyn_cast<GlobalValue>(ArrayInfo.getBase())})
266-
BaseType = GV->getValueType();
267-
else if (auto *AI{dyn_cast<AllocaInst>(ArrayInfo.getBase())})
268-
BaseType = AI->getAllocatedType();
269-
if (!addExtraConstZero(Range, ExtraZeroCount, BaseType, 0,
270-
NewSubscripts)) {
271-
LLVM_DEBUG(dbgs() << "[DELINEARIZE]: unable to delinearize access to "
272-
"constant dimensions\n");
273-
continue;
274-
}
275-
std::swap(Range.Subscripts, NewSubscripts);
276-
LLVM_DEBUG(dbgs() << "[DELINEARIZE]: add " << ExtraZeroCount
277-
<< " extra zero subscripts\n");
253+
SmallVector<const SCEV *, 4> NewSubscripts;
254+
auto *BaseType{ArrayInfo.getBase()->getType()};
255+
if (auto *GV{dyn_cast<GlobalValue>(ArrayInfo.getBase())})
256+
BaseType = GV->getValueType();
257+
else if (auto *AI{dyn_cast<AllocaInst>(ArrayInfo.getBase())})
258+
BaseType = AI->getAllocatedType();
259+
if (!addExtraConstZero(Range, BaseType, 0, NewSubscripts) ||
260+
NewSubscripts.size() > ArrayInfo.getNumberOfDims()) {
261+
LLVM_DEBUG(dbgs() << "[DELINEARIZE]: unable to delinearize access to "
262+
"constant dimensions\n");
263+
continue;
278264
}
265+
if (Range.is(Array::Range::NeedExtraZero))
266+
for (unsigned I = NewSubscripts.size(),
267+
EI = ArrayInfo.getNumberOfDims();
268+
I < EI; ++I)
269+
NewSubscripts.push_back(mSE->getZero(mIndexTy));
270+
LLVM_DEBUG(if (Range.Subscripts.size() < NewSubscripts.size())
271+
dbgs() << "[DELINEARIZE]: add "
272+
<< NewSubscripts.size() - Range.Subscripts.size()
273+
<< " extra zero subscripts\n");
274+
std::swap(Range.Subscripts, NewSubscripts);
275+
Range.setProperty(Array::Range::IsDelinearized);
279276
assert(Range.Subscripts.size() <= ArrayInfo.getNumberOfDims() &&
280277
"Unable to delinearize element access!");
281-
Range.setProperty(Array::Range::IsDelinearized);
282278
}
283279
return;
284280
}
@@ -295,10 +291,11 @@ void DelinearizationPass::cleanSubscripts(Array &ArrayInfo) {
295291
// So, we try to add extra zero subscripts.
296292
auto ExtraZeroCount = Range.is(Array::Range::NeedExtraZero) ?
297293
ArrayInfo.getNumberOfDims() - Range.Subscripts.size() : 0;
298-
std::size_t DimIdx = 0, SubscriptIdx = 0;
299-
std::size_t DimIdxE = std::min(LastConstDim - 1, Range.Subscripts.size());
294+
unsigned DimIdx = 0, SubscriptIdx = 0;
295+
unsigned NumberOfKnownSubscripts = Range.Subscripts.size();
300296
SmallVector<const SCEV *, 4> NewSubscripts;
301-
for (; DimIdx < DimIdxE; ++DimIdx) {
297+
for (; DimIdx < LastConstDim - 1 && DimIdx < NumberOfKnownSubscripts;
298+
++DimIdx) {
302299
auto *Subscript = Range.Subscripts[SubscriptIdx++];
303300
LLVM_DEBUG(dbgs() << "[DELINEARIZE]: subscript " << DimIdx << ": "
304301
<< *Subscript << "\n");
@@ -326,34 +323,51 @@ void DelinearizationPass::cleanSubscripts(Array &ArrayInfo) {
326323
NewSubscripts.push_back(Subscript);
327324
LLVM_DEBUG(dbgs() << "[DELINEARIZE]: set to " << *Subscript << "\n");
328325
}
329-
if (DimIdx < DimIdxE)
326+
if (DimIdx < LastConstDim - 1 && DimIdx < NumberOfKnownSubscripts)
330327
continue;
331-
if (ExtraZeroCount > 0) {
332-
LLVM_DEBUG(dbgs() << "[DELINEARIZE]: add " << ExtraZeroCount
333-
<< " extra zero subscripts after " << SubscriptIdx
334-
<< "\n");
335-
if (Range.Subscripts.size() > SubscriptIdx) {
336-
NewSubscripts.push_back(Range.Subscripts[SubscriptIdx]);
337-
if (!addExtraConstZero(Range, ExtraZeroCount, Range.Types[SubscriptIdx],
338-
SubscriptIdx + 1, NewSubscripts)) {
339-
if (SubscriptIdx == 0 ||
340-
!(NewSubscripts.pop_back(),
341-
addExtraConstZero(Range, ExtraZeroCount,
342-
Range.Types[SubscriptIdx - 1], SubscriptIdx,
343-
NewSubscripts))) {
344-
LLVM_DEBUG(dbgs() << "[DELINEARIZE]: unable to delinearize\n");
345-
continue;
346-
}
328+
if (LastConstDim == ArrayInfo.getNumberOfDims()) {
329+
if (NumberOfKnownSubscripts > SubscriptIdx) {
330+
NewSubscripts.push_back(Range.Subscripts[SubscriptIdx++]);
331+
if (SubscriptIdx != NumberOfKnownSubscripts) {
332+
LLVM_DEBUG(dbgs() << "[DELINEARIZE]: unable to delinearize, too many "
333+
"subscripts left\n");
334+
continue;
347335
}
348-
} else {
349-
for (std::size_t I = 0; I < ExtraZeroCount; ++I)
350-
NewSubscripts.push_back(mSE->getZero(mIndexTy));
336+
assert(NewSubscripts.size() == ArrayInfo.getNumberOfDims() &&
337+
ExtraZeroCount == 0 && "Invariant broken!");
351338
}
352-
} else {
353-
// Add subscripts for constant dimensions.
354-
for (auto EI = Range.Subscripts.size(); SubscriptIdx < EI; ++SubscriptIdx)
355-
NewSubscripts.push_back(Range.Subscripts[SubscriptIdx]);
339+
} else if (NumberOfKnownSubscripts > SubscriptIdx) {
340+
unsigned StashSize = NewSubscripts.size();
341+
NewSubscripts.push_back(Range.Subscripts[SubscriptIdx]);
342+
if (!addExtraConstZero(Range, Range.Types[SubscriptIdx],
343+
SubscriptIdx + 1, NewSubscripts) ||
344+
NewSubscripts.size() > ArrayInfo.getNumberOfDims()) {
345+
if (SubscriptIdx == 0 ||
346+
!(NewSubscripts.resize(StashSize),
347+
NewSubscripts.push_back(mSE->getZero(mIndexTy)),
348+
addExtraConstZero(Range, Range.Types[SubscriptIdx - 1],
349+
SubscriptIdx, NewSubscripts)) ||
350+
NewSubscripts.size() > ArrayInfo.getNumberOfDims()) {
351+
LLVM_DEBUG(dbgs() << "[DELINEARIZE]: unable to delinearize\n");
352+
continue;
353+
}
354+
}
355+
LLVM_DEBUG(
356+
if (NumberOfKnownSubscripts - SubscriptIdx + StashSize <
357+
NewSubscripts.size())
358+
dbgs() << "[DELINEARIZE]: add "
359+
<< NewSubscripts.size() -
360+
(NumberOfKnownSubscripts - SubscriptIdx + StashSize)
361+
<< " extra zero subscripts after " << SubscriptIdx << "\n");
362+
ExtraZeroCount = std::min(
363+
ArrayInfo.getNumberOfDims() - NewSubscripts.size(), ExtraZeroCount);
356364
}
365+
LLVM_DEBUG(
366+
if (ExtraZeroCount > 0)
367+
dbgs() << "[DELINEARIZE]: add " << ExtraZeroCount
368+
<< " extra zero subscripts to finalize range delinearization\n");
369+
for (std::size_t I = 0; I < ExtraZeroCount; ++I)
370+
NewSubscripts.push_back(mSE->getZero(mIndexTy));
357371
std::swap(Range.Subscripts, NewSubscripts);
358372
assert(Range.Subscripts.size() <= ArrayInfo.getNumberOfDims() &&
359373
"Unable to delinearize element access!");
@@ -588,100 +602,106 @@ void DelinearizationPass::findArrayDimensionsFromDbgInfo(Array &ArrayInfo) {
588602

589603
void DelinearizationPass::collectArrays(Function &F) {
590604
DenseSet<const Value *> Visited;
591-
for (auto &I : instructions(F)) {
592-
auto processMemory = [this, &Visited](Instruction &I, MemoryLocation Loc,
593-
unsigned, AccessInfo, AccessInfo) {
594-
if (auto II = dyn_cast<IntrinsicInst>(&I)) {
595-
if (isMemoryMarkerIntrinsic(II->getIntrinsicID()) ||
596-
isDbgInfoIntrinsic(II->getIntrinsicID()))
597-
return;
598-
}
599-
if (!Visited.insert(Loc.Ptr).second)
605+
DenseSet<const Instruction *> Deferred;
606+
auto processMemory = [this, &Visited](const Instruction &I,
607+
MemoryLocation Loc, unsigned, AccessInfo, AccessInfo) {
608+
if (auto II = dyn_cast<IntrinsicInst>(&I)) {
609+
if (isMemoryMarkerIntrinsic(II->getIntrinsicID()) ||
610+
isDbgInfoIntrinsic(II->getIntrinsicID()))
600611
return;
601-
LLVM_DEBUG(dbgs() << "[DELINEARIZE]: process instruction ";
602-
I.print(dbgs()); dbgs() << "\n");
603-
auto &DL = I.getModule()->getDataLayout();
604-
auto *BasePtr = const_cast<Value *>(Loc.Ptr);
605-
auto *DataPtr = BasePtr;
606-
bool IsAddressOfVariable;
607-
std::tie(BasePtr, DataPtr, IsAddressOfVariable) =
608-
GetUnderlyingArray(BasePtr, DL);
609-
LLVM_DEBUG(
610-
dbgs() << "[DELINEARIZE]: strip to "
611-
<< (IsAddressOfVariable ? "address of " : "")
612-
<< "base " << *BasePtr << "\n");
613-
auto ArrayPtr = mDelinearizeInfo.findArray(BasePtr, IsAddressOfVariable);
614-
if (!ArrayPtr) {
615-
ArrayPtr = *mDelinearizeInfo.getArrays().insert(
616-
new Array(BasePtr, IsAddressOfVariable)).first;
617-
findArrayDimensionsFromDbgInfo(*ArrayPtr);
618-
}
619-
auto NumberOfDims = ArrayPtr->getNumberOfDims();
620-
SmallVector<GEPOperator *, 4> GEPs;
621-
auto RangePtr = Loc.Ptr;
622-
auto *GEP = dyn_cast<GEPOperator>(const_cast<Value *>(RangePtr));
623-
while (!GEP) {
624-
if (Operator::getOpcode(RangePtr) == Instruction::BitCast ||
625-
Operator::getOpcode(RangePtr) == Instruction::AddrSpaceCast)
626-
RangePtr = cast<Operator>(RangePtr)->getOperand(0);
627-
else
628-
break;
629-
GEP = dyn_cast<GEPOperator>(const_cast<Value *>(RangePtr));
630-
}
631-
while (GEP) {
632-
GEPs.push_back(GEP);
633-
GEP = dyn_cast<GEPOperator>(GEP->getPointerOperand());
612+
}
613+
if (!Visited.insert(Loc.Ptr).second)
614+
return;
615+
LLVM_DEBUG(dbgs() << "[DELINEARIZE]: process instruction ";
616+
I.print(dbgs()); dbgs() << "\n");
617+
auto &DL = I.getModule()->getDataLayout();
618+
auto *BasePtr = const_cast<Value *>(Loc.Ptr);
619+
auto *DataPtr = BasePtr;
620+
bool IsAddressOfVariable;
621+
std::tie(BasePtr, DataPtr, IsAddressOfVariable) =
622+
GetUnderlyingArray(BasePtr, DL);
623+
LLVM_DEBUG(
624+
dbgs() << "[DELINEARIZE]: strip to "
625+
<< (IsAddressOfVariable ? "address of " : "")
626+
<< "base " << *BasePtr << "\n");
627+
auto ArrayPtr = mDelinearizeInfo.findArray(BasePtr, IsAddressOfVariable);
628+
if (!ArrayPtr) {
629+
ArrayPtr = *mDelinearizeInfo.getArrays().insert(
630+
new Array(BasePtr, IsAddressOfVariable)).first;
631+
findArrayDimensionsFromDbgInfo(*ArrayPtr);
632+
}
633+
auto NumberOfDims = ArrayPtr->getNumberOfDims();
634+
SmallVector<GEPOperator *, 4> GEPs;
635+
auto RangePtr = Loc.Ptr;
636+
auto *GEP = dyn_cast<GEPOperator>(const_cast<Value *>(RangePtr));
637+
while (!GEP) {
638+
if (Operator::getOpcode(RangePtr) == Instruction::BitCast ||
639+
Operator::getOpcode(RangePtr) == Instruction::AddrSpaceCast)
640+
RangePtr = cast<Operator>(RangePtr)->getOperand(0);
641+
else
642+
break;
643+
GEP = dyn_cast<GEPOperator>(const_cast<Value *>(RangePtr));
644+
}
645+
while (GEP) {
646+
GEPs.push_back(GEP);
647+
GEP = dyn_cast<GEPOperator>(GEP->getPointerOperand());
648+
}
649+
auto &Range = ArrayPtr->addRange(const_cast<Value *>(Loc.Ptr));
650+
if (!isa<GEPOperator>(RangePtr) &&
651+
!(RangePtr == BasePtr && !IsAddressOfVariable) &&
652+
!(RangePtr == DataPtr && IsAddressOfVariable))
653+
Range.setProperty(Array::Range::NoGEP);
654+
SmallVector<std::tuple<Value *, Type *>, 4> SubscriptValues;
655+
bool UseAllSubscripts = extractSubscriptsFromGEPs(
656+
GEPs.begin(), GEPs.end(), NumberOfDims, SubscriptValues);
657+
if (!UseAllSubscripts)
658+
Range.setProperty(Array::Range::IgnoreGEP);
659+
// In some cases zero subscript is dropping out by optimization passes.
660+
// So, we try to add extra zero subscripts later. We add subscripts for
661+
// instructions which access a single element, for example, in case of
662+
// a call it is possible to pass a whole array as a parameter
663+
// (without GEPs).
664+
if (isa<LoadInst>(I) || isa<StoreInst>(I) ||
665+
isa<AtomicRMWInst>(I) || isa<AtomicCmpXchgInst>(I)) {
666+
Range.setProperty(Array::Range::IsElement);
667+
if (SubscriptValues.size() < NumberOfDims) {
668+
ArrayPtr->setRangeRef();
669+
Range.setProperty(Array::Range::NeedExtraZero);
634670
}
635-
auto &Range = ArrayPtr->addRange(const_cast<Value *>(Loc.Ptr));
636-
if (!isa<GEPOperator>(RangePtr) &&
637-
!(RangePtr == BasePtr && !IsAddressOfVariable) &&
638-
!(RangePtr == DataPtr && IsAddressOfVariable))
639-
Range.setProperty(Array::Range::NoGEP);
640-
SmallVector<std::tuple<Value *, Type *>, 4> SubscriptValues;
641-
bool UseAllSubscripts = extractSubscriptsFromGEPs(
642-
GEPs.begin(), GEPs.end(), NumberOfDims, SubscriptValues);
643-
if (!UseAllSubscripts)
644-
Range.setProperty(Array::Range::IgnoreGEP);
645-
// In some cases zero subscript is dropping out by optimization passes.
646-
// So, we try to add extra zero subscripts later. We add subscripts for
647-
// instructions which access a single element, for example, in case of
648-
// a call it is possible to pass a whole array as a parameter
649-
// (without GEPs).
650-
if (isa<LoadInst>(I) || isa<StoreInst>(I) ||
651-
isa<AtomicRMWInst>(I) || isa<AtomicCmpXchgInst>(I)) {
652-
Range.setProperty(Array::Range::IsElement);
653-
if (SubscriptValues.size() < NumberOfDims) {
654-
ArrayPtr->setRangeRef();
655-
Range.setProperty(Array::Range::NeedExtraZero);
656-
}
671+
}
672+
if (!SubscriptValues.empty()) {
673+
ArrayPtr->setRangeRef();
674+
for (auto &&[V, T] : SubscriptValues) {
675+
Range.Subscripts.push_back(mSE->getSCEV(V));
676+
Range.Types.push_back(T);
657677
}
658-
if (!SubscriptValues.empty()) {
659-
ArrayPtr->setRangeRef();
660-
for (auto &&[V, T] : SubscriptValues) {
661-
Range.Subscripts.push_back(mSE->getSCEV(V));
662-
Range.Types.push_back(T);
663-
}
678+
}
679+
LLVM_DEBUG(
680+
dbgs() << "[DELINEARIZE]: number of dimensions "
681+
<< NumberOfDims << "\n";
682+
dbgs() << "[DELINEARIZE]: number of subscripts "
683+
<< Range.Subscripts.size() << "\n";
684+
if (Range.is(Array::Range::NeedExtraZero))
685+
dbgs() << "[DELINEARIZE]: need extra zero subscripts\n";
686+
if (Range.is(Array::Range::IgnoreGEP))
687+
dbgs() << "[DELINEARIZE]: ignore some beginning GEPs\n";
688+
if (Range.is(Array::Range::NoGEP))
689+
dbgs() << "[DELINEARIZE]: no GEPs found\n";
690+
dbgs() << "[DELINEARIZE]: subscripts: \n";
691+
for (auto *Subscript : Range.Subscripts) {
692+
dbgs() << " "; Subscript->print(dbgs()); dbgs() <<"\n";
664693
}
665-
LLVM_DEBUG(
666-
dbgs() << "[DELINEARIZE]: number of dimensions "
667-
<< NumberOfDims << "\n";
668-
dbgs() << "[DELINEARIZE]: number of subscripts "
669-
<< Range.Subscripts.size() << "\n";
670-
if (Range.is(Array::Range::NeedExtraZero))
671-
dbgs() << "[DELINEARIZE]: need extra zero subscripts\n";
672-
if (Range.is(Array::Range::IgnoreGEP))
673-
dbgs() << "[DELINEARIZE]: ignore some beginning GEPs\n";
674-
if (Range.is(Array::Range::NoGEP))
675-
dbgs() << "[DELINEARIZE]: no GEPs found\n";
676-
dbgs() << "[DELINEARIZE]: subscripts: \n";
677-
for (auto *Subscript : Range.Subscripts) {
678-
dbgs() << " "; Subscript->print(dbgs()); dbgs() <<"\n";
679-
}
680-
);
681-
};
694+
);
695+
};
696+
for (auto &I : instructions(F)) {
682697
for_each_memory(I, *mTLI, processMemory,
683698
[](Instruction &, AccessInfo, AccessInfo) {});
699+
if (isa<GetElementPtrInst>(I))
700+
Deferred.insert(&I);
684701
}
702+
for (auto *I : Deferred)
703+
processMemory(*I, MemoryLocation(I, LocationSize::beforeOrAfterPointer()),
704+
0, AccessInfo::No, AccessInfo::No);
685705
// Now, we remove all object which is not arrays.
686706
for (auto Itr = mDelinearizeInfo.getArrays().begin(),
687707
ItrE = mDelinearizeInfo.getArrays().end(); Itr != ItrE;) {

‎lib/Analysis/Memory/EstimateMemory.cpp‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1115,6 +1115,14 @@ bool EstimateMemoryPass::runOnFunction(Function &F) {
11151115
return;
11161116
if (AccessedMemory.count(V))
11171117
return;
1118+
if (auto *GEP{dyn_cast<GEPOperator>(V)}) {
1119+
auto IsOnlyGEPUsers{true};
1120+
for_each_user_insts(const_cast<Value &>(*V), [&IsOnlyGEPUsers](auto *U) {
1121+
IsOnlyGEPUsers &= isa<GEPOperator>(U);
1122+
});
1123+
if (IsOnlyGEPUsers)
1124+
return;
1125+
}
11181126
auto PointeeTy{getPointerElementType(*V)};
11191127
addLocation(MemoryLocation(
11201128
V, PointeeTy && PointeeTy->isSized()

0 commit comments

Comments
 (0)