@@ -361,6 +361,74 @@ static void addVPMetadata(Module &M, Instruction &I,
361361 }
362362}
363363
364+ static void
365+ handleAllocSite (Instruction &I, CallBase *CI,
366+ ArrayRef<uint64_t > InlinedCallStack, LLVMContext &Ctx,
367+ OptimizationRemarkEmitter &ORE, uint64_t MaxColdSize,
368+ const std::set<const AllocationInfo *> &AllocInfoSet,
369+ std::map<std::pair<uint64_t , unsigned >, AllocMatchInfo>
370+ &FullStackIdToAllocMatchInfo) {
371+ // We may match this instruction's location list to multiple MIB
372+ // contexts. Add them to a Trie specialized for trimming the contexts to
373+ // the minimal needed to disambiguate contexts with unique behavior.
374+ CallStackTrie AllocTrie (&ORE, MaxColdSize);
375+ uint64_t TotalSize = 0 ;
376+ uint64_t TotalColdSize = 0 ;
377+ for (auto *AllocInfo : AllocInfoSet) {
378+ // Check the full inlined call stack against this one.
379+ // If we found and thus matched all frames on the call, include
380+ // this MIB.
381+ if (stackFrameIncludesInlinedCallStack (AllocInfo->CallStack ,
382+ InlinedCallStack)) {
383+ NumOfMemProfMatchedAllocContexts++;
384+ uint64_t FullStackId = 0 ;
385+ if (ClPrintMemProfMatchInfo || recordContextSizeInfoForAnalysis ())
386+ FullStackId = computeFullStackId (AllocInfo->CallStack );
387+ auto AllocType = addCallStack (AllocTrie, AllocInfo, FullStackId);
388+ TotalSize += AllocInfo->Info .getTotalSize ();
389+ if (AllocType == AllocationType::Cold)
390+ TotalColdSize += AllocInfo->Info .getTotalSize ();
391+ // Record information about the allocation if match info printing
392+ // was requested.
393+ if (ClPrintMemProfMatchInfo) {
394+ assert (FullStackId != 0 );
395+ FullStackIdToAllocMatchInfo[std::make_pair (FullStackId,
396+ InlinedCallStack.size ())] = {
397+ AllocInfo->Info .getTotalSize (), AllocType};
398+ }
399+ }
400+ }
401+ // If the threshold for the percent of cold bytes is less than 100%,
402+ // and not all bytes are cold, see if we should still hint this
403+ // allocation as cold without context sensitivity.
404+ if (TotalColdSize < TotalSize && MinMatchedColdBytePercent < 100 &&
405+ TotalColdSize * 100 >= MinMatchedColdBytePercent * TotalSize) {
406+ AllocTrie.addSingleAllocTypeAttribute (CI, AllocationType::Cold, " dominant" );
407+ return ;
408+ }
409+
410+ // We might not have matched any to the full inlined call stack.
411+ // But if we did, create and attach metadata, or a function attribute if
412+ // all contexts have identical profiled behavior.
413+ if (!AllocTrie.empty ()) {
414+ NumOfMemProfMatchedAllocs++;
415+ // MemprofMDAttached will be false if a function attribute was
416+ // attached.
417+ bool MemprofMDAttached = AllocTrie.buildAndAttachMIBMetadata (CI);
418+ assert (MemprofMDAttached == I.hasMetadata (LLVMContext::MD_memprof));
419+ if (MemprofMDAttached) {
420+ // Add callsite metadata for the instruction's location list so that
421+ // it simpler later on to identify which part of the MIB contexts
422+ // are from this particular instruction (including during inlining,
423+ // when the callsite metadata will be updated appropriately).
424+ // FIXME: can this be changed to strip out the matching stack
425+ // context ids from the MIB contexts and not add any callsite
426+ // metadata here to save space?
427+ addCallsiteMetadata (I, InlinedCallStack, Ctx);
428+ }
429+ }
430+ }
431+
364432static void readMemprof (Module &M, Function &F,
365433 IndexedInstrProfReader *MemProfReader,
366434 const TargetLibraryInfo &TLI,
@@ -554,66 +622,8 @@ static void readMemprof(Module &M, Function &F,
554622 if (AllocInfoIter != LocHashToAllocInfo.end () &&
555623 // Only consider allocations which support hinting.
556624 isAllocationWithHotColdVariant (CI->getCalledFunction (), TLI)) {
557- // We may match this instruction's location list to multiple MIB
558- // contexts. Add them to a Trie specialized for trimming the contexts to
559- // the minimal needed to disambiguate contexts with unique behavior.
560- CallStackTrie AllocTrie (&ORE, MaxColdSize);
561- uint64_t TotalSize = 0 ;
562- uint64_t TotalColdSize = 0 ;
563- for (auto *AllocInfo : AllocInfoIter->second ) {
564- // Check the full inlined call stack against this one.
565- // If we found and thus matched all frames on the call, include
566- // this MIB.
567- if (stackFrameIncludesInlinedCallStack (AllocInfo->CallStack ,
568- InlinedCallStack)) {
569- NumOfMemProfMatchedAllocContexts++;
570- uint64_t FullStackId = 0 ;
571- if (ClPrintMemProfMatchInfo || recordContextSizeInfoForAnalysis ())
572- FullStackId = computeFullStackId (AllocInfo->CallStack );
573- auto AllocType = addCallStack (AllocTrie, AllocInfo, FullStackId);
574- TotalSize += AllocInfo->Info .getTotalSize ();
575- if (AllocType == AllocationType::Cold)
576- TotalColdSize += AllocInfo->Info .getTotalSize ();
577- // Record information about the allocation if match info printing
578- // was requested.
579- if (ClPrintMemProfMatchInfo) {
580- assert (FullStackId != 0 );
581- FullStackIdToAllocMatchInfo[std::make_pair (
582- FullStackId, InlinedCallStack.size ())] = {
583- AllocInfo->Info .getTotalSize (), AllocType};
584- }
585- }
586- }
587- // If the threshold for the percent of cold bytes is less than 100%,
588- // and not all bytes are cold, see if we should still hint this
589- // allocation as cold without context sensitivity.
590- if (TotalColdSize < TotalSize && MinMatchedColdBytePercent < 100 &&
591- TotalColdSize * 100 >= MinMatchedColdBytePercent * TotalSize) {
592- AllocTrie.addSingleAllocTypeAttribute (CI, AllocationType::Cold,
593- " dominant" );
594- continue ;
595- }
596-
597- // We might not have matched any to the full inlined call stack.
598- // But if we did, create and attach metadata, or a function attribute if
599- // all contexts have identical profiled behavior.
600- if (!AllocTrie.empty ()) {
601- NumOfMemProfMatchedAllocs++;
602- // MemprofMDAttached will be false if a function attribute was
603- // attached.
604- bool MemprofMDAttached = AllocTrie.buildAndAttachMIBMetadata (CI);
605- assert (MemprofMDAttached == I.hasMetadata (LLVMContext::MD_memprof));
606- if (MemprofMDAttached) {
607- // Add callsite metadata for the instruction's location list so that
608- // it simpler later on to identify which part of the MIB contexts
609- // are from this particular instruction (including during inlining,
610- // when the callsite metadata will be updated appropriately).
611- // FIXME: can this be changed to strip out the matching stack
612- // context ids from the MIB contexts and not add any callsite
613- // metadata here to save space?
614- addCallsiteMetadata (I, InlinedCallStack, Ctx);
615- }
616- }
625+ handleAllocSite (I, CI, InlinedCallStack, Ctx, ORE, MaxColdSize,
626+ AllocInfoIter->second , FullStackIdToAllocMatchInfo);
617627 continue ;
618628 }
619629
0 commit comments