@@ -379,50 +379,55 @@ bool PreISelIntrinsicLowering::expandMemIntrinsicUses(Function &F) const {
379379 case Intrinsic::experimental_memset_pattern: {
380380 auto *Memset = cast<MemSetPatternInst>(Inst);
381381 const TargetLibraryInfo &TLI = LookupTLI (*Memset->getFunction ());
382- if (Constant *PatternValue = getMemSetPattern16Value (Memset, TLI)) {
383- // FIXME: There is currently no profitability calculation for emitting
384- // the libcall vs expanding the memset.pattern directly.
385- IRBuilder<> Builder (Inst);
386- Module *M = Memset->getModule ();
387- const DataLayout &DL = Memset->getDataLayout ();
388-
389- StringRef FuncName = " memset_pattern16" ;
390- FunctionCallee MSP = getOrInsertLibFunc (
391- M, TLI, LibFunc_memset_pattern16, Builder.getVoidTy (),
392- Memset->getRawDest ()->getType (), Builder.getPtrTy (),
393- Memset->getLength ()->getType ());
394- inferNonMandatoryLibFuncAttrs (M, FuncName, TLI);
395-
396- // Otherwise we should form a memset_pattern16. PatternValue is known
397- // to be an constant array of 16-bytes. Put the value into a mergable
398- // global.
399- assert (Memset->getRawDest ()->getType ()->getPointerAddressSpace () == 0 &&
400- " Should have skipped if non-zero AS" );
401- GlobalVariable *GV = new GlobalVariable (
402- *M, PatternValue->getType (), /* isConstant=*/ true ,
403- GlobalValue::PrivateLinkage, PatternValue, " .memset_pattern" );
404- GV->setUnnamedAddr (
405- GlobalValue::UnnamedAddr::Global); // Ok to merge these.
406- // TODO: Consider relaxing alignment requirement.
407- GV->setAlignment (Align (16 ));
408- Value *PatternPtr = GV;
409- Value *NumBytes = Builder.CreateMul (
410- Builder.getInt64 (
411- DL.getTypeSizeInBits (Memset->getValue ()->getType ()) / 8 ),
412- Memset->getLength ());
413- CallInst *MemsetPattern16Call = Builder.CreateCall (
414- MSP, {Memset->getRawDest (), PatternPtr, NumBytes});
415- MemsetPattern16Call->setAAMetadata (Memset->getAAMetadata ());
416- // Preserve any call site attributes on the destination pointer
417- // argument (e.g. alignment).
418- AttrBuilder ArgAttrs (Memset->getContext (),
419- Memset->getAttributes ().getParamAttrs (0 ));
420- MemsetPattern16Call->setAttributes (
421- MemsetPattern16Call->getAttributes ().addParamAttributes (
422- Memset->getContext (), 0 , ArgAttrs));
423- } else {
382+ Constant *PatternValue = getMemSetPattern16Value (Memset, TLI);
383+ if (!PatternValue) {
384+ // If it isn't possible to emit a memset_pattern16 libcall, expand to
385+ // a loop instead.
424386 expandMemSetPatternAsLoop (Memset);
387+ Changed = true ;
388+ Memset->eraseFromParent ();
389+ break ;
425390 }
391+ // FIXME: There is currently no profitability calculation for emitting
392+ // the libcall vs expanding the memset.pattern directly.
393+ IRBuilder<> Builder (Inst);
394+ Module *M = Memset->getModule ();
395+ const DataLayout &DL = Memset->getDataLayout ();
396+
397+ StringRef FuncName = " memset_pattern16" ;
398+ FunctionCallee MSP = getOrInsertLibFunc (
399+ M, TLI, LibFunc_memset_pattern16, Builder.getVoidTy (),
400+ Memset->getRawDest ()->getType (), Builder.getPtrTy (),
401+ Memset->getLength ()->getType ());
402+ inferNonMandatoryLibFuncAttrs (M, FuncName, TLI);
403+
404+ // Otherwise we should form a memset_pattern16. PatternValue is known
405+ // to be an constant array of 16-bytes. Put the value into a mergable
406+ // global.
407+ assert (Memset->getRawDest ()->getType ()->getPointerAddressSpace () == 0 &&
408+ " Should have skipped if non-zero AS" );
409+ GlobalVariable *GV = new GlobalVariable (
410+ *M, PatternValue->getType (), /* isConstant=*/ true ,
411+ GlobalValue::PrivateLinkage, PatternValue, " .memset_pattern" );
412+ GV->setUnnamedAddr (
413+ GlobalValue::UnnamedAddr::Global); // Ok to merge these.
414+ // TODO: Consider relaxing alignment requirement.
415+ GV->setAlignment (Align (16 ));
416+ Value *PatternPtr = GV;
417+ Value *NumBytes = Builder.CreateMul (
418+ Builder.getInt64 (DL.getTypeSizeInBits (Memset->getValue ()->getType ()) /
419+ 8 ),
420+ Memset->getLength ());
421+ CallInst *MemsetPattern16Call =
422+ Builder.CreateCall (MSP, {Memset->getRawDest (), PatternPtr, NumBytes});
423+ MemsetPattern16Call->setAAMetadata (Memset->getAAMetadata ());
424+ // Preserve any call site attributes on the destination pointer
425+ // argument (e.g. alignment).
426+ AttrBuilder ArgAttrs (Memset->getContext (),
427+ Memset->getAttributes ().getParamAttrs (0 ));
428+ MemsetPattern16Call->setAttributes (
429+ MemsetPattern16Call->getAttributes ().addParamAttributes (
430+ Memset->getContext (), 0 , ArgAttrs));
426431 Changed = true ;
427432 Memset->eraseFromParent ();
428433 break ;
0 commit comments