@@ -1109,74 +1109,38 @@ struct AAAMDWavesPerEU : public AAAMDSizeRangeAttribute {
11091109 Function *F = getAssociatedFunction ();
11101110 auto &InfoCache =
static_cast <AMDGPUInformationCache &>(A.
getInfoCache ());
11111111
1112- auto TakeRange = [&](std::pair<unsigned , unsigned > R) {
1113- auto [Min, Max] = R;
1114- ConstantRange Range (APInt (32 , Min), APInt (32 , Max + 1 ));
1115- IntegerRangeState RangeState (Range);
1116- clampStateAndIndicateChange (this ->getState (), RangeState);
1117- indicateOptimisticFixpoint ();
1118- };
1119-
1120- std::pair<unsigned , unsigned > MaxWavesPerEURange{
1121- 1U , InfoCache.getMaxWavesPerEU (*F)};
1122-
11231112 // If the attribute exists, we will honor it if it is not the default.
11241113 if (auto Attr = InfoCache.getWavesPerEUAttr (*F)) {
1114+ std::pair<unsigned , unsigned > MaxWavesPerEURange{
1115+ 1U , InfoCache.getMaxWavesPerEU (*F)};
11251116 if (*Attr != MaxWavesPerEURange) {
1126- TakeRange (*Attr);
1117+ auto [Min, Max] = *Attr;
1118+ ConstantRange Range (APInt (32 , Min), APInt (32 , Max + 1 ));
1119+ IntegerRangeState RangeState (Range);
1120+ clampStateAndIndicateChange (this ->getState (), RangeState);
1121+ indicateOptimisticFixpoint ();
11271122 return ;
11281123 }
11291124 }
11301125
1131- // Unlike AAAMDFlatWorkGroupSize, it's getting trickier here. Since the
1132- // calculation of waves per EU involves flat work group size, we can't
1133- // simply use an assumed flat work group size as a start point, because the
1134- // update of flat work group size is in an inverse direction of waves per
1135- // EU. However, we can still do something if it is an entry function. Since
1136- // an entry function is a terminal node, and flat work group size either
1137- // from attribute or default will be used anyway, we can take that value and
1138- // calculate the waves per EU based on it. This result can't be updated by
1139- // no means, but that could still allow us to propagate it.
1140- if (AMDGPU::isEntryFunctionCC (F->getCallingConv ())) {
1141- std::pair<unsigned , unsigned > FlatWorkGroupSize;
1142- if (auto Attr = InfoCache.getFlatWorkGroupSizeAttr (*F))
1143- FlatWorkGroupSize = *Attr;
1144- else
1145- FlatWorkGroupSize = InfoCache.getDefaultFlatWorkGroupSize (*F);
1146- TakeRange (InfoCache.getEffectiveWavesPerEU (*F, MaxWavesPerEURange,
1147- FlatWorkGroupSize));
1148- }
1126+ if (AMDGPU::isEntryFunctionCC (F->getCallingConv ()))
1127+ indicatePessimisticFixpoint ();
11491128 }
11501129
11511130 ChangeStatus updateImpl (Attributor &A) override {
1152- auto &InfoCache =
static_cast <AMDGPUInformationCache &>(A.
getInfoCache ());
11531131 ChangeStatus Change = ChangeStatus::UNCHANGED;
11541132
11551133 auto CheckCallSite = [&](AbstractCallSite CS) {
11561134 Function *Caller = CS.getInstruction ()->getFunction ();
1157- Function *Func = getAssociatedFunction ();
1158- LLVM_DEBUG (dbgs () << ' [' getName () << " ] Call " getName ()
1159- << " ->" getName () << ' \n '
1160-
11611135 const auto *CallerInfo = A.
getAAFor <AAAMDWavesPerEU>(
11621136 *this , IRPosition::function (*Caller), DepClassTy::REQUIRED);
1163- const auto *AssumedGroupSize = A.
getAAFor <AAAMDFlatWorkGroupSize>(
1164- *this , IRPosition::function (*Func), DepClassTy::REQUIRED);
1165- if (!CallerInfo || !AssumedGroupSize || !CallerInfo->isValidState () ||
1166- !AssumedGroupSize->isValidState ())
1137+ if (!CallerInfo || !CallerInfo->isValidState ())
11671138 return false ;
1168-
1169- unsigned Min, Max;
1170- std::tie (Min, Max) = InfoCache.getEffectiveWavesPerEU (
1171- *Caller,
1172- {CallerInfo->getAssumed ().getLower ().getZExtValue (),
1173- CallerInfo->getAssumed ().getUpper ().getZExtValue () - 1 },
1174- {AssumedGroupSize->getAssumed ().getLower ().getZExtValue (),
1175- AssumedGroupSize->getAssumed ().getUpper ().getZExtValue () - 1 });
1176- ConstantRange CallerRange (APInt (32 , Min), APInt (32 , Max + 1 ));
1139+ unsigned Min = CallerInfo->getAssumed ().getLower ().getZExtValue ();
1140+ unsigned Max = CallerInfo->getAssumed ().getUpper ().getZExtValue ();
1141+ ConstantRange CallerRange (APInt (32 , Min), APInt (32 , Max));
11771142 IntegerRangeState CallerRangeState (CallerRange);
11781143 Change |= clampStateAndIndicateChange (this ->getState (), CallerRangeState);
1179-
11801144 return true ;
11811145 };
11821146
@@ -1329,6 +1293,59 @@ static void addPreloadKernArgHint(Function &F, TargetMachine &TM) {
13291293 }
13301294}
13311295
1296+ static void checkWavesPerEU (Module &M, TargetMachine &TM) {
1297+ for (Function &F : M) {
1298+ const GCNSubtarget &ST = TM.getSubtarget <GCNSubtarget>(F);
1299+
1300+ auto FlatWgrpSizeAttr =
1301+ AMDGPU::getIntegerPairAttribute (F, " amdgpu-flat-work-group-size"
1302+ auto WavesPerEUAttr = AMDGPU::getIntegerPairAttribute (
1303+ F, " amdgpu-waves-per-eu" /* OnlyFirstRequired=*/ true );
1304+
1305+ unsigned MinWavesPerEU = ST.getMinWavesPerEU ();
1306+ unsigned MaxWavesPerEU = ST.getMaxWavesPerEU ();
1307+
1308+ unsigned MinFlatWgrpSize = 1U ;
1309+ unsigned MaxFlatWgrpSize = 1024U ;
1310+ if (FlatWgrpSizeAttr.has_value ()) {
1311+ MinFlatWgrpSize = FlatWgrpSizeAttr->first ;
1312+ MaxFlatWgrpSize = *(FlatWgrpSizeAttr->second );
1313+ }
1314+
1315+ // Start with the max range.
1316+ unsigned Min = MinWavesPerEU;
1317+ unsigned Max = MaxWavesPerEU;
1318+
1319+ // If the attribute exists, set them to the value from the attribute.
1320+ if (WavesPerEUAttr.has_value ()) {
1321+ Min = WavesPerEUAttr->first ;
1322+ if (WavesPerEUAttr->second .has_value ())
1323+ Max = *(WavesPerEUAttr->second );
1324+ }
1325+
1326+ // Compute the range from flat workgroup size.
1327+ auto [MinFromFlatWgrpSize, MaxFromFlatWgrpSize] =
1328+ ST.getWavesPerEU (F, std::make_pair (MinFlatWgrpSize, MaxFlatWgrpSize));
1329+
1330+ // For the lower bound, we have to "tighten" it.
1331+ Min = std::max (Min, MinFromFlatWgrpSize);
1332+ // For the upper bound, we have to "extend" it.
1333+ Max = std::max (Max, MaxFromFlatWgrpSize);
1334+
1335+ // Clamp the range to the max range.
1336+ Min = std::max (Min, MinWavesPerEU);
1337+ Max = std::min (Max, MaxWavesPerEU);
1338+
1339+ // Update the attribute if it is not the max.
1340+ if (Min != MinWavesPerEU || Max != MaxWavesPerEU) {
1341+ SmallString<10 > Buffer;
1342+ raw_svector_ostream OS (Buffer);
1343+ OS << Min << ' ,'
1344+ F.addFnAttr (" amdgpu-waves-per-eu" str ());
1345+ }
1346+ }
1347+ }
1348+
13321349static bool runImpl (Module &M, AnalysisGetter &AG, TargetMachine &TM,
13331350 AMDGPUAttributorOptions Options,
13341351 ThinOrFullLTOPhase LTOPhase) {
@@ -1421,8 +1438,14 @@ static bool runImpl(Module &M, AnalysisGetter &AG, TargetMachine &TM,
14211438 }
14221439 }
14231440
1424- ChangeStatus Change = A.run ();
1425- return Change == ChangeStatus::CHANGED;
1441+ bool Changed = A.run () == ChangeStatus::CHANGED;
1442+
1443+ if (Changed && (LTOPhase == ThinOrFullLTOPhase::None ||
1444+ LTOPhase == ThinOrFullLTOPhase::FullLTOPostLink ||
1445+ LTOPhase == ThinOrFullLTOPhase::ThinLTOPostLink))
1446+ checkWavesPerEU (M, TM);
1447+
1448+ return Changed;
14261449}
14271450
14281451class AMDGPUAttributorLegacy : public ModulePass {
0 commit comments