diff --git a/clang/include/clang/Basic/Attr.td b/clang/include/clang/Basic/Attr.td index b055cbd769bb5..746c05be31377 100644 --- a/clang/include/clang/Basic/Attr.td +++ b/clang/include/clang/Basic/Attr.td @@ -3260,33 +3260,28 @@ def Target : InheritableAttr { let Subjects = SubjectList<[Function], ErrorDiag>; let Documentation = [TargetDocs]; let AdditionalMembers = [{ - StringRef getArchitecture() const { + std::optional getX86Architecture() const { StringRef Features = getFeaturesStr(); - if (Features == "default") return {}; - - SmallVector AttrFeatures; - Features.split(AttrFeatures, ","); - - for (auto &Feature : AttrFeatures) { + SmallVector AttrFeatures; + Features.split(AttrFeatures, ','); + for (StringRef Feature : AttrFeatures) { Feature = Feature.trim(); if (Feature.starts_with("arch=")) return Feature.drop_front(sizeof("arch=") - 1); } - return ""; + return std::nullopt; } // Gets the list of features as simple string-refs with no +/- or 'no-'. // Only adds the items to 'Out' that are additions. - void getAddedFeatures(llvm::SmallVectorImpl &Out) const { + void getX86AddedFeatures(llvm::SmallVectorImpl &Out) const { + if (isDefaultVersion()) + return; StringRef Features = getFeaturesStr(); - if (Features == "default") return; - - SmallVector AttrFeatures; - Features.split(AttrFeatures, ","); - + SmallVector AttrFeatures; + Features.split(AttrFeatures, ','); for (auto &Feature : AttrFeatures) { Feature = Feature.trim(); - if (!Feature.starts_with("no-") && !Feature.starts_with("arch=") && !Feature.starts_with("fpmath=") && !Feature.starts_with("tune=")) Out.push_back(Feature); @@ -3304,17 +3299,17 @@ def TargetVersion : InheritableAttr, TargetSpecificAttr &Out) const { - if (isDefaultVersion()) return; - StringRef Features = getName(); - SmallVector AttrFeatures; - Features.split(AttrFeatures, "+"); + bool isDefaultVersion() const { return getName() == "default"; } - for (auto &Feature : AttrFeatures) { + void getFeatures(llvm::SmallVectorImpl &Out, + char Delim = '+') const { + if (isDefaultVersion()) + return; + StringRef Features = getName(); + SmallVector AttrFeatures; + Features.split(AttrFeatures, Delim); + for (StringRef Feature : AttrFeatures) { Feature = Feature.trim(); Out.push_back(Feature); } @@ -3331,20 +3326,40 @@ def TargetClones : InheritableAttr { StringRef getFeatureStr(unsigned Index) const { return *(featuresStrs_begin() + Index); } + bool isDefaultVersion(unsigned Index) const { return getFeatureStr(Index) == "default"; } + void getFeatures(llvm::SmallVectorImpl &Out, - unsigned Index) const { - if (isDefaultVersion(Index)) return; + unsigned Index, char Delim = '+') const { + if (isDefaultVersion(Index)) + return; StringRef Features = getFeatureStr(Index); - SmallVector AttrFeatures; - Features.split(AttrFeatures, "+"); - for (auto &Feature : AttrFeatures) { + SmallVector AttrFeatures; + Features.split(AttrFeatures, Delim); + for (StringRef Feature : AttrFeatures) { Feature = Feature.trim(); Out.push_back(Feature); } } + + std::optional getX86Architecture(unsigned Index) const { + StringRef Feature = getFeatureStr(Index); + if (Feature.starts_with("arch=")) + return Feature.drop_front(sizeof("arch=") - 1); + return std::nullopt; + } + + void getX86Feature(llvm::SmallVectorImpl &Out, + unsigned Index) const { + if (isDefaultVersion(Index)) + return; + if (getX86Architecture(Index)) + return; + Out.push_back(getFeatureStr(Index)); + } + // Given an index into the 'featuresStrs' sequence, compute a unique // ID to be used with function name mangling for the associated variant. // This mapping is necessary due to a requirement that the mangling ID diff --git a/clang/include/clang/Basic/TargetInfo.h b/clang/include/clang/Basic/TargetInfo.h index 9cd23d123f2ba..4420228793e95 100644 --- a/clang/include/clang/Basic/TargetInfo.h +++ b/clang/include/clang/Basic/TargetInfo.h @@ -1525,14 +1525,10 @@ class TargetInfo : public TransferrableTargetInfo, // Return the target-specific priority for features/cpus/vendors so // that they can be properly sorted for checking. - virtual unsigned multiVersionSortPriority(StringRef Name) const { + virtual unsigned getFMVPriority(ArrayRef Features) const { return 0; } - // Return the target-specific cost for feature - // that taken into account in priority sorting. - virtual unsigned multiVersionFeatureCost() const { return 0; } - // Validate the contents of the __builtin_cpu_is(const char*) // argument. virtual bool validateCpuIs(StringRef Name) const { return false; } diff --git a/clang/lib/Basic/Targets/AArch64.cpp b/clang/lib/Basic/Targets/AArch64.cpp index eb8a3ada03448..4efc1841c836d 100644 --- a/clang/lib/Basic/Targets/AArch64.cpp +++ b/clang/lib/Basic/Targets/AArch64.cpp @@ -705,18 +705,8 @@ AArch64TargetInfo::getVScaleRange(const LangOptions &LangOpts) const { return std::nullopt; } -unsigned AArch64TargetInfo::multiVersionSortPriority(StringRef Name) const { - if (Name == "default") - return 0; - if (auto Ext = llvm::AArch64::parseFMVExtension(Name)) - return Ext->Priority; - return 0; -} - -unsigned AArch64TargetInfo::multiVersionFeatureCost() const { - // Take the maximum priority as per feature cost, so more features win. - constexpr unsigned MaxFMVPriority = 1000; - return MaxFMVPriority; +unsigned AArch64TargetInfo::getFMVPriority(ArrayRef Features) const { + return llvm::AArch64::getFMVPriority(Features); } bool AArch64TargetInfo::doesFeatureAffectCodeGen(StringRef Name) const { diff --git a/clang/lib/Basic/Targets/AArch64.h b/clang/lib/Basic/Targets/AArch64.h index 4c25bdb5bb16d..68a8b1ebad8cd 100644 --- a/clang/lib/Basic/Targets/AArch64.h +++ b/clang/lib/Basic/Targets/AArch64.h @@ -137,8 +137,7 @@ class LLVM_LIBRARY_VISIBILITY AArch64TargetInfo : public TargetInfo { void fillValidCPUList(SmallVectorImpl &Values) const override; bool setCPU(const std::string &Name) override; - unsigned multiVersionSortPriority(StringRef Name) const override; - unsigned multiVersionFeatureCost() const override; + unsigned getFMVPriority(ArrayRef Features) const override; bool useFP16ConversionIntrinsics() const override { return false; diff --git a/clang/lib/Basic/Targets/RISCV.cpp b/clang/lib/Basic/Targets/RISCV.cpp index 0f8607c22d447..a541dfedc9b8e 100644 --- a/clang/lib/Basic/Targets/RISCV.cpp +++ b/clang/lib/Basic/Targets/RISCV.cpp @@ -423,6 +423,24 @@ ParsedTargetAttr RISCVTargetInfo::parseTargetAttr(StringRef Features) const { Features.split(AttrFeatures, ";"); bool FoundArch = false; + auto handleArchExtension = [](StringRef AttrString, + std::vector &Features) { + SmallVector Exts; + AttrString.split(Exts, ","); + for (auto Ext : Exts) { + if (Ext.empty()) + continue; + + StringRef ExtName = Ext.substr(1); + std::string TargetFeature = + llvm::RISCVISAInfo::getTargetFeatureForExtension(ExtName); + if (!TargetFeature.empty()) + Features.push_back(Ext.front() + TargetFeature); + else + Features.push_back(Ext.str()); + } + }; + for (auto &Feature : AttrFeatures) { Feature = Feature.trim(); StringRef AttrString = Feature.split("=").second.trim(); @@ -436,20 +454,7 @@ ParsedTargetAttr RISCVTargetInfo::parseTargetAttr(StringRef Features) const { if (AttrString.starts_with("+")) { // EXTENSION like arch=+v,+zbb - SmallVector Exts; - AttrString.split(Exts, ","); - for (auto Ext : Exts) { - if (Ext.empty()) - continue; - - StringRef ExtName = Ext.substr(1); - std::string TargetFeature = - llvm::RISCVISAInfo::getTargetFeatureForExtension(ExtName); - if (!TargetFeature.empty()) - Ret.Features.push_back(Ext.front() + TargetFeature); - else - Ret.Features.push_back(Ext.str()); - } + handleArchExtension(AttrString, Ret.Features); } else { // full-arch-string like arch=rv64gcv handleFullArchString(AttrString, Ret.Features); @@ -475,11 +480,35 @@ ParsedTargetAttr RISCVTargetInfo::parseTargetAttr(StringRef Features) const { Ret.Tune = AttrString; } else if (Feature.starts_with("priority")) { // Skip because it only use for FMV. + } else if (Feature.starts_with("+")) { + // Handle target_version/target_clones attribute strings + // that are already delimited by ',' + handleArchExtension(Feature, Ret.Features); } } return Ret; } +unsigned RISCVTargetInfo::getFMVPriority(ArrayRef Features) const { + // Priority is explicitly specified on RISC-V unlike on other targets, where + // it is derived by all the features of a specific version. Therefore if a + // feature contains the priority string, then return it immediately. + for (StringRef Feature : Features) { + auto [LHS, RHS] = Feature.rsplit(';'); + if (LHS.consume_front("priority=")) + Feature = LHS; + else if (RHS.consume_front("priority=")) + Feature = RHS; + else + continue; + unsigned Priority; + if (!Feature.getAsInteger(0, Priority)) + return Priority; + } + // Default Priority is zero. + return 0; +} + TargetInfo::CallingConvCheckResult RISCVTargetInfo::checkCallingConvention(CallingConv CC) const { switch (CC) { diff --git a/clang/lib/Basic/Targets/RISCV.h b/clang/lib/Basic/Targets/RISCV.h index 3544ea64cb5e7..68f10e74ba98c 100644 --- a/clang/lib/Basic/Targets/RISCV.h +++ b/clang/lib/Basic/Targets/RISCV.h @@ -122,6 +122,7 @@ class RISCVTargetInfo : public TargetInfo { void fillValidTuneCPUList(SmallVectorImpl &Values) const override; bool supportsTargetAttributeTune() const override { return true; } ParsedTargetAttr parseTargetAttr(StringRef Str) const override; + unsigned getFMVPriority(ArrayRef Features) const override; std::pair hardwareInterferenceSizes() const override { return std::make_pair(32, 32); diff --git a/clang/lib/Basic/Targets/X86.cpp b/clang/lib/Basic/Targets/X86.cpp index 5993257e27d5a..38714d0e7d3c5 100644 --- a/clang/lib/Basic/Targets/X86.cpp +++ b/clang/lib/Basic/Targets/X86.cpp @@ -1364,19 +1364,26 @@ static llvm::X86::ProcessorFeatures getFeature(StringRef Name) { // correct, so it asserts if the value is out of range. } -unsigned X86TargetInfo::multiVersionSortPriority(StringRef Name) const { - // Valid CPUs have a 'key feature' that compares just better than its key - // feature. - using namespace llvm::X86; - CPUKind Kind = parseArchX86(Name); - if (Kind != CK_None) { - ProcessorFeatures KeyFeature = getKeyFeature(Kind); - return (getFeaturePriority(KeyFeature) << 1) + 1; - } - - // Now we know we have a feature, so get its priority and shift it a few so - // that we have sufficient room for the CPUs (above). - return getFeaturePriority(getFeature(Name)) << 1; +unsigned X86TargetInfo::getFMVPriority(ArrayRef Features) const { + auto getPriority = [this](StringRef Feature) -> unsigned { + // Valid CPUs have a 'key feature' that compares just better than its key + // feature. + using namespace llvm::X86; + CPUKind Kind = parseArchX86(Feature); + if (Kind != CK_None) { + ProcessorFeatures KeyFeature = getKeyFeature(Kind); + return (getFeaturePriority(KeyFeature) << 1) + 1; + } + // Now we know we have a feature, so get its priority and shift it a few so + // that we have sufficient room for the CPUs (above). + return getFeaturePriority(getFeature(Feature)) << 1; + }; + + unsigned Priority = 0; + for (StringRef Feature : Features) + if (!Feature.empty()) + Priority = std::max(Priority, getPriority(Feature)); + return Priority; } bool X86TargetInfo::validateCPUSpecificCPUDispatch(StringRef Name) const { diff --git a/clang/lib/Basic/Targets/X86.h b/clang/lib/Basic/Targets/X86.h index 06a7eed8177cb..3ed36c8fa724b 100644 --- a/clang/lib/Basic/Targets/X86.h +++ b/clang/lib/Basic/Targets/X86.h @@ -384,7 +384,7 @@ class LLVM_LIBRARY_VISIBILITY X86TargetInfo : public TargetInfo { return CPU != llvm::X86::CK_None; } - unsigned multiVersionSortPriority(StringRef Name) const override; + unsigned getFMVPriority(ArrayRef Features) const override; bool setFPMath(StringRef Name) override; diff --git a/clang/lib/CodeGen/ABIInfo.cpp b/clang/lib/CodeGen/ABIInfo.cpp index edd7146dc1ac7..8e76cf15b642c 100644 --- a/clang/lib/CodeGen/ABIInfo.cpp +++ b/clang/lib/CodeGen/ABIInfo.cpp @@ -218,8 +218,8 @@ void ABIInfo::appendAttributeMangling(StringRef AttrStr, // only have "+" prefixes here. assert(LHS.starts_with("+") && RHS.starts_with("+") && "Features should always have a prefix."); - return TI.multiVersionSortPriority(LHS.substr(1)) > - TI.multiVersionSortPriority(RHS.substr(1)); + return TI.getFMVPriority({LHS.substr(1)}) > + TI.getFMVPriority({RHS.substr(1)}); }); bool IsFirst = true; diff --git a/clang/lib/CodeGen/CodeGenFunction.cpp b/clang/lib/CodeGen/CodeGenFunction.cpp index ef6bb4f049d6e..f8138e9cee250 100644 --- a/clang/lib/CodeGen/CodeGenFunction.cpp +++ b/clang/lib/CodeGen/CodeGenFunction.cpp @@ -2828,23 +2828,17 @@ void CodeGenFunction::EmitKCFIOperandBundle( Bundles.emplace_back("kcfi", CGM.CreateKCFITypeId(FP->desugar())); } -llvm::Value *CodeGenFunction::FormAArch64ResolverCondition( - const MultiVersionResolverOption &RO) { - llvm::SmallVector CondFeatures; - for (const StringRef &Feature : RO.Conditions.Features) - CondFeatures.push_back(Feature); - if (!CondFeatures.empty()) { - return EmitAArch64CpuSupports(CondFeatures); - } - return nullptr; +llvm::Value * +CodeGenFunction::FormAArch64ResolverCondition(const FMVResolverOption &RO) { + return RO.Features.empty() ? nullptr : EmitAArch64CpuSupports(RO.Features); } -llvm::Value *CodeGenFunction::FormX86ResolverCondition( - const MultiVersionResolverOption &RO) { +llvm::Value * +CodeGenFunction::FormX86ResolverCondition(const FMVResolverOption &RO) { llvm::Value *Condition = nullptr; - if (!RO.Conditions.Architecture.empty()) { - StringRef Arch = RO.Conditions.Architecture; + if (RO.Architecture) { + StringRef Arch = *RO.Architecture; // If arch= specifies an x86-64 micro-architecture level, test the feature // with __builtin_cpu_supports, otherwise use __builtin_cpu_is. if (Arch.starts_with("x86-64")) @@ -2853,8 +2847,8 @@ llvm::Value *CodeGenFunction::FormX86ResolverCondition( Condition = EmitX86CpuIs(Arch); } - if (!RO.Conditions.Features.empty()) { - llvm::Value *FeatureCond = EmitX86CpuSupports(RO.Conditions.Features); + if (!RO.Features.empty()) { + llvm::Value *FeatureCond = EmitX86CpuSupports(RO.Features); Condition = Condition ? Builder.CreateAnd(Condition, FeatureCond) : FeatureCond; } @@ -2884,7 +2878,7 @@ static void CreateMultiVersionResolverReturn(CodeGenModule &CGM, } void CodeGenFunction::EmitMultiVersionResolver( - llvm::Function *Resolver, ArrayRef Options) { + llvm::Function *Resolver, ArrayRef Options) { llvm::Triple::ArchType ArchType = getContext().getTargetInfo().getTriple().getArch(); @@ -2907,26 +2901,8 @@ void CodeGenFunction::EmitMultiVersionResolver( } } -static unsigned getPriorityFromAttrString(StringRef AttrStr) { - SmallVector Attrs; - - AttrStr.split(Attrs, ';'); - - // Default Priority is zero. - unsigned Priority = 0; - for (auto Attr : Attrs) { - if (Attr.consume_front("priority=")) { - unsigned Result; - if (!Attr.getAsInteger(0, Result)) - Priority = Result; - } - } - - return Priority; -} - void CodeGenFunction::EmitRISCVMultiVersionResolver( - llvm::Function *Resolver, ArrayRef Options) { + llvm::Function *Resolver, ArrayRef Options) { if (getContext().getTargetInfo().getTriple().getOS() != llvm::Triple::OSType::Linux) { @@ -2942,20 +2918,10 @@ void CodeGenFunction::EmitRISCVMultiVersionResolver( bool HasDefault = false; unsigned DefaultIndex = 0; - SmallVector CurrOptions( - Options); - - llvm::stable_sort( - CurrOptions, [](const CodeGenFunction::MultiVersionResolverOption &LHS, - const CodeGenFunction::MultiVersionResolverOption &RHS) { - return getPriorityFromAttrString(LHS.Conditions.Features[0]) > - getPriorityFromAttrString(RHS.Conditions.Features[0]); - }); - // Check the each candidate function. - for (unsigned Index = 0; Index < CurrOptions.size(); Index++) { + for (unsigned Index = 0; Index < Options.size(); Index++) { - if (CurrOptions[Index].Conditions.Features[0].starts_with("default")) { + if (Options[Index].Features.empty()) { HasDefault = true; DefaultIndex = Index; continue; @@ -2963,15 +2929,6 @@ void CodeGenFunction::EmitRISCVMultiVersionResolver( Builder.SetInsertPoint(CurBlock); - std::vector TargetAttrFeats = - getContext() - .getTargetInfo() - .parseTargetAttr(CurrOptions[Index].Conditions.Features[0]) - .Features; - - if (TargetAttrFeats.empty()) - continue; - // FeaturesCondition: The bitmask of the required extension has been // enabled by the runtime object. // (__riscv_feature_bits.features[i] & REQUIRED_BITMASK) == @@ -2994,20 +2951,32 @@ void CodeGenFunction::EmitRISCVMultiVersionResolver( // Without checking the length first, we may access an incorrect memory // address when using different versions. llvm::SmallVector CurrTargetAttrFeats; + llvm::SmallVector TargetAttrFeats; - for (auto &Feat : TargetAttrFeats) { - StringRef CurrFeat = Feat; - if (CurrFeat.starts_with('+')) - CurrTargetAttrFeats.push_back(CurrFeat.substr(1)); + for (StringRef Feat : Options[Index].Features) { + std::vector FeatStr = + getContext().getTargetInfo().parseTargetAttr(Feat).Features; + + assert(FeatStr.size() == 1 && "Feature string not delimited"); + + std::string &CurrFeat = FeatStr.front(); + if (CurrFeat[0] == '+') + TargetAttrFeats.push_back(CurrFeat.substr(1)); } + if (TargetAttrFeats.empty()) + continue; + + for (std::string &Feat : TargetAttrFeats) + CurrTargetAttrFeats.push_back(Feat); + Builder.SetInsertPoint(CurBlock); llvm::Value *FeatsCondition = EmitRISCVCpuSupports(CurrTargetAttrFeats); llvm::BasicBlock *RetBlock = createBasicBlock("resolver_return", Resolver); CGBuilderTy RetBuilder(*this, RetBlock); - CreateMultiVersionResolverReturn( - CGM, Resolver, RetBuilder, CurrOptions[Index].Function, SupportsIFunc); + CreateMultiVersionResolverReturn(CGM, Resolver, RetBuilder, + Options[Index].Function, SupportsIFunc); llvm::BasicBlock *ElseBlock = createBasicBlock("resolver_else", Resolver); Builder.SetInsertPoint(CurBlock); @@ -3019,9 +2988,8 @@ void CodeGenFunction::EmitRISCVMultiVersionResolver( // Finally, emit the default one. if (HasDefault) { Builder.SetInsertPoint(CurBlock); - CreateMultiVersionResolverReturn(CGM, Resolver, Builder, - CurrOptions[DefaultIndex].Function, - SupportsIFunc); + CreateMultiVersionResolverReturn( + CGM, Resolver, Builder, Options[DefaultIndex].Function, SupportsIFunc); return; } @@ -3035,17 +3003,16 @@ void CodeGenFunction::EmitRISCVMultiVersionResolver( } void CodeGenFunction::EmitAArch64MultiVersionResolver( - llvm::Function *Resolver, ArrayRef Options) { + llvm::Function *Resolver, ArrayRef Options) { assert(!Options.empty() && "No multiversion resolver options found"); - assert(Options.back().Conditions.Features.size() == 0 && - "Default case must be last"); + assert(Options.back().Features.size() == 0 && "Default case must be last"); bool SupportsIFunc = getContext().getTargetInfo().supportsIFunc(); assert(SupportsIFunc && "Multiversion resolver requires target IFUNC support"); bool AArch64CpuInitialized = false; llvm::BasicBlock *CurBlock = createBasicBlock("resolver_entry", Resolver); - for (const MultiVersionResolverOption &RO : Options) { + for (const FMVResolverOption &RO : Options) { Builder.SetInsertPoint(CurBlock); llvm::Value *Condition = FormAArch64ResolverCondition(RO); @@ -3081,7 +3048,7 @@ void CodeGenFunction::EmitAArch64MultiVersionResolver( } void CodeGenFunction::EmitX86MultiVersionResolver( - llvm::Function *Resolver, ArrayRef Options) { + llvm::Function *Resolver, ArrayRef Options) { bool SupportsIFunc = getContext().getTargetInfo().supportsIFunc(); @@ -3090,7 +3057,7 @@ void CodeGenFunction::EmitX86MultiVersionResolver( Builder.SetInsertPoint(CurBlock); EmitX86CpuInit(); - for (const MultiVersionResolverOption &RO : Options) { + for (const FMVResolverOption &RO : Options) { Builder.SetInsertPoint(CurBlock); llvm::Value *Condition = FormX86ResolverCondition(RO); diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h index 5c4d76c2267a7..eaea0d8a08ac0 100644 --- a/clang/lib/CodeGen/CodeGenFunction.h +++ b/clang/lib/CodeGen/CodeGenFunction.h @@ -5334,35 +5334,27 @@ class CodeGenFunction : public CodeGenTypeCache { void EmitSanitizerStatReport(llvm::SanitizerStatKind SSK); - struct MultiVersionResolverOption { + struct FMVResolverOption { llvm::Function *Function; - struct Conds { - StringRef Architecture; - llvm::SmallVector Features; + llvm::SmallVector Features; + std::optional Architecture; - Conds(StringRef Arch, ArrayRef Feats) - : Architecture(Arch), Features(Feats) {} - } Conditions; - - MultiVersionResolverOption(llvm::Function *F, StringRef Arch, - ArrayRef Feats) - : Function(F), Conditions(Arch, Feats) {} + FMVResolverOption(llvm::Function *F, ArrayRef Feats, + std::optional Arch = std::nullopt) + : Function(F), Features(Feats), Architecture(Arch) {} }; // Emits the body of a multiversion function's resolver. Assumes that the // options are already sorted in the proper order, with the 'default' option // last (if it exists). void EmitMultiVersionResolver(llvm::Function *Resolver, - ArrayRef Options); - void - EmitX86MultiVersionResolver(llvm::Function *Resolver, - ArrayRef Options); - void - EmitAArch64MultiVersionResolver(llvm::Function *Resolver, - ArrayRef Options); - void - EmitRISCVMultiVersionResolver(llvm::Function *Resolver, - ArrayRef Options); + ArrayRef Options); + void EmitX86MultiVersionResolver(llvm::Function *Resolver, + ArrayRef Options); + void EmitAArch64MultiVersionResolver(llvm::Function *Resolver, + ArrayRef Options); + void EmitRISCVMultiVersionResolver(llvm::Function *Resolver, + ArrayRef Options); private: QualType getVarArgType(const Expr *Arg); @@ -5381,10 +5373,9 @@ class CodeGenFunction : public CodeGenTypeCache { llvm::Value *EmitX86CpuSupports(ArrayRef FeatureStrs); llvm::Value *EmitX86CpuSupports(std::array FeatureMask); llvm::Value *EmitX86CpuInit(); - llvm::Value *FormX86ResolverCondition(const MultiVersionResolverOption &RO); + llvm::Value *FormX86ResolverCondition(const FMVResolverOption &RO); llvm::Value *EmitAArch64CpuInit(); - llvm::Value * - FormAArch64ResolverCondition(const MultiVersionResolverOption &RO); + llvm::Value *FormAArch64ResolverCondition(const FMVResolverOption &RO); llvm::Value *EmitAArch64CpuSupports(const CallExpr *E); llvm::Value *EmitAArch64CpuSupports(ArrayRef FeatureStrs); }; diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp index 716c43431667c..7189a4689e815 100644 --- a/clang/lib/CodeGen/CodeGenModule.cpp +++ b/clang/lib/CodeGen/CodeGenModule.cpp @@ -4223,23 +4223,12 @@ void CodeGenModule::EmitGlobalDefinition(GlobalDecl GD, llvm::GlobalValue *GV) { static void ReplaceUsesOfNonProtoTypeWithRealFunction(llvm::GlobalValue *Old, llvm::Function *NewFn); -static unsigned -TargetMVPriority(const TargetInfo &TI, - const CodeGenFunction::MultiVersionResolverOption &RO) { - unsigned Priority = 0; - unsigned NumFeatures = 0; - for (StringRef Feat : RO.Conditions.Features) { - Priority = std::max(Priority, TI.multiVersionSortPriority(Feat)); - NumFeatures++; - } - - if (!RO.Conditions.Architecture.empty()) - Priority = std::max( - Priority, TI.multiVersionSortPriority(RO.Conditions.Architecture)); - - Priority += TI.multiVersionFeatureCost() * NumFeatures; - - return Priority; +static unsigned getFMVPriority(const TargetInfo &TI, + const CodeGenFunction::FMVResolverOption &RO) { + llvm::SmallVector Features{RO.Features}; + if (RO.Architecture) + Features.push_back(*RO.Architecture); + return TI.getFMVPriority(Features); } // Multiversion functions should be at most 'WeakODRLinkage' so that a different @@ -4285,7 +4274,7 @@ void CodeGenModule::emitMultiVersionFunctions() { // target_version("default")) or target_clones() is present and defined // in this TU. For other architectures it is always emitted. bool ShouldEmitResolver = !getTarget().getTriple().isAArch64(); - SmallVector Options; + SmallVector Options; getContext().forEachMultiversionedFunctionVersion( FD, [&](const FunctionDecl *CurFD) { @@ -4293,20 +4282,17 @@ void CodeGenModule::emitMultiVersionFunctions() { bool IsDefined = CurFD->doesThisDeclarationHaveABody(); if (const auto *TA = CurFD->getAttr()) { - TA->getAddedFeatures(Feats); + assert(getTarget().getTriple().isX86() && "Unsupported target"); + TA->getX86AddedFeatures(Feats); llvm::Function *Func = createFunction(CurFD); - Options.emplace_back(Func, TA->getArchitecture(), Feats); + Options.emplace_back(Func, Feats, TA->getX86Architecture()); } else if (const auto *TVA = CurFD->getAttr()) { if (TVA->isDefaultVersion() && IsDefined) ShouldEmitResolver = true; llvm::Function *Func = createFunction(CurFD); - if (getTarget().getTriple().isRISCV()) { - Feats.push_back(TVA->getName()); - } else { - assert(getTarget().getTriple().isAArch64()); - TVA->getFeatures(Feats); - } - Options.emplace_back(Func, /*Architecture*/ "", Feats); + char Delim = getTarget().getTriple().isAArch64() ? '+' : ','; + TVA->getFeatures(Feats, Delim); + Options.emplace_back(Func, Feats); } else if (const auto *TC = CurFD->getAttr()) { if (IsDefined) ShouldEmitResolver = true; @@ -4315,21 +4301,15 @@ void CodeGenModule::emitMultiVersionFunctions() { continue; llvm::Function *Func = createFunction(CurFD, I); - StringRef Architecture; Feats.clear(); - if (getTarget().getTriple().isAArch64()) - TC->getFeatures(Feats, I); - else if (getTarget().getTriple().isRISCV()) { - StringRef Version = TC->getFeatureStr(I); - Feats.push_back(Version); + if (getTarget().getTriple().isX86()) { + TC->getX86Feature(Feats, I); + Options.emplace_back(Func, Feats, TC->getX86Architecture(I)); } else { - StringRef Version = TC->getFeatureStr(I); - if (Version.starts_with("arch=")) - Architecture = Version.drop_front(sizeof("arch=") - 1); - else if (Version != "default") - Feats.push_back(Version); + char Delim = getTarget().getTriple().isAArch64() ? '+' : ','; + TC->getFeatures(Feats, I, Delim); + Options.emplace_back(Func, Feats); } - Options.emplace_back(Func, Architecture, Feats); } } else llvm_unreachable("unexpected MultiVersionKind"); @@ -4368,9 +4348,9 @@ void CodeGenModule::emitMultiVersionFunctions() { const TargetInfo &TI = getTarget(); llvm::stable_sort( - Options, [&TI](const CodeGenFunction::MultiVersionResolverOption &LHS, - const CodeGenFunction::MultiVersionResolverOption &RHS) { - return TargetMVPriority(TI, LHS) > TargetMVPriority(TI, RHS); + Options, [&TI](const CodeGenFunction::FMVResolverOption &LHS, + const CodeGenFunction::FMVResolverOption &RHS) { + return getFMVPriority(TI, LHS) > getFMVPriority(TI, RHS); }); CodeGenFunction CGF(*this); CGF.EmitMultiVersionResolver(ResolverFunc, Options); @@ -4429,7 +4409,7 @@ void CodeGenModule::emitCPUDispatchDefinition(GlobalDecl GD) { ResolverFunc->setComdat( getModule().getOrInsertComdat(ResolverFunc->getName())); - SmallVector Options; + SmallVector Options; const TargetInfo &Target = getTarget(); unsigned Index = 0; for (const IdentifierInfo *II : DD->cpus()) { @@ -4463,25 +4443,23 @@ void CodeGenModule::emitCPUDispatchDefinition(GlobalDecl GD) { llvm::erase_if(Features, [&Target](StringRef Feat) { return !Target.validateCpuSupports(Feat); }); - Options.emplace_back(cast(Func), StringRef{}, Features); + Options.emplace_back(cast(Func), Features); ++Index; } - llvm::stable_sort( - Options, [](const CodeGenFunction::MultiVersionResolverOption &LHS, - const CodeGenFunction::MultiVersionResolverOption &RHS) { - return llvm::X86::getCpuSupportsMask(LHS.Conditions.Features) > - llvm::X86::getCpuSupportsMask(RHS.Conditions.Features); - }); + llvm::stable_sort(Options, [](const CodeGenFunction::FMVResolverOption &LHS, + const CodeGenFunction::FMVResolverOption &RHS) { + return llvm::X86::getCpuSupportsMask(LHS.Features) > + llvm::X86::getCpuSupportsMask(RHS.Features); + }); // If the list contains multiple 'default' versions, such as when it contains // 'pentium' and 'generic', don't emit the call to the generic one (since we // always run on at least a 'pentium'). We do this by deleting the 'least // advanced' (read, lowest mangling letter). - while (Options.size() > 1 && - llvm::all_of(llvm::X86::getCpuSupportsMask( - (Options.end() - 2)->Conditions.Features), - [](auto X) { return X == 0; })) { + while (Options.size() > 1 && llvm::all_of(llvm::X86::getCpuSupportsMask( + (Options.end() - 2)->Features), + [](auto X) { return X == 0; })) { StringRef LHSName = (Options.end() - 2)->Function->getName(); StringRef RHSName = (Options.end() - 1)->Function->getName(); if (LHSName.compare(RHSName) < 0) diff --git a/llvm/include/llvm/TargetParser/AArch64TargetParser.h b/llvm/include/llvm/TargetParser/AArch64TargetParser.h index d7b1ba511f95d..1311329821828 100644 --- a/llvm/include/llvm/TargetParser/AArch64TargetParser.h +++ b/llvm/include/llvm/TargetParser/AArch64TargetParser.h @@ -268,6 +268,9 @@ void fillValidCPUArchList(SmallVectorImpl &Values); bool isX18ReservedByDefault(const Triple &TT); +// Return the priority for a given set of FMV features. +unsigned getFMVPriority(ArrayRef Features); + // For given feature names, return a bitmask corresponding to the entries of // AArch64::CPUFeatures. The values in CPUFeatures are not bitmasks // themselves, they are sequential (0, 1, 2, 3, ...). diff --git a/llvm/lib/TargetParser/AArch64TargetParser.cpp b/llvm/lib/TargetParser/AArch64TargetParser.cpp index 45ecc4f24c2af..fe5ab0fabefa6 100644 --- a/llvm/lib/TargetParser/AArch64TargetParser.cpp +++ b/llvm/lib/TargetParser/AArch64TargetParser.cpp @@ -48,6 +48,19 @@ std::optional AArch64::ArchInfo::findBySubArch(StringRef SubA return {}; } +unsigned AArch64::getFMVPriority(ArrayRef Features) { + constexpr unsigned MaxFMVPriority = 1000; + unsigned Priority = 0; + unsigned NumFeatures = 0; + for (StringRef Feature : Features) { + if (auto Ext = parseFMVExtension(Feature)) { + Priority = std::max(Priority, Ext->Priority); + NumFeatures++; + } + } + return Priority + MaxFMVPriority * NumFeatures; +} + uint64_t AArch64::getCpuSupportsMask(ArrayRef FeatureStrs) { uint64_t FeaturesMask = 0; for (const StringRef &FeatureStr : FeatureStrs) {