diff --git a/llvm/include/llvm/Passes/CodeGenPassBuilder.h b/llvm/include/llvm/Passes/CodeGenPassBuilder.h index 80b74785473f7..a030d03d413a0 100644 --- a/llvm/include/llvm/Passes/CodeGenPassBuilder.h +++ b/llvm/include/llvm/Passes/CodeGenPassBuilder.h @@ -103,6 +103,7 @@ #include "llvm/IRPrinter/IRPrintingPasses.h" #include "llvm/MC/MCAsmInfo.h" #include "llvm/MC/MCTargetOptions.h" +#include "llvm/Passes/PassBuilder.h" #include "llvm/Support/CodeGen.h" #include "llvm/Support/Debug.h" #include "llvm/Support/Error.h" @@ -120,6 +121,7 @@ #include "llvm/Transforms/Utils/EntryExitInstrumenter.h" #include "llvm/Transforms/Utils/LowerInvoke.h" #include +#include #include #include @@ -161,8 +163,9 @@ template class CodeGenPassBuilder { public: explicit CodeGenPassBuilder(TargetMachineT &TM, const CGPassBuilderOption &Opts, - PassInstrumentationCallbacks *PIC) - : TM(TM), Opt(Opts), PIC(PIC) { + PassInstrumentationCallbacks *PIC, + PassBuilder &PB) + : TM(TM), Opt(Opts), PIC(PIC), PB(PB) { // Target could set CGPassBuilderOption::MISchedPostRA to true to achieve // substitutePass(&PostRASchedulerID, &PostMachineSchedulerID) @@ -298,6 +301,7 @@ template class CodeGenPassBuilder { TargetMachineT &TM; CGPassBuilderOption Opt; PassInstrumentationCallbacks *PIC; + PassBuilder &PB; template TMC &getTM() const { return static_cast(TM); } CodeGenOptLevel getOptLevel() const { return TM.getOptLevel(); } @@ -505,6 +509,15 @@ template class CodeGenPassBuilder { /// addMachinePasses helper to create the target-selected or overriden /// regalloc pass. void addRegAllocPass(AddMachinePass &, bool Optimized) const; + /// Read the --regalloc-npm option to add the next pass in line. + /// Returns false if no pass is left in the option. + bool addRegAllocPassFromOpt(AddMachinePass &, + StringRef MatchPassTo = StringRef{}) const; + /// Add the next pass in the cli option or the pass specified if no pass is + /// left in the option. + template + void addRegAllocPassOrOpt(AddMachinePass &, + RegAllocPassBuilderT PassBuilder) const; /// Add core register alloator passes which do the actual register assignment /// and rewriting. \returns true if any passes were added. @@ -601,6 +614,11 @@ Error CodeGenPassBuilder::buildPipeline( if (PrintMIR) addPass(PrintMIRPass(Out), /*Force=*/true); + if (!Opt.RegAllocPipeline.empty()) + return make_error( + "extra passes in regalloc pipeline: " + Opt.RegAllocPipeline, + std::make_error_code(std::errc::invalid_argument)); + return verifyStartStop(*StartStopInfo); } @@ -1098,6 +1116,48 @@ void CodeGenPassBuilder::addTargetRegisterAllocator( addPass(RegAllocFastPass()); } +template +template +void CodeGenPassBuilder::addRegAllocPassOrOpt( + AddMachinePass &addPass, RegAllocPassBuilderT PassBuilder) const { + if (!addRegAllocPassFromOpt(addPass)) + addPass(std::move(PassBuilder())); +} + +template +bool CodeGenPassBuilder::addRegAllocPassFromOpt( + AddMachinePass &addPass, StringRef MatchPassTo) const { + if (!Opt.RegAllocPipeline.empty()) { + StringRef PassOpt; + std::tie(PassOpt, Opt.RegAllocPipeline) = Opt.RegAllocPipeline.split(','); + // Reuse the registered parser to parse the pass name. +#define RA_PASS_WITH_PARAMS(NAME, CLASS, CREATE_PASS, PARSER, PARAMS) \ + if (PB.checkParametrizedPassName(PassOpt, NAME)) { \ + auto Params = PB.parsePassParameters(PARSER, PassOpt, NAME, \ + const_cast(PB)); \ + if (!Params) { \ + auto Err = Params.takeError(); \ + ExitOnError()(std::move(Err)); \ + } \ + if (!MatchPassTo.empty()) { \ + if (MatchPassTo != CLASS) \ + report_fatal_error("expected " + \ + PIC->getPassNameForClassName(MatchPassTo) + \ + " in option --regalloc-npm", \ + false); \ + } \ + addPass(CREATE_PASS(Params.get())); \ + return true; \ + } +#include "llvm/Passes/MachinePassRegistry.def" + if (PassOpt != "default") { + report_fatal_error("unknown register allocator pass: " + PassOpt, false); + } + } + // If user did not give a specific pass, use the default provided. + return false; +} + /// Find and instantiate the register allocation pass requested by this target /// at the current optimization level. Different register allocators are /// defined as separate passes because they may require different analysis. @@ -1108,22 +1168,13 @@ template void CodeGenPassBuilder::addRegAllocPass( AddMachinePass &addPass, bool Optimized) const { // Use the specified -regalloc-npm={basic|greedy|fast|pbqp} - if (Opt.RegAlloc > RegAllocType::Default) { - switch (Opt.RegAlloc) { - case RegAllocType::Fast: - addPass(RegAllocFastPass()); - break; - case RegAllocType::Greedy: - addPass(RAGreedyPass()); - break; - default: - report_fatal_error("register allocator not supported yet", false); - } - return; + StringRef RegAllocPassName; + if (!Optimized) + RegAllocPassName = RegAllocFastPass::name(); + + if (!addRegAllocPassFromOpt(addPass, RegAllocPassName)) { + derived().addTargetRegisterAllocator(addPass, Optimized); } - // -regalloc=default or unspecified, so pick based on the optimization level - // or ask the target for the regalloc pass. - derived().addTargetRegisterAllocator(addPass, Optimized); } template diff --git a/llvm/include/llvm/Passes/MachinePassRegistry.def b/llvm/include/llvm/Passes/MachinePassRegistry.def index c69573ee3ed97..255d4ee0d7238 100644 --- a/llvm/include/llvm/Passes/MachinePassRegistry.def +++ b/llvm/include/llvm/Passes/MachinePassRegistry.def @@ -222,8 +222,8 @@ MACHINE_FUNCTION_PASS_WITH_PARAMS( MACHINE_FUNCTION_PASS_WITH_PARAMS( "branch-folder", "BranchFolderPass", [](bool EnableTailMerge) { return BranchFolderPass(EnableTailMerge); }, - [](StringRef Params) { - return parseSinglePassOption(Params, "enable-tail-merge", + [](StringRef Params, const PassBuilder &) { + return PassBuilder::parseSinglePassOption(Params, "enable-tail-merge", "BranchFolderPass"); }, "enable-tail-merge") @@ -233,8 +233,8 @@ MACHINE_FUNCTION_PASS_WITH_PARAMS( [](bool ShouldEmitDebugEntryValues) { return LiveDebugValuesPass(ShouldEmitDebugEntryValues); }, - [](StringRef Params) { - return parseSinglePassOption(Params, "emit-debug-entry-values", + [](StringRef Params, const PassBuilder &) { + return PassBuilder::parseSinglePassOption(Params, "emit-debug-entry-values", "LiveDebugValuesPass"); }, "emit-debug-entry-values") @@ -247,27 +247,36 @@ MACHINE_FUNCTION_PASS_WITH_PARAMS( parseMachineSinkingPassOptions, "enable-sink-fold") MACHINE_FUNCTION_PASS_WITH_PARAMS( + "virt-reg-rewriter", "VirtRegRewriterPass", + [](bool ClearVirtRegs) { return VirtRegRewriterPass(ClearVirtRegs); }, + parseVirtRegRewriterPassOptions, "no-clear-vregs;clear-vregs") + +#ifndef RA_PASS_WITH_PARAMS +// Define MachineFunction passes that are register allocators. +// This is to differentiate them from other MachineFunction passes +// to be used in the --regalloc-npm option. +#define RA_PASS_WITH_PARAMS(NAME, CLASS, CREATE_PASS, PARSER, PARAMS) \ + MACHINE_FUNCTION_PASS_WITH_PARAMS(NAME, CLASS, CREATE_PASS, PARSER, PARAMS) +#endif + +RA_PASS_WITH_PARAMS( "regallocfast", "RegAllocFastPass", [](RegAllocFastPass::Options Opts) { return RegAllocFastPass(Opts); }, - [PB = this](StringRef Params) { - return parseRegAllocFastPassOptions(*PB, Params); + [](StringRef Params, const PassBuilder &PB) { + return parseRegAllocFastPassOptions(PB, Params); }, "filter=reg-filter;no-clear-vregs") // 'all' is the default filter. -MACHINE_FUNCTION_PASS_WITH_PARAMS( +RA_PASS_WITH_PARAMS( "greedy", "RAGreedyPass", [](RAGreedyPass::Options Opts) { return RAGreedyPass(Opts); }, - [PB = this](StringRef Params) { - return parseRegAllocGreedyFilterFunc(*PB, Params); - }, "reg-filter" -) - -MACHINE_FUNCTION_PASS_WITH_PARAMS( - "virt-reg-rewriter", "VirtRegRewriterPass", - [](bool ClearVirtRegs) { return VirtRegRewriterPass(ClearVirtRegs); }, - parseVirtRegRewriterPassOptions, "no-clear-vregs;clear-vregs") + [](StringRef Params, const PassBuilder &PB) { + return parseRegAllocGreedyFilterFunc(PB, Params); + }, + "reg-filter") +#undef RA_PASS_WITH_PARAMS #undef MACHINE_FUNCTION_PASS_WITH_PARAMS // After a pass is converted to new pass manager, its entry should be moved from diff --git a/llvm/include/llvm/Passes/PassBuilder.h b/llvm/include/llvm/Passes/PassBuilder.h index 51ccaa53447d7..6bf505efe23f5 100644 --- a/llvm/include/llvm/Passes/PassBuilder.h +++ b/llvm/include/llvm/Passes/PassBuilder.h @@ -18,6 +18,7 @@ #include "llvm/Analysis/CGSCCPassManager.h" #include "llvm/CodeGen/MachinePassManager.h" #include "llvm/CodeGen/RegAllocCommon.h" +#include "llvm/CodeGen/RegAllocGreedyPass.h" #include "llvm/IR/PassManager.h" #include "llvm/Passes/OptimizationLevel.h" #include "llvm/Support/Error.h" @@ -397,7 +398,7 @@ class PassBuilder { /// Parse RegAllocFilterName to get RegAllocFilterFunc. std::optional - parseRegAllocFilter(StringRef RegAllocFilterName); + parseRegAllocFilter(StringRef RegAllocFilterName) const; /// Print pass names. void printPassNames(raw_ostream &OS); @@ -688,11 +689,13 @@ class PassBuilder { /// parameter list in a form of a custom parameters type, all wrapped into /// Expected<> template class. /// - template + template static auto parsePassParameters(ParametersParseCallableT &&Parser, - StringRef Name, StringRef PassName) - -> decltype(Parser(StringRef{})) { - using ParametersT = typename decltype(Parser(StringRef{}))::value_type; + StringRef Name, StringRef PassName, + ExtraArgs &&...Args) + -> decltype(Parser(StringRef{}, std::forward(Args)...)) { + using ParametersT = typename decltype(Parser( + StringRef{}, std::forward(Args)...))::value_type; StringRef Params = Name; if (!Params.consume_front(PassName)) { @@ -704,7 +707,8 @@ class PassBuilder { llvm_unreachable("invalid format for parametrized pass name"); } - Expected Result = Parser(Params); + Expected Result = + Parser(Params, std::forward(Args)...); assert((Result || Result.template errorIsA()) && "Pass parameter parser can only return StringErrors."); return Result; @@ -980,6 +984,18 @@ class NoOpLoopAnalysis : public AnalysisInfoMixin { /// Common option used by multiple tools to print pipeline passes extern cl::opt PrintPipelinePasses; +Expected +parseRegAllocGreedyFilterFunc(const PassBuilder &PB, StringRef Params); + +Expected +parseRegAllocFastPassOptions(const PassBuilder &PB, StringRef Params); + +Expected parseMachineSinkingPassOptions(StringRef Params, + const PassBuilder &); +Expected parseMachineBlockPlacementPassOptions(StringRef Params, + const PassBuilder &); +Expected parseVirtRegRewriterPassOptions(StringRef Params, + const PassBuilder &); } #endif diff --git a/llvm/include/llvm/Passes/TargetPassRegistry.inc b/llvm/include/llvm/Passes/TargetPassRegistry.inc index 521913cb25a4a..da9fa7242ab80 100644 --- a/llvm/include/llvm/Passes/TargetPassRegistry.inc +++ b/llvm/include/llvm/Passes/TargetPassRegistry.inc @@ -83,7 +83,7 @@ if (PIC) { #define ADD_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \ if (PassBuilder::checkParametrizedPassName(Name, NAME)) { \ - auto Params = PassBuilder::parsePassParameters(PARSER, Name, NAME); \ + auto Params = PassBuilder::parsePassParameters(PARSER, Name, NAME, PB); \ if (!Params) { \ errs() << NAME ": " << toString(Params.takeError()) << '\n'; \ return false; \ diff --git a/llvm/include/llvm/Target/CGPassBuilderOption.h b/llvm/include/llvm/Target/CGPassBuilderOption.h index 51f25c1360b87..8d60decacb018 100644 --- a/llvm/include/llvm/Target/CGPassBuilderOption.h +++ b/llvm/include/llvm/Target/CGPassBuilderOption.h @@ -21,22 +21,6 @@ namespace llvm { enum class RunOutliner { TargetDefault, AlwaysOutline, NeverOutline }; -enum class RegAllocType { Unset, Default, Basic, Fast, Greedy, PBQP }; - -class RegAllocTypeParser : public cl::parser { -public: - RegAllocTypeParser(cl::Option &O) : cl::parser(O) {} - void initialize() { - cl::parser::initialize(); - addLiteralOption("default", RegAllocType::Default, - "Default register allocator"); - addLiteralOption("pbqp", RegAllocType::PBQP, "PBQP register allocator"); - addLiteralOption("fast", RegAllocType::Fast, "Fast register allocator"); - addLiteralOption("basic", RegAllocType::Basic, "Basic register allocator"); - addLiteralOption("greedy", RegAllocType::Greedy, - "Greedy register allocator"); - } -}; // Not one-on-one but mostly corresponding to commandline options in // TargetPassConfig.cpp. @@ -70,7 +54,7 @@ struct CGPassBuilderOption { bool RequiresCodeGenSCCOrder = false; RunOutliner EnableMachineOutliner = RunOutliner::TargetDefault; - RegAllocType RegAlloc = RegAllocType::Unset; + mutable StringRef RegAllocPipeline; std::optional EnableGlobalISelAbort; std::string FSProfileFile; std::string FSRemappingFile; diff --git a/llvm/include/llvm/Target/TargetMachine.h b/llvm/include/llvm/Target/TargetMachine.h index 566e7dba6792b..cce6c19acac18 100644 --- a/llvm/include/llvm/Target/TargetMachine.h +++ b/llvm/include/llvm/Target/TargetMachine.h @@ -472,7 +472,8 @@ class TargetMachine { virtual Error buildCodeGenPipeline(ModulePassManager &, raw_pwrite_stream &, raw_pwrite_stream *, CodeGenFileType, const CGPassBuilderOption &, - PassInstrumentationCallbacks *) { + PassInstrumentationCallbacks *, + PassBuilder &) { return make_error("buildCodeGenPipeline is not overridden", inconvertibleErrorCode()); } diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp index 7740f622ede7c..57b5ee1018cd0 100644 --- a/llvm/lib/Passes/PassBuilder.cpp +++ b/llvm/lib/Passes/PassBuilder.cpp @@ -1394,39 +1394,6 @@ Expected> parseInternalizeGVs(StringRef Params) { return Expected>(std::move(PreservedGVs)); } -Expected -parseRegAllocFastPassOptions(PassBuilder &PB, StringRef Params) { - RegAllocFastPass::Options Opts; - while (!Params.empty()) { - StringRef ParamName; - std::tie(ParamName, Params) = Params.split(';'); - - if (ParamName.consume_front("filter=")) { - std::optional Filter = - PB.parseRegAllocFilter(ParamName); - if (!Filter) { - return make_error( - formatv("invalid regallocfast register filter '{0}' ", ParamName) - .str(), - inconvertibleErrorCode()); - } - Opts.Filter = *Filter; - Opts.FilterName = ParamName; - continue; - } - - if (ParamName == "no-clear-vregs") { - Opts.ClearVRegs = false; - continue; - } - - return make_error( - formatv("invalid regallocfast pass parameter '{0}' ", ParamName).str(), - inconvertibleErrorCode()); - } - return Opts; -} - Expected parseBoundsCheckingOptions(StringRef Params) { BoundsCheckingPass::Options Options; @@ -1475,26 +1442,17 @@ parseBoundsCheckingOptions(StringRef Params) { return Options; } -Expected -parseRegAllocGreedyFilterFunc(PassBuilder &PB, StringRef Params) { - if (Params.empty() || Params == "all") - return RAGreedyPass::Options(); - - std::optional Filter = PB.parseRegAllocFilter(Params); - if (Filter) - return RAGreedyPass::Options{*Filter, Params}; - - return make_error( - formatv("invalid regallocgreedy register filter '{0}' ", Params).str(), - inconvertibleErrorCode()); -} +} // namespace -Expected parseMachineSinkingPassOptions(StringRef Params) { +Expected llvm::parseMachineSinkingPassOptions(StringRef Params, + const PassBuilder &) { return PassBuilder::parseSinglePassOption(Params, "enable-sink-fold", "MachineSinkingPass"); } -Expected parseMachineBlockPlacementPassOptions(StringRef Params) { +Expected +llvm::parseMachineBlockPlacementPassOptions(StringRef Params, + const PassBuilder &) { bool AllowTailMerge = true; if (!Params.empty()) { AllowTailMerge = !Params.consume_front("no-"); @@ -1507,7 +1465,8 @@ Expected parseMachineBlockPlacementPassOptions(StringRef Params) { return AllowTailMerge; } -Expected parseVirtRegRewriterPassOptions(StringRef Params) { +Expected llvm::parseVirtRegRewriterPassOptions(StringRef Params, + const PassBuilder &) { bool ClearVirtRegs = true; if (!Params.empty()) { ClearVirtRegs = !Params.consume_front("no-"); @@ -1520,8 +1479,52 @@ Expected parseVirtRegRewriterPassOptions(StringRef Params) { return ClearVirtRegs; } -} // namespace +Expected +llvm::parseRegAllocFastPassOptions(const PassBuilder &PB, StringRef Params) { + RegAllocFastPass::Options Opts; + while (!Params.empty()) { + StringRef ParamName; + std::tie(ParamName, Params) = Params.split(';'); + if (ParamName.consume_front("filter=")) { + std::optional Filter = + PB.parseRegAllocFilter(ParamName); + if (!Filter) { + return make_error( + formatv("invalid regallocfast register filter '{0}' ", ParamName) + .str(), + inconvertibleErrorCode()); + } + Opts.Filter = *Filter; + Opts.FilterName = ParamName; + continue; + } + + if (ParamName == "no-clear-vregs") { + Opts.ClearVRegs = false; + continue; + } + + return make_error( + formatv("invalid regallocfast pass parameter '{0}' ", ParamName).str(), + inconvertibleErrorCode()); + } + return Opts; +} + +Expected +llvm::parseRegAllocGreedyFilterFunc(const PassBuilder &PB, StringRef Params) { + if (Params.empty() || Params == "all") + return RAGreedyPass::Options(); + + std::optional Filter = PB.parseRegAllocFilter(Params); + if (Filter) + return RAGreedyPass::Options{*Filter, Params}; + + return make_error( + formatv("invalid regallocgreedy register filter '{0}' ", Params).str(), + inconvertibleErrorCode()); +} /// Tests whether a pass name starts with a valid prefix for a default pipeline /// alias. static bool startsWithDefaultPipelineAliasPrefix(StringRef Name) { @@ -2240,7 +2243,8 @@ Error PassBuilder::parseMachinePass(MachineFunctionPassManager &MFPM, #define MACHINE_FUNCTION_PASS_WITH_PARAMS(NAME, CLASS, CREATE_PASS, PARSER, \ PARAMS) \ if (checkParametrizedPassName(Name, NAME)) { \ - auto Params = parsePassParameters(PARSER, Name, NAME); \ + auto Params = parsePassParameters(PARSER, Name, NAME, \ + const_cast(*this)); \ if (!Params) \ return Params.takeError(); \ MFPM.addPass(CREATE_PASS(Params.get())); \ @@ -2506,7 +2510,7 @@ Error PassBuilder::parseAAPipeline(AAManager &AA, StringRef PipelineText) { } std::optional -PassBuilder::parseRegAllocFilter(StringRef FilterName) { +PassBuilder::parseRegAllocFilter(StringRef FilterName) const { if (FilterName == "all") return nullptr; for (auto &C : RegClassFilterParsingCallbacks) diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp index 1a5f415f906e6..e8dc939b1ef7c 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp @@ -773,7 +773,7 @@ void AMDGPUTargetMachine::registerDefaultAliasAnalyses(AAManager &AAM) { } static Expected -parseAMDGPUAtomicOptimizerStrategy(StringRef Params) { +parseAMDGPUAtomicOptimizerStrategy(StringRef Params, const PassBuilder &) { if (Params.empty()) return ScanOptions::Iterative; Params.consume_front("strategy="); @@ -787,8 +787,8 @@ parseAMDGPUAtomicOptimizerStrategy(StringRef Params) { return make_error("invalid parameter", inconvertibleErrorCode()); } -Expected -parseAMDGPUAttributorPassOptions(StringRef Params) { +static Expected +parseAMDGPUAttributorPassOptions(StringRef Params, const PassBuilder &PB) { AMDGPUAttributorOptions Result; while (!Params.empty()) { StringRef ParamName; @@ -1076,8 +1076,8 @@ GCNTargetMachine::getTargetTransformInfo(const Function &F) const { Error GCNTargetMachine::buildCodeGenPipeline( ModulePassManager &MPM, raw_pwrite_stream &Out, raw_pwrite_stream *DwoOut, CodeGenFileType FileType, const CGPassBuilderOption &Opts, - PassInstrumentationCallbacks *PIC) { - AMDGPUCodeGenPassBuilder CGPB(*this, Opts, PIC); + PassInstrumentationCallbacks *PIC, PassBuilder &PB) { + AMDGPUCodeGenPassBuilder CGPB(*this, Opts, PIC, PB); return CGPB.buildPipeline(MPM, Out, DwoOut, FileType); } @@ -1963,8 +1963,8 @@ bool GCNTargetMachine::parseMachineFunctionInfo( AMDGPUCodeGenPassBuilder::AMDGPUCodeGenPassBuilder( GCNTargetMachine &TM, const CGPassBuilderOption &Opts, - PassInstrumentationCallbacks *PIC) - : CodeGenPassBuilder(TM, Opts, PIC) { + PassInstrumentationCallbacks *PIC, PassBuilder &PB) + : CodeGenPassBuilder(TM, Opts, PIC, PB) { Opt.MISchedPostRA = true; Opt.RequiresCodeGenSCCOrder = true; // Exceptions and StackMaps are not supported, so these passes will never do @@ -2168,7 +2168,8 @@ Error AMDGPUCodeGenPassBuilder::addRegAssignmentOptimized( addPass(GCNPreRALongBranchRegPass()); - addPass(RAGreedyPass({onlyAllocateSGPRs, "sgpr"})); + addRegAllocPassOrOpt( + addPass, []() { return RAGreedyPass({onlyAllocateSGPRs, "sgpr"}); }); // Commit allocated register changes. This is mostly necessary because too // many things rely on the use lists of the physical registers, such as the @@ -2188,21 +2189,20 @@ Error AMDGPUCodeGenPassBuilder::addRegAssignmentOptimized( addPass(SIPreAllocateWWMRegsPass()); // For allocating other wwm register operands. - // addRegAlloc(addPass, RegAllocPhase::WWM); - addPass(RAGreedyPass({onlyAllocateWWMRegs, "wwm"})); + addRegAllocPassOrOpt( + addPass, []() { return RAGreedyPass({onlyAllocateWWMRegs, "wwm"}); }); addPass(SILowerWWMCopiesPass()); addPass(VirtRegRewriterPass(false)); addPass(AMDGPUReserveWWMRegsPass()); // For allocating per-thread VGPRs. - // addRegAlloc(addPass, RegAllocPhase::VGPR); - addPass(RAGreedyPass({onlyAllocateVGPRs, "vgpr"})); - + addRegAllocPassOrOpt( + addPass, []() { return RAGreedyPass({onlyAllocateVGPRs, "vgpr"}); }); addPreRewrite(addPass); addPass(VirtRegRewriterPass(true)); - // TODO: addPass(AMDGPUMarkLastScratchLoadPass()); + addPass(AMDGPUMarkLastScratchLoadPass()); return Error::success(); } diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h index 589123274d0f5..5a0d6b8d04beb 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h +++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h @@ -102,7 +102,8 @@ class GCNTargetMachine final : public AMDGPUTargetMachine { raw_pwrite_stream *DwoOut, CodeGenFileType FileType, const CGPassBuilderOption &Opts, - PassInstrumentationCallbacks *PIC) override; + PassInstrumentationCallbacks *PIC, + PassBuilder &) override; void registerMachineRegisterInfoCallback(MachineFunction &MF) const override; @@ -169,7 +170,7 @@ class AMDGPUCodeGenPassBuilder public: AMDGPUCodeGenPassBuilder(GCNTargetMachine &TM, const CGPassBuilderOption &Opts, - PassInstrumentationCallbacks *PIC); + PassInstrumentationCallbacks *PIC, PassBuilder &PB); void addIRPasses(AddIRPass &) const; void addCodeGenPrepare(AddIRPass &) const; diff --git a/llvm/lib/Target/AMDGPU/R600TargetMachine.cpp b/llvm/lib/Target/AMDGPU/R600TargetMachine.cpp index 2a3b42e9453bd..1340a8dcca217 100644 --- a/llvm/lib/Target/AMDGPU/R600TargetMachine.cpp +++ b/llvm/lib/Target/AMDGPU/R600TargetMachine.cpp @@ -150,8 +150,8 @@ TargetPassConfig *R600TargetMachine::createPassConfig(PassManagerBase &PM) { Error R600TargetMachine::buildCodeGenPipeline( ModulePassManager &MPM, raw_pwrite_stream &Out, raw_pwrite_stream *DwoOut, CodeGenFileType FileType, const CGPassBuilderOption &Opts, - PassInstrumentationCallbacks *PIC) { - R600CodeGenPassBuilder CGPB(*this, Opts, PIC); + PassInstrumentationCallbacks *PIC, PassBuilder &PB) { + R600CodeGenPassBuilder CGPB(*this, Opts, PIC, PB); return CGPB.buildPipeline(MPM, Out, DwoOut, FileType); } @@ -168,8 +168,8 @@ MachineFunctionInfo *R600TargetMachine::createMachineFunctionInfo( R600CodeGenPassBuilder::R600CodeGenPassBuilder( R600TargetMachine &TM, const CGPassBuilderOption &Opts, - PassInstrumentationCallbacks *PIC) - : CodeGenPassBuilder(TM, Opts, PIC) { + PassInstrumentationCallbacks *PIC, PassBuilder &PB) + : CodeGenPassBuilder(TM, Opts, PIC, PB) { Opt.RequiresCodeGenSCCOrder = true; } diff --git a/llvm/lib/Target/AMDGPU/R600TargetMachine.h b/llvm/lib/Target/AMDGPU/R600TargetMachine.h index eb4cb91cb704d..421b48a200f6e 100644 --- a/llvm/lib/Target/AMDGPU/R600TargetMachine.h +++ b/llvm/lib/Target/AMDGPU/R600TargetMachine.h @@ -42,7 +42,8 @@ class R600TargetMachine final : public AMDGPUTargetMachine { raw_pwrite_stream *DwoOut, CodeGenFileType FileType, const CGPassBuilderOption &Opt, - PassInstrumentationCallbacks *PIC) override; + PassInstrumentationCallbacks *PIC, + PassBuilder &) override; const TargetSubtargetInfo *getSubtargetImpl(const Function &) const override; @@ -65,7 +66,7 @@ class R600CodeGenPassBuilder : public CodeGenPassBuilder { public: R600CodeGenPassBuilder(R600TargetMachine &TM, const CGPassBuilderOption &Opts, - PassInstrumentationCallbacks *PIC); + PassInstrumentationCallbacks *PIC, PassBuilder &PB); void addPreISel(AddIRPass &addPass) const; void addAsmPrinter(AddMachinePass &, CreateMCStreamer) const; diff --git a/llvm/lib/Target/BPF/BPFTargetMachine.cpp b/llvm/lib/Target/BPF/BPFTargetMachine.cpp index 873719e6983ae..efe24574df41f 100644 --- a/llvm/lib/Target/BPF/BPFTargetMachine.cpp +++ b/llvm/lib/Target/BPF/BPFTargetMachine.cpp @@ -110,7 +110,8 @@ TargetPassConfig *BPFTargetMachine::createPassConfig(PassManagerBase &PM) { return new BPFPassConfig(*this, PM); } -static Expected parseBPFPreserveStaticOffsetOptions(StringRef Params) { +static Expected parseBPFPreserveStaticOffsetOptions(StringRef Params, + const PassBuilder &) { return PassBuilder::parseSinglePassOption(Params, "allow-partial", "BPFPreserveStaticOffsetPass"); } diff --git a/llvm/lib/Target/X86/X86CodeGenPassBuilder.cpp b/llvm/lib/Target/X86/X86CodeGenPassBuilder.cpp index d979517e12af6..14f7dee75b6df 100644 --- a/llvm/lib/Target/X86/X86CodeGenPassBuilder.cpp +++ b/llvm/lib/Target/X86/X86CodeGenPassBuilder.cpp @@ -26,13 +26,27 @@ class X86CodeGenPassBuilder public: explicit X86CodeGenPassBuilder(X86TargetMachine &TM, const CGPassBuilderOption &Opts, - PassInstrumentationCallbacks *PIC) - : CodeGenPassBuilder(TM, Opts, PIC) {} + PassInstrumentationCallbacks *PIC, + PassBuilder &PB) + : CodeGenPassBuilder(TM, Opts, PIC, PB) {} + using Base = CodeGenPassBuilder; void addPreISel(AddIRPass &addPass) const; void addAsmPrinter(AddMachinePass &, CreateMCStreamer) const; Error addInstSelector(AddMachinePass &) const; + Error addRegAssignmentOptimized(AddMachinePass &addPass) const; }; +Error X86CodeGenPassBuilder::addRegAssignmentOptimized( + AddMachinePass &addPass) const { + if (EnableTileRAPass) { + addRegAllocPassOrOpt(addPass, []() { + return RAGreedyPass({onlyAllocateTileRegisters, "tile-reg"}); + }); + // TODO: addPass(X86TileConfigPass()); + } + return Base::addRegAssignmentOptimized(addPass); +} + void X86CodeGenPassBuilder::addPreISel(AddIRPass &addPass) const { // TODO: Add passes pre instruction selection. } @@ -53,12 +67,20 @@ Error X86CodeGenPassBuilder::addInstSelector(AddMachinePass &addPass) const { void X86TargetMachine::registerPassBuilderCallbacks(PassBuilder &PB) { #define GET_PASS_REGISTRY "X86PassRegistry.def" #include "llvm/Passes/TargetPassRegistry.inc" + + PB.registerRegClassFilterParsingCallback( + [](StringRef FilterName) -> RegAllocFilterFunc { + if (FilterName == "tile-reg") { + return onlyAllocateTileRegisters; + } + return nullptr; + }); } Error X86TargetMachine::buildCodeGenPipeline( ModulePassManager &MPM, raw_pwrite_stream &Out, raw_pwrite_stream *DwoOut, CodeGenFileType FileType, const CGPassBuilderOption &Opt, - PassInstrumentationCallbacks *PIC) { - auto CGPB = X86CodeGenPassBuilder(*this, Opt, PIC); + PassInstrumentationCallbacks *PIC, PassBuilder &PB) { + auto CGPB = X86CodeGenPassBuilder(*this, Opt, PIC, PB); return CGPB.buildPipeline(MPM, Out, DwoOut, FileType); } diff --git a/llvm/lib/Target/X86/X86TargetMachine.cpp b/llvm/lib/Target/X86/X86TargetMachine.cpp index 975b94c18fb7f..f0c88336f8c4b 100644 --- a/llvm/lib/Target/X86/X86TargetMachine.cpp +++ b/llvm/lib/Target/X86/X86TargetMachine.cpp @@ -57,11 +57,10 @@ using namespace llvm; static cl::opt EnableMachineCombinerPass("x86-machine-combiner", cl::desc("Enable the machine combiner pass"), cl::init(true), cl::Hidden); - -static cl::opt - EnableTileRAPass("x86-tile-ra", - cl::desc("Enable the tile register allocation pass"), - cl::init(true), cl::Hidden); +cl::opt + llvm::EnableTileRAPass("x86-tile-ra", + cl::desc("Enable the tile register allocation pass"), + cl::init(true), cl::Hidden); extern "C" LLVM_C_ABI void LLVMInitializeX86Target() { // Register the target. @@ -680,9 +679,9 @@ std::unique_ptr X86PassConfig::getCSEConfig() const { return getStandardCSEConfigForOpt(TM->getOptLevel()); } -static bool onlyAllocateTileRegisters(const TargetRegisterInfo &TRI, - const MachineRegisterInfo &MRI, - const Register Reg) { +bool llvm::onlyAllocateTileRegisters(const TargetRegisterInfo &TRI, + const MachineRegisterInfo &MRI, + const Register Reg) { const TargetRegisterClass *RC = MRI.getRegClass(Reg); return static_cast(TRI).isTileRegisterClass(RC); } diff --git a/llvm/lib/Target/X86/X86TargetMachine.h b/llvm/lib/Target/X86/X86TargetMachine.h index ced0a9c71fdd8..71653c85a3630 100644 --- a/llvm/lib/Target/X86/X86TargetMachine.h +++ b/llvm/lib/Target/X86/X86TargetMachine.h @@ -22,9 +22,15 @@ namespace llvm { +extern cl::opt EnableTileRAPass; + class StringRef; class TargetTransformInfo; +bool onlyAllocateTileRegisters(const TargetRegisterInfo &TRI, + const MachineRegisterInfo &MRI, + const Register Reg); + class X86TargetMachine final : public CodeGenTargetMachineImpl { std::unique_ptr TLOF; mutable StringMap> SubtargetMap; @@ -74,7 +80,8 @@ class X86TargetMachine final : public CodeGenTargetMachineImpl { Error buildCodeGenPipeline(ModulePassManager &, raw_pwrite_stream &, raw_pwrite_stream *, CodeGenFileType, const CGPassBuilderOption &, - PassInstrumentationCallbacks *) override; + PassInstrumentationCallbacks *, + PassBuilder &) override; bool isJIT() const { return IsJIT; } diff --git a/llvm/test/tools/llc/new-pm/regalloc-amdgpu.mir b/llvm/test/tools/llc/new-pm/regalloc-amdgpu.mir index 07f2d350ffd9c..7c8b9a9e04dc1 100644 --- a/llvm/test/tools/llc/new-pm/regalloc-amdgpu.mir +++ b/llvm/test/tools/llc/new-pm/regalloc-amdgpu.mir @@ -2,11 +2,16 @@ # RUN: llc -mtriple=amdgcn --passes='regallocfast,regallocfast,regallocfast' --print-pipeline-passes --filetype=null %s | FileCheck %s --check-prefix=PASS # RUN: not llc -mtriple=amdgcn --passes='regallocfast' --print-pipeline-passes --filetype=null %s 2>&1 | FileCheck %s --check-prefix=BAD-FILTER +# RUN: llc -mtriple=amdgcn -enable-new-pm --regalloc-npm="regallocfast,greedy" --print-pipeline-passes --filetype=null %s | FileCheck %s --check-prefix=RA-NPM + # PASS: regallocfast # PASS: regallocfast # PASS: regallocfast # BAD-FILTER: invalid regallocfast register filter 'bad-filter' +# RA-NPM: regallocfast +# RA-NPM: greedy +# RA-NPM: greedy --- name: f ... diff --git a/llvm/test/tools/llc/new-pm/x86_64-regalloc-pipeline.mir b/llvm/test/tools/llc/new-pm/x86_64-regalloc-pipeline.mir index f1a30bc7964ca..356224d1187d3 100644 --- a/llvm/test/tools/llc/new-pm/x86_64-regalloc-pipeline.mir +++ b/llvm/test/tools/llc/new-pm/x86_64-regalloc-pipeline.mir @@ -1,6 +1,28 @@ # REQUIRES x86_64-registered-target -# RUN: llc -mtriple=x86_64-unknown-linux-gnu -enable-new-pm -O3 -regalloc-npm=fast -print-pipeline-passes %s -o - 2>&1 | FileCheck %s -# RUN: llc -mtriple=x86_64-unknown-linux-gnu -enable-new-pm -O3 -regalloc-npm=greedy -print-pipeline-passes %s -o - 2>&1 | FileCheck %s --check-prefix=CHECK-GREEDY +# Test default pipeline +# RUN: llc -mtriple=x86_64-unknown-linux-gnu -enable-new-pm -O3 -print-pipeline-passes %s -o - 2>&1 | FileCheck %s -# CHECK: regallocfast +# Test the -regalloc-npm option +# RUN: llc -mtriple=x86_64-unknown-linux-gnu -enable-new-pm -O3 -regalloc-npm=regallocfast -print-pipeline-passes %s -o - 2>&1 | FileCheck %s --check-prefix=CHECK-FAST + +# RUN: llc -mtriple=x86_64-unknown-linux-gnu -enable-new-pm -O3 -regalloc-npm='greedy' -print-pipeline-passes %s -o - 2>&1 | FileCheck %s --check-prefix=CHECK-GREEDY-FILTER + +# This test attempts to bypass the tile register filter pass, inserts the default greedy pass and then +# attempts to insert an extraneous pass. +# RUN: not llc -mtriple=x86_64-unknown-linux-gnu -enable-new-pm -O3 -regalloc-npm='default,default,basic' 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR + +# RUN: not llc -mtriple=x86_64-unknown-linux-gnu -enable-new-pm -O3 -regalloc-npm='machine-cp' -print-pipeline-passes %s -o - 2>&1 | FileCheck %s --check-prefix=CHECK-INVALID +# RUN: not llc -mtriple=x86_64-unknown-linux-gnu -enable-new-pm -passes='machine-sink' -regalloc-npm='greedy' -print-pipeline-passes %s -o - 2>&1 | FileCheck %s --check-prefix=CHECK-BOTH-INVALID + +# RUN: not llc -mtriple=x86_64-unknown-linux-gnu -enable-new-pm -O0 -regalloc-npm='greedy' -print-pipeline-passes %s -o - 2>&1 | FileCheck %s --check-prefix=CHECK-ONLY-FAST + +# CHECK: greedy +# CHECK: greedy + +# CHECK-FAST: regallocfast # CHECK-GREEDY: greedy +# CHECK-GREEDY-FILTER: greedy +# CHECK-ERROR: extra passes in regalloc pipeline: basic +# CHECK-INVALID: unknown register allocator pass: machine-cp +# CHECK-BOTH-INVALID: --passes and --regalloc-npm cannot be used together +# CHECK-ONLY-FAST: expected regallocfast in option --regalloc-npm diff --git a/llvm/tools/llc/NewPMDriver.cpp b/llvm/tools/llc/NewPMDriver.cpp index fa82689ecf9ae..eb80f116f9672 100644 --- a/llvm/tools/llc/NewPMDriver.cpp +++ b/llvm/tools/llc/NewPMDriver.cpp @@ -48,10 +48,11 @@ using namespace llvm; -static cl::opt - RegAlloc("regalloc-npm", - cl::desc("Register allocator to use for new pass manager"), - cl::Hidden, cl::init(RegAllocType::Unset)); +static cl::opt RegAllocPasses( + "regalloc-npm", + cl::desc("Register allocator pipeline for the new pass manager. Same as " + "--passes, but only regalloc passes can be used here."), + cl::Hidden); static cl::opt DebugPM("debug-pass-manager", cl::Hidden, @@ -98,6 +99,11 @@ int llvm::compileModuleWithNewPM( << TargetPassConfig::getLimitedCodeGenPipelineReason() << ".\n"; return 1; } + if (!PassPipeline.empty() && !RegAllocPasses.empty()) { + WithColor::error(errs(), Arg0) + << "--passes and --regalloc-npm cannot be used together.\n"; + return 1; + } raw_pwrite_stream *OS = &Out->os(); @@ -105,7 +111,7 @@ int llvm::compileModuleWithNewPM( CGPassBuilderOption Opt = getCGPassBuilderOption(); Opt.DisableVerify = VK != VerifierKind::InputOutput; Opt.DebugPM = DebugPM; - Opt.RegAlloc = RegAlloc; + Opt.RegAllocPipeline = RegAllocPasses; MachineModuleInfo MMI(Target.get()); @@ -155,7 +161,7 @@ int llvm::compileModuleWithNewPM( } else { ExitOnErr(Target->buildCodeGenPipeline( - MPM, *OS, DwoOut ? &DwoOut->os() : nullptr, FileType, Opt, &PIC)); + MPM, *OS, DwoOut ? &DwoOut->os() : nullptr, FileType, Opt, &PIC, PB)); } // If user only wants to print the pipeline, print it before parsing the MIR.