diff --git a/llvm/include/llvm/Transforms/Scalar/LowerMatrixIntrinsics.h b/llvm/include/llvm/Transforms/Scalar/LowerMatrixIntrinsics.h index d44d297dd4ffc..d095298f00c0f 100644 --- a/llvm/include/llvm/Transforms/Scalar/LowerMatrixIntrinsics.h +++ b/llvm/include/llvm/Transforms/Scalar/LowerMatrixIntrinsics.h @@ -14,6 +14,7 @@ #define LLVM_TRANSFORMS_SCALAR_LOWERMATRIXINTRINSICS_H #include "llvm/IR/PassManager.h" +#include "llvm/Transforms/Utils/ExtraPassManager.h" namespace llvm { class LowerMatrixIntrinsicsPass @@ -27,6 +28,15 @@ class LowerMatrixIntrinsicsPass function_ref MapClassName2PassName); static bool isRequired() { return true; } }; + +/// A marker analysis to signal if extra passes should be run after lowering +/// matrix intrinsics. +struct ShouldRunExtraMatrixPasses + : public ShouldRunExtraPasses, + public AnalysisInfoMixin { + static AnalysisKey Key; +}; + } // namespace llvm #endif diff --git a/llvm/lib/Passes/PassBuilderPipelines.cpp b/llvm/lib/Passes/PassBuilderPipelines.cpp index d737ea5ab070a..0a30a91be73ab 100644 --- a/llvm/lib/Passes/PassBuilderPipelines.cpp +++ b/llvm/lib/Passes/PassBuilderPipelines.cpp @@ -270,10 +270,6 @@ static cl::opt EnableOrderFileInstrumentation( "enable-order-file-instrumentation", cl::init(false), cl::Hidden, cl::desc("Enable order file instrumentation (default = off)")); -static cl::opt - EnableMatrix("enable-matrix", cl::init(false), cl::Hidden, - cl::desc("Enable lowering of the matrix intrinsics")); - static cl::opt EnableConstraintElimination( "enable-constraint-elimination", cl::init(true), cl::Hidden, cl::desc( @@ -1489,10 +1485,10 @@ PassBuilder::buildModuleOptimizationPipeline(OptimizationLevel Level, OptimizePM.addPass(Float2IntPass()); OptimizePM.addPass(LowerConstantIntrinsicsPass()); - if (EnableMatrix) { - OptimizePM.addPass(LowerMatrixIntrinsicsPass()); - OptimizePM.addPass(EarlyCSEPass()); - } + OptimizePM.addPass(LowerMatrixIntrinsicsPass()); + ExtraFunctionPassManager ExtraPasses; + ExtraPasses.addPass(EarlyCSEPass()); + OptimizePM.addPass(std::move(ExtraPasses)); // CHR pass should only be applied with the profile information. // The check is to check the profile summary information in CHR. @@ -2189,9 +2185,8 @@ PassBuilder::buildO0DefaultPipeline(OptimizationLevel Level, if (PTO.MergeFunctions) MPM.addPass(MergeFunctionsPass()); - if (EnableMatrix) - MPM.addPass( - createModuleToFunctionPassAdaptor(LowerMatrixIntrinsicsPass(true))); + MPM.addPass( + createModuleToFunctionPassAdaptor(LowerMatrixIntrinsicsPass(true))); if (!CGSCCOptimizerLateEPCallbacks.empty()) { CGSCCPassManager CGPM; diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def index 2ddebb07017c2..3ce028278ff8f 100644 --- a/llvm/lib/Passes/PassRegistry.def +++ b/llvm/lib/Passes/PassRegistry.def @@ -305,6 +305,8 @@ FUNCTION_ANALYSIS("scalar-evolution", ScalarEvolutionAnalysis()) FUNCTION_ANALYSIS("should-not-run-function-passes", ShouldNotRunFunctionPassesAnalysis()) FUNCTION_ANALYSIS("should-run-extra-vector-passes", + ShouldRunExtraMatrixPasses()) +FUNCTION_ANALYSIS("should-run-extra-matrix-passes", ShouldRunExtraVectorPasses()) FUNCTION_ANALYSIS("ssp-layout", SSPLayoutAnalysis()) FUNCTION_ANALYSIS("stack-safety-local", StackSafetyAnalysis()) diff --git a/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp b/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp index 796fba67ee257..168c7016d5b0c 100644 --- a/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp +++ b/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp @@ -117,6 +117,8 @@ auto m_AnyAdd(const LTy &L, const RTy &R) { return m_CombineOr(m_Add(L, R), m_FAdd(L, R)); } +AnalysisKey ShouldRunExtraMatrixPasses::Key; + namespace { // Given an element pointer \p BasePtr to the start of a (sub) matrix, compute @@ -2651,6 +2653,8 @@ PreservedAnalyses LowerMatrixIntrinsicsPass::run(Function &F, if (!Minimal) { PA.preserve(); PA.preserve(); + AM.getResult(F); + PA.preserve(); } return PA; } diff --git a/llvm/test/Other/cse-after-matrix-lowering.ll b/llvm/test/Other/cse-after-matrix-lowering.ll new file mode 100644 index 0000000000000..1963aa8d0b435 --- /dev/null +++ b/llvm/test/Other/cse-after-matrix-lowering.ll @@ -0,0 +1,17 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt -disable-verify -verify-analysis-invalidation=0 -eagerly-invalidate-analyses=0 -debug-pass-manager \ +; RUN: -passes='default' -S %s 2>&1 | FileCheck %s + +define <4 x float> @multiply_2x2(<4 x float> %a, <4 x float> %b) { +; CHECK: Running pass: LowerMatrixIntrinsicsPass +; CHECK-NEXT: Running analysis: ShouldRunExtraMatrixPasses +; CHECK-NEXT: Invalidating analysis: PostDominatorTreeAnalysis +; CHECK-NEXT: Running pass: EarlyCSEPass +; CHECK-NEXT: Invalidating analysis: ShouldRunExtraMatrixPasses +; +entry: + %c = call <4 x float> @llvm.matrix.multiply.v4f32.v4f32.v4f32(<4 x float> %a, <4 x float> %b, i32 2, i32 2, i32 2) + ret <4 x float> %c +} + +declare <4 x float> @llvm.matrix.multiply.v4f32.v4f32.v4f32(<4 x float>, <4 x float>, i32, i32, i32) diff --git a/llvm/test/Other/new-pm-O0-defaults.ll b/llvm/test/Other/new-pm-O0-defaults.ll index e8131ac7fab45..59b91dfdc308b 100644 --- a/llvm/test/Other/new-pm-O0-defaults.ll +++ b/llvm/test/Other/new-pm-O0-defaults.ll @@ -10,9 +10,6 @@ ; RUN: opt -disable-verify -verify-analysis-invalidation=0 -debug-pass-manager \ ; RUN: -passes='default' -S %s 2>&1 \ ; RUN: | FileCheck %s --check-prefixes=CHECK,CHECK-DEFAULT,CHECK-CORO -; RUN: opt -disable-verify -verify-analysis-invalidation=0 -debug-pass-manager -enable-matrix \ -; RUN: -passes='default' -S %s 2>&1 \ -; RUN: | FileCheck %s --check-prefixes=CHECK,CHECK-DEFAULT,CHECK-MATRIX,CHECK-CORO ; RUN: opt -disable-verify -verify-analysis-invalidation=0 -debug-pass-manager -debug-info-for-profiling \ ; RUN: -passes='default' -S %s 2>&1 \ ; RUN: | FileCheck %s --check-prefixes=CHECK,CHECK-DIS,CHECK-CORO @@ -34,12 +31,14 @@ ; CHECK-DIS-NEXT: Running pass: AddDiscriminatorsPass ; CHECK-DIS-NEXT: Running pass: AlwaysInlinerPass ; CHECK-DIS-NEXT: Running analysis: ProfileSummaryAnalysis +; CHECK-DIS: Running pass: LowerMatrixIntrinsicsPass +; CHECK-DIS-NEXT: Running analysis: TargetIRAnalysis ; CHECK-DEFAULT: Running analysis: InnerAnalysisManagerProxy ; CHECK-DEFAULT-NEXT: Running pass: EntryExitInstrumenterPass ; CHECK-DEFAULT-NEXT: Running pass: AlwaysInlinerPass ; CHECK-DEFAULT-NEXT: Running analysis: ProfileSummaryAnalysis -; CHECK-MATRIX: Running pass: LowerMatrixIntrinsicsPass -; CHECK-MATRIX-NEXT: Running analysis: TargetIRAnalysis +; CHECK-DEFAULT: Running pass: LowerMatrixIntrinsicsPass +; CHECK-DEFAULT-NEXT: Running analysis: TargetIRAnalysis ; CHECK-CORO-NEXT: Running pass: CoroConditionalWrapper ; CHECK-PRE-LINK: Running pass: CanonicalizeAliasesPass ; CHECK-PRE-LINK-NEXT: Running pass: NameAnonGlobalPass diff --git a/llvm/test/Other/new-pm-defaults.ll b/llvm/test/Other/new-pm-defaults.ll index 7cf035b0c6f37..8e6390dbbcab7 100644 --- a/llvm/test/Other/new-pm-defaults.ll +++ b/llvm/test/Other/new-pm-defaults.ll @@ -75,10 +75,6 @@ ; RUN: -passes='default' -enable-jump-table-to-switch -S %s 2>&1 \ ; RUN: | FileCheck %s --check-prefixes=CHECK-O,CHECK-DEFAULT,CHECK-O3,CHECK-JUMP-TABLE-TO-SWITCH,CHECK-O23SZ,%llvmcheckext -; RUN: opt -disable-verify -verify-analysis-invalidation=0 -eagerly-invalidate-analyses=0 -debug-pass-manager \ -; RUN: -passes='default' -enable-matrix -S %s 2>&1 \ -; RUN: | FileCheck %s --check-prefixes=CHECK-O,CHECK-DEFAULT,CHECK-O3,CHECK-O23SZ,%llvmcheckext,CHECK-MATRIX - ; RUN: opt -disable-verify -verify-analysis-invalidation=0 -eagerly-invalidate-analyses=0 -debug-pass-manager \ ; RUN: -passes='default' -enable-merge-functions -S %s 2>&1 \ ; RUN: | FileCheck %s --check-prefixes=CHECK-O,CHECK-DEFAULT,CHECK-O3,CHECK-O23SZ,%llvmcheckext,CHECK-MERGE-FUNCS @@ -242,8 +238,7 @@ ; CHECK-EP-OPTIMIZER-EARLY: Running pass: NoOpModulePass ; CHECK-O-NEXT: Running pass: Float2IntPass ; CHECK-O-NEXT: Running pass: LowerConstantIntrinsicsPass on foo -; CHECK-MATRIX: Running pass: LowerMatrixIntrinsicsPass on f -; CHECK-MATRIX-NEXT: Running pass: EarlyCSEPass on f +; CHECK-O-NEXT: Running pass: LowerMatrixIntrinsicsPass ; CHECK-O3-NEXT: Running pass: ControlHeightReductionPass ; CHECK-EP-VECTORIZER-START-NEXT: Running pass: NoOpFunctionPass ; CHECK-EXT: Running pass: {{.*}}::Bye on foo diff --git a/llvm/test/Other/new-pm-thinlto-postlink-defaults.ll b/llvm/test/Other/new-pm-thinlto-postlink-defaults.ll index ed13402e1c4b1..01e5e73c9b1fe 100644 --- a/llvm/test/Other/new-pm-thinlto-postlink-defaults.ll +++ b/llvm/test/Other/new-pm-thinlto-postlink-defaults.ll @@ -168,6 +168,7 @@ ; CHECK-POST-EP-OPT-EARLY-NEXT: Running pass: NoOpModulePass ; CHECK-POSTLINK-O-NEXT: Running pass: Float2IntPass ; CHECK-POSTLINK-O-NEXT: Running pass: LowerConstantIntrinsicsPass +; CHECK-POSTLINK-O-NEXT: Running pass: LowerMatrixIntrinsicsPass ; CHECK-POSTLINK-O3-NEXT: Running pass: ControlHeightReductionPass ; CHECK-EXT: Running pass: {{.*}}::Bye ; CHECK-POSTLINK-O-NEXT: Running pass: LoopSimplifyPass diff --git a/llvm/test/Other/new-pm-thinlto-postlink-pgo-defaults.ll b/llvm/test/Other/new-pm-thinlto-postlink-pgo-defaults.ll index c82c34f7ff01e..b2ad9901bffaa 100644 --- a/llvm/test/Other/new-pm-thinlto-postlink-pgo-defaults.ll +++ b/llvm/test/Other/new-pm-thinlto-postlink-pgo-defaults.ll @@ -151,6 +151,7 @@ ; CHECK-O-NEXT: Running pass: RecomputeGlobalsAAPass ; CHECK-O-NEXT: Running pass: Float2IntPass ; CHECK-O-NEXT: Running pass: LowerConstantIntrinsicsPass +; CHECK-O-NEXT: Running pass: LowerMatrixIntrinsicsPass ; CHECK-O3-NEXT: Running pass: ControlHeightReductionPass ; CHECK-O3-NEXT: Running analysis: RegionInfoAnalysis ; CHECK-O3-NEXT: Running analysis: DominanceFrontierAnalysis diff --git a/llvm/test/Other/new-pm-thinlto-postlink-samplepgo-defaults.ll b/llvm/test/Other/new-pm-thinlto-postlink-samplepgo-defaults.ll index d375747547d61..44423e342c9f3 100644 --- a/llvm/test/Other/new-pm-thinlto-postlink-samplepgo-defaults.ll +++ b/llvm/test/Other/new-pm-thinlto-postlink-samplepgo-defaults.ll @@ -160,6 +160,7 @@ ; CHECK-O-NEXT: Running pass: RecomputeGlobalsAAPass ; CHECK-O-NEXT: Running pass: Float2IntPass ; CHECK-O-NEXT: Running pass: LowerConstantIntrinsicsPass +; CHECK-O-NEXT: Running pass: LowerMatrixIntrinsicsPass ; CHECK-O3-NEXT: Running pass: ControlHeightReductionPass ; CHECK-O3-NEXT: Running analysis: RegionInfoAnalysis ; CHECK-O3-NEXT: Running analysis: DominanceFrontierAnalysis