[Passes] Always run LowerMatrixIntrinsics, request EarlyCSE on demand.

llvm/include/llvm/Transforms/Scalar/LowerMatrixIntrinsics.h

@@ -14,6 +14,7 @@
 #define LLVM_TRANSFORMS_SCALAR_LOWERMATRIXINTRINSICS_H
 
 #include "llvm/IR/PassManager.h"
+#include "llvm/Transforms/Utils/ExtraPassManager.h"
 
 namespace llvm {
 class LowerMatrixIntrinsicsPass
@@ -27,6 +28,15 @@ class LowerMatrixIntrinsicsPass
                      function_ref<StringRef(StringRef)> MapClassName2PassName);
   static bool isRequired() { return true; }
 };
+
+/// A marker analysis to signal if extra passes should be run after lowering
+/// matrix intrinsics.
+struct ShouldRunExtraMatrixPasses
+    : public ShouldRunExtraPasses<ShouldRunExtraMatrixPasses>,
+      public AnalysisInfoMixin<ShouldRunExtraMatrixPasses> {
+  static AnalysisKey Key;
+};
+
 } // namespace llvm
 
 #endif

llvm/lib/Passes/PassBuilderPipelines.cpp

@@ -270,10 +270,6 @@ static cl::opt<bool> EnableOrderFileInstrumentation(
     "enable-order-file-instrumentation", cl::init(false), cl::Hidden,
     cl::desc("Enable order file instrumentation (default = off)"));
 
-static cl::opt<bool>
-    EnableMatrix("enable-matrix", cl::init(false), cl::Hidden,
-                 cl::desc("Enable lowering of the matrix intrinsics"));
-
 static cl::opt<bool> EnableConstraintElimination(
     "enable-constraint-elimination", cl::init(true), cl::Hidden,
     cl::desc(
@@ -1489,10 +1485,10 @@ PassBuilder::buildModuleOptimizationPipeline(OptimizationLevel Level,
   OptimizePM.addPass(Float2IntPass());
   OptimizePM.addPass(LowerConstantIntrinsicsPass());
 
-  if (EnableMatrix) {
-    OptimizePM.addPass(LowerMatrixIntrinsicsPass());
-    OptimizePM.addPass(EarlyCSEPass());
-  }
+  OptimizePM.addPass(LowerMatrixIntrinsicsPass());
+  ExtraFunctionPassManager<ShouldRunExtraMatrixPasses> ExtraPasses;
+  ExtraPasses.addPass(EarlyCSEPass());
+  OptimizePM.addPass(std::move(ExtraPasses));
 
   // CHR pass should only be applied with the profile information.
   // The check is to check the profile summary information in CHR.
@@ -2189,9 +2185,8 @@ PassBuilder::buildO0DefaultPipeline(OptimizationLevel Level,
   if (PTO.MergeFunctions)
     MPM.addPass(MergeFunctionsPass());
 
-  if (EnableMatrix)
-    MPM.addPass(
-        createModuleToFunctionPassAdaptor(LowerMatrixIntrinsicsPass(true)));
+  MPM.addPass(
+      createModuleToFunctionPassAdaptor(LowerMatrixIntrinsicsPass(true)));
 
   if (!CGSCCOptimizerLateEPCallbacks.empty()) {
     CGSCCPassManager CGPM;

llvm/lib/Passes/PassRegistry.def

@@ -305,6 +305,8 @@ FUNCTION_ANALYSIS("scalar-evolution", ScalarEvolutionAnalysis())
 FUNCTION_ANALYSIS("should-not-run-function-passes",
                   ShouldNotRunFunctionPassesAnalysis())
 FUNCTION_ANALYSIS("should-run-extra-vector-passes",
+                  ShouldRunExtraMatrixPasses())
+FUNCTION_ANALYSIS("should-run-extra-matrix-passes",
                   ShouldRunExtraVectorPasses())
 FUNCTION_ANALYSIS("ssp-layout", SSPLayoutAnalysis())
 FUNCTION_ANALYSIS("stack-safety-local", StackSafetyAnalysis())

llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp

@@ -117,6 +117,8 @@ auto m_AnyAdd(const LTy &L, const RTy &R) {
   return m_CombineOr(m_Add(L, R), m_FAdd(L, R));
 }
 
+AnalysisKey ShouldRunExtraMatrixPasses::Key;
+
 namespace {
 
 // Given an element pointer \p BasePtr to the start of a (sub) matrix, compute
@@ -2651,6 +2653,8 @@ PreservedAnalyses LowerMatrixIntrinsicsPass::run(Function &F,
     if (!Minimal) {
       PA.preserve<LoopAnalysis>();
       PA.preserve<DominatorTreeAnalysis>();
+      AM.getResult<ShouldRunExtraMatrixPasses>(F);
+      PA.preserve<ShouldRunExtraMatrixPasses>();
     }
     return PA;
   }

llvm/test/Other/cse-after-matrix-lowering.ll

@@ -0,0 +1,17 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -disable-verify -verify-analysis-invalidation=0 -eagerly-invalidate-analyses=0 -debug-pass-manager \
+; RUN:     -passes='default<O3>' -S %s 2>&1 | FileCheck %s
+
+define <4 x float> @multiply_2x2(<4 x float> %a, <4 x float> %b) {
+; CHECK:      Running pass: LowerMatrixIntrinsicsPass
+; CHECK-NEXT: Running analysis: ShouldRunExtraMatrixPasses
+; CHECK-NEXT: Invalidating analysis: PostDominatorTreeAnalysis
+; CHECK-NEXT: Running pass: EarlyCSEPass
+; CHECK-NEXT: Invalidating analysis: ShouldRunExtraMatrixPasses
+;
+entry:
+  %c = call <4 x float> @llvm.matrix.multiply.v4f32.v4f32.v4f32(<4 x float> %a, <4 x float> %b, i32 2, i32 2, i32 2)
+  ret <4 x float> %c
+}
+
+declare <4 x float> @llvm.matrix.multiply.v4f32.v4f32.v4f32(<4 x float>, <4 x float>, i32, i32, i32)

llvm/test/Other/new-pm-O0-defaults.ll

@@ -10,9 +10,6 @@
 ; RUN: opt -disable-verify -verify-analysis-invalidation=0 -debug-pass-manager \
 ; RUN:     -passes='default<O0>' -S %s 2>&1 \
 ; RUN:     | FileCheck %s --check-prefixes=CHECK,CHECK-DEFAULT,CHECK-CORO
-; RUN: opt -disable-verify -verify-analysis-invalidation=0 -debug-pass-manager -enable-matrix \
-; RUN:     -passes='default<O0>' -S %s 2>&1 \
-; RUN:     | FileCheck %s --check-prefixes=CHECK,CHECK-DEFAULT,CHECK-MATRIX,CHECK-CORO
 ; RUN: opt -disable-verify -verify-analysis-invalidation=0 -debug-pass-manager -debug-info-for-profiling \
 ; RUN:     -passes='default<O0>' -S %s 2>&1 \
 ; RUN:     | FileCheck %s --check-prefixes=CHECK,CHECK-DIS,CHECK-CORO
@@ -34,12 +31,14 @@
 ; CHECK-DIS-NEXT: Running pass: AddDiscriminatorsPass
 ; CHECK-DIS-NEXT: Running pass: AlwaysInlinerPass
 ; CHECK-DIS-NEXT: Running analysis: ProfileSummaryAnalysis
+; CHECK-DIS: Running pass: LowerMatrixIntrinsicsPass
+; CHECK-DIS-NEXT: Running analysis: TargetIRAnalysis
 ; CHECK-DEFAULT: Running analysis: InnerAnalysisManagerProxy
 ; CHECK-DEFAULT-NEXT: Running pass: EntryExitInstrumenterPass
 ; CHECK-DEFAULT-NEXT: Running pass: AlwaysInlinerPass
 ; CHECK-DEFAULT-NEXT: Running analysis: ProfileSummaryAnalysis
-; CHECK-MATRIX: Running pass: LowerMatrixIntrinsicsPass
-; CHECK-MATRIX-NEXT: Running analysis: TargetIRAnalysis
+; CHECK-DEFAULT: Running pass: LowerMatrixIntrinsicsPass
+; CHECK-DEFAULT-NEXT: Running analysis: TargetIRAnalysis
 ; CHECK-CORO-NEXT: Running pass: CoroConditionalWrapper
 ; CHECK-PRE-LINK: Running pass: CanonicalizeAliasesPass
 ; CHECK-PRE-LINK-NEXT: Running pass: NameAnonGlobalPass

llvm/test/Other/new-pm-defaults.ll

@@ -75,10 +75,6 @@
 ; RUN:     -passes='default<O3>' -enable-jump-table-to-switch -S  %s 2>&1 \
 ; RUN:     | FileCheck %s --check-prefixes=CHECK-O,CHECK-DEFAULT,CHECK-O3,CHECK-JUMP-TABLE-TO-SWITCH,CHECK-O23SZ,%llvmcheckext
 
-; RUN: opt -disable-verify -verify-analysis-invalidation=0 -eagerly-invalidate-analyses=0 -debug-pass-manager \
-; RUN:     -passes='default<O3>' -enable-matrix -S  %s 2>&1 \
-; RUN:     | FileCheck %s --check-prefixes=CHECK-O,CHECK-DEFAULT,CHECK-O3,CHECK-O23SZ,%llvmcheckext,CHECK-MATRIX
-
 ; RUN: opt -disable-verify -verify-analysis-invalidation=0 -eagerly-invalidate-analyses=0 -debug-pass-manager \
 ; RUN:     -passes='default<O3>' -enable-merge-functions -S  %s 2>&1 \
 ; RUN:     | FileCheck %s --check-prefixes=CHECK-O,CHECK-DEFAULT,CHECK-O3,CHECK-O23SZ,%llvmcheckext,CHECK-MERGE-FUNCS
@@ -242,8 +238,7 @@
 ; CHECK-EP-OPTIMIZER-EARLY: Running pass: NoOpModulePass
 ; CHECK-O-NEXT: Running pass: Float2IntPass
 ; CHECK-O-NEXT: Running pass: LowerConstantIntrinsicsPass on foo
-; CHECK-MATRIX: Running pass: LowerMatrixIntrinsicsPass on f
-; CHECK-MATRIX-NEXT: Running pass: EarlyCSEPass on f
+; CHECK-O-NEXT: Running pass: LowerMatrixIntrinsicsPass
 ; CHECK-O3-NEXT: Running pass: ControlHeightReductionPass
 ; CHECK-EP-VECTORIZER-START-NEXT: Running pass: NoOpFunctionPass
 ; CHECK-EXT: Running pass: {{.*}}::Bye on foo

llvm/test/Other/new-pm-thinlto-postlink-defaults.ll

@@ -168,6 +168,7 @@
 ; CHECK-POST-EP-OPT-EARLY-NEXT: Running pass: NoOpModulePass
 ; CHECK-POSTLINK-O-NEXT: Running pass: Float2IntPass
 ; CHECK-POSTLINK-O-NEXT: Running pass: LowerConstantIntrinsicsPass
+; CHECK-POSTLINK-O-NEXT: Running pass: LowerMatrixIntrinsicsPass
 ; CHECK-POSTLINK-O3-NEXT: Running pass: ControlHeightReductionPass
 ; CHECK-EXT: Running pass: {{.*}}::Bye
 ; CHECK-POSTLINK-O-NEXT: Running pass: LoopSimplifyPass

llvm/test/Other/new-pm-thinlto-postlink-pgo-defaults.ll

@@ -151,6 +151,7 @@
 ; CHECK-O-NEXT: Running pass: RecomputeGlobalsAAPass
 ; CHECK-O-NEXT: Running pass: Float2IntPass
 ; CHECK-O-NEXT: Running pass: LowerConstantIntrinsicsPass
+; CHECK-O-NEXT: Running pass: LowerMatrixIntrinsicsPass
 ; CHECK-O3-NEXT: Running pass: ControlHeightReductionPass
 ; CHECK-O3-NEXT: Running analysis: RegionInfoAnalysis
 ; CHECK-O3-NEXT: Running analysis: DominanceFrontierAnalysis

llvm/test/Other/new-pm-thinlto-postlink-samplepgo-defaults.ll

@@ -160,6 +160,7 @@
 ; CHECK-O-NEXT: Running pass: RecomputeGlobalsAAPass
 ; CHECK-O-NEXT: Running pass: Float2IntPass
 ; CHECK-O-NEXT: Running pass: LowerConstantIntrinsicsPass
+; CHECK-O-NEXT: Running pass: LowerMatrixIntrinsicsPass
 ; CHECK-O3-NEXT: Running pass: ControlHeightReductionPass
 ; CHECK-O3-NEXT: Running analysis: RegionInfoAnalysis
 ; CHECK-O3-NEXT: Running analysis: DominanceFrontierAnalysis