diff --git a/llvm/include/llvm/SandboxIR/Region.h b/llvm/include/llvm/SandboxIR/Region.h index 66eda34ea2edb..ecc6c2049ef8d 100644 --- a/llvm/include/llvm/SandboxIR/Region.h +++ b/llvm/include/llvm/SandboxIR/Region.h @@ -120,6 +120,10 @@ class Region { /// Set \p I as the \p Idx'th element in the auxiliary vector. /// NOTE: This is for internal use, it does not set the metadata. void setAux(unsigned Idx, Instruction *I); + /// Helper for dropping Aux metadata for \p I. + void dropAuxMetadata(Instruction *I); + /// Remove instruction \p I from Aux and drop metadata. + void removeFromAux(Instruction *I); public: Region(Context &Ctx, TargetTransformInfo &TTI); diff --git a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.h b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.h index 147a86de4e34e..daf6499213d48 100644 --- a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.h +++ b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.h @@ -16,14 +16,23 @@ #include "llvm/ADT/StringRef.h" #include "llvm/SandboxIR/Constant.h" #include "llvm/SandboxIR/Pass.h" -#include "llvm/SandboxIR/PassManager.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Transforms/Vectorize/SandboxVectorizer/InstrMaps.h" #include "llvm/Transforms/Vectorize/SandboxVectorizer/Legality.h" namespace llvm::sandboxir { -class BottomUpVec final : public FunctionPass { +/// This is a simple bottom-up vectorizer Region pass. +/// It expects a "seed slice" as an input in the Region's Aux vector. +/// The "seed slice" is a vector of instructions that can be used as a starting +/// point for vectorization, like stores to consecutive memory addresses. +/// Starting from the seed instructions, it walks up the def-use chain looking +/// for more instructions that can be vectorized. This pass will generate vector +/// code if it can legally vectorize the code, regardless of whether it is +/// profitable or not. For now profitability is checked at the end of the region +/// pass pipeline by a dedicated pass that accepts or rejects the IR +/// transaction, depending on the cost. +class BottomUpVec final : public RegionPass { bool Change = false; std::unique_ptr Legality; /// The original instructions that are potentially dead after vectorization. @@ -55,16 +64,9 @@ class BottomUpVec final : public FunctionPass { /// Entry point for vectorization starting from \p Seeds. bool tryVectorize(ArrayRef Seeds); - /// The PM containing the pipeline of region passes. - RegionPassManager RPM; - public: - BottomUpVec(StringRef Pipeline); - bool runOnFunction(Function &F, const Analyses &A) final; - void printPipeline(raw_ostream &OS) const final { - OS << getName() << "\n"; - RPM.printPipeline(OS); - } + BottomUpVec() : RegionPass("bottom-up-vec") {} + bool runOnRegion(Region &Rgn, const Analyses &A) final; }; } // namespace llvm::sandboxir diff --git a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/SeedCollection.h b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/SeedCollection.h new file mode 100644 index 0000000000000..286d971ff4851 --- /dev/null +++ b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/SeedCollection.h @@ -0,0 +1,40 @@ +//===- SeedCollection.h -----------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// The seed-collection pass of the bottom-up vectorizer. +// + +#ifndef LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_PASSES_SEEDCOLLECTION_H +#define LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_PASSES_SEEDCOLLECTION_H + +#include "llvm/SandboxIR/Pass.h" +#include "llvm/SandboxIR/PassManager.h" + +namespace llvm::sandboxir { + +/// This pass collects the instructions that can become vectorization "seeds", +/// like stores to consecutive memory addresses. It then goes over the collected +/// seeds, slicing them into appropriately sized chunks, creating a Region with +/// the seed slice as the Auxiliary vector and runs the region pass pipeline. +class SeedCollection final : public FunctionPass { + + /// The PM containing the pipeline of region passes. + RegionPassManager RPM; + +public: + SeedCollection(StringRef Pipeline); + bool runOnFunction(Function &F, const Analyses &A) final; + void printPipeline(raw_ostream &OS) const final { + OS << getName() << "\n"; + RPM.printPipeline(OS); + } +}; + +} // namespace llvm::sandboxir + +#endif // LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_PASSES_SEEDCOLLECTION_H diff --git a/llvm/lib/SandboxIR/Region.cpp b/llvm/lib/SandboxIR/Region.cpp index ede738c35f032..086993e6dc872 100644 --- a/llvm/lib/SandboxIR/Region.cpp +++ b/llvm/lib/SandboxIR/Region.cpp @@ -40,8 +40,10 @@ Region::Region(Context &Ctx, TargetTransformInfo &TTI) CreateInstCB = Ctx.registerCreateInstrCallback( [this](Instruction *NewInst) { add(NewInst); }); - EraseInstCB = Ctx.registerEraseInstrCallback( - [this](Instruction *ErasedInst) { remove(ErasedInst); }); + EraseInstCB = Ctx.registerEraseInstrCallback([this](Instruction *ErasedInst) { + remove(ErasedInst); + removeFromAux(ErasedInst); + }); } Region::~Region() { @@ -84,11 +86,22 @@ void Region::setAux(unsigned Idx, Instruction *I) { Aux[Idx] = I; } +void Region::dropAuxMetadata(Instruction *I) { + auto *LLVMI = cast(I->Val); + LLVMI->setMetadata(AuxMDKind, nullptr); +} + +void Region::removeFromAux(Instruction *I) { + auto It = find(Aux, I); + if (It == Aux.end()) + return; + dropAuxMetadata(I); + Aux.erase(It); +} + void Region::clearAux() { - for (unsigned Idx : seq(0, Aux.size())) { - auto *LLVMI = cast(Aux[Idx]->Val); - LLVMI->setMetadata(AuxMDKind, nullptr); - } + for (unsigned Idx : seq(0, Aux.size())) + dropAuxMetadata(Aux[Idx]); Aux.clear(); } diff --git a/llvm/lib/Transforms/Vectorize/CMakeLists.txt b/llvm/lib/Transforms/Vectorize/CMakeLists.txt index 872e055294d55..676d23e1ebdf0 100644 --- a/llvm/lib/Transforms/Vectorize/CMakeLists.txt +++ b/llvm/lib/Transforms/Vectorize/CMakeLists.txt @@ -9,6 +9,7 @@ add_llvm_component_library(LLVMVectorize SandboxVectorizer/Legality.cpp SandboxVectorizer/Passes/BottomUpVec.cpp SandboxVectorizer/Passes/RegionsFromMetadata.cpp + SandboxVectorizer/Passes/SeedCollection.cpp SandboxVectorizer/Passes/TransactionAcceptOrRevert.cpp SandboxVectorizer/SandboxVectorizer.cpp SandboxVectorizer/SandboxVectorizerPassBuilder.cpp diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp index 0ccef5aecd28b..d57732090dcd6 100644 --- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp +++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp @@ -14,20 +14,10 @@ #include "llvm/SandboxIR/Module.h" #include "llvm/SandboxIR/Region.h" #include "llvm/SandboxIR/Utils.h" -#include "llvm/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizerPassBuilder.h" -#include "llvm/Transforms/Vectorize/SandboxVectorizer/SeedCollector.h" #include "llvm/Transforms/Vectorize/SandboxVectorizer/VecUtils.h" namespace llvm { -static cl::opt - OverrideVecRegBits("sbvec-vec-reg-bits", cl::init(0), cl::Hidden, - cl::desc("Override the vector register size in bits, " - "which is otherwise found by querying TTI.")); -static cl::opt - AllowNonPow2("sbvec-allow-non-pow2", cl::init(false), cl::Hidden, - cl::desc("Allow non-power-of-2 vectorization.")); - #ifndef NDEBUG static cl::opt AlwaysVerify("sbvec-always-verify", cl::init(false), cl::Hidden, @@ -37,10 +27,6 @@ static cl::opt namespace sandboxir { -BottomUpVec::BottomUpVec(StringRef Pipeline) - : FunctionPass("bottom-up-vec"), - RPM("rpm", Pipeline, SandboxVectorizerPassBuilder::createRegionPass) {} - static SmallVector getOperand(ArrayRef Bndl, unsigned OpIdx) { SmallVector Operands; @@ -413,6 +399,7 @@ Value *BottomUpVec::vectorizeRec(ArrayRef Bndl, } bool BottomUpVec::tryVectorize(ArrayRef Bndl) { + Change = false; DeadInstrCandidates.clear(); Legality->clear(); vectorizeRec(Bndl, {}, /*Depth=*/0); @@ -420,83 +407,21 @@ bool BottomUpVec::tryVectorize(ArrayRef Bndl) { return Change; } -bool BottomUpVec::runOnFunction(Function &F, const Analyses &A) { +bool BottomUpVec::runOnRegion(Region &Rgn, const Analyses &A) { + const auto &SeedSlice = Rgn.getAux(); + assert(SeedSlice.size() >= 2 && "Bad slice!"); + Function &F = *SeedSlice[0]->getParent()->getParent(); IMaps = std::make_unique(F.getContext()); Legality = std::make_unique( A.getAA(), A.getScalarEvolution(), F.getParent()->getDataLayout(), F.getContext(), *IMaps); - Change = false; - const auto &DL = F.getParent()->getDataLayout(); - unsigned VecRegBits = - OverrideVecRegBits != 0 - ? OverrideVecRegBits - : A.getTTI() - .getRegisterBitWidth(TargetTransformInfo::RGK_FixedWidthVector) - .getFixedValue(); - - // TODO: Start from innermost BBs first - for (auto &BB : F) { - SeedCollector SC(&BB, A.getScalarEvolution()); - for (SeedBundle &Seeds : SC.getStoreSeeds()) { - unsigned ElmBits = - Utils::getNumBits(VecUtils::getElementType(Utils::getExpectedType( - Seeds[Seeds.getFirstUnusedElementIdx()])), - DL); - - auto DivideBy2 = [](unsigned Num) { - auto Floor = VecUtils::getFloorPowerOf2(Num); - if (Floor == Num) - return Floor / 2; - return Floor; - }; - // Try to create the largest vector supported by the target. If it fails - // reduce the vector size by half. - for (unsigned SliceElms = std::min(VecRegBits / ElmBits, - Seeds.getNumUnusedBits() / ElmBits); - SliceElms >= 2u; SliceElms = DivideBy2(SliceElms)) { - if (Seeds.allUsed()) - break; - // Keep trying offsets after FirstUnusedElementIdx, until we vectorize - // the slice. This could be quite expensive, so we enforce a limit. - for (unsigned Offset = Seeds.getFirstUnusedElementIdx(), - OE = Seeds.size(); - Offset + 1 < OE; Offset += 1) { - // Seeds are getting used as we vectorize, so skip them. - if (Seeds.isUsed(Offset)) - continue; - if (Seeds.allUsed()) - break; - auto SeedSlice = - Seeds.getSlice(Offset, SliceElms * ElmBits, !AllowNonPow2); - if (SeedSlice.empty()) - continue; - - assert(SeedSlice.size() >= 2 && "Should have been rejected!"); - - // TODO: Refactor to remove the unnecessary copy to SeedSliceVals. - SmallVector SeedSliceVals(SeedSlice.begin(), - SeedSlice.end()); - // Create an empty region. Instructions get added to the region - // automatically by the callbacks. - auto &Ctx = F.getContext(); - Region Rgn(Ctx, A.getTTI()); - // Save the state of the IR before we make any changes. The - // transaction gets accepted/reverted by the tr-accept-or-revert pass. - Ctx.save(); - // Try to vectorize starting from the seed slice. The returned value - // is true if we found vectorizable code and generated some vector - // code for it. It does not mean that the code is profitable. - bool VecSuccess = tryVectorize(SeedSliceVals); - if (VecSuccess) - // WARNING: All passes should return false, except those that - // accept/revert the state. - Change |= RPM.runOnRegion(Rgn, A); - } - } - } - } - return Change; + // TODO: Refactor to remove the unnecessary copy to SeedSliceVals. + SmallVector SeedSliceVals(SeedSlice.begin(), SeedSlice.end()); + // Try to vectorize starting from the seed slice. The returned value + // is true if we found vectorizable code and generated some vector + // code for it. It does not mean that the code is profitable. + return tryVectorize(SeedSliceVals); } } // namespace sandboxir diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/PassRegistry.def b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/PassRegistry.def index f3aa12729860f..722c6f5db4192 100644 --- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/PassRegistry.def +++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/PassRegistry.def @@ -21,6 +21,7 @@ REGION_PASS("null", ::llvm::sandboxir::NullPass) REGION_PASS("print-instruction-count", ::llvm::sandboxir::PrintInstructionCount) REGION_PASS("tr-accept", ::llvm::sandboxir::TransactionAlwaysAccept) REGION_PASS("tr-accept-or-revert", ::llvm::sandboxir::TransactionAcceptOrRevert) +REGION_PASS("bottom-up-vec", ::llvm::sandboxir::BottomUpVec) #undef REGION_PASS @@ -28,7 +29,7 @@ REGION_PASS("tr-accept-or-revert", ::llvm::sandboxir::TransactionAcceptOrRevert) #define FUNCTION_PASS_WITH_PARAMS(NAME, CLASS_NAME) #endif -FUNCTION_PASS_WITH_PARAMS("bottom-up-vec", ::llvm::sandboxir::BottomUpVec) +FUNCTION_PASS_WITH_PARAMS("seed-collection", ::llvm::sandboxir::SeedCollection) FUNCTION_PASS_WITH_PARAMS("regions-from-metadata", ::llvm::sandboxir::RegionsFromMetadata) #undef FUNCTION_PASS_WITH_PARAMS diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/SeedCollection.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/SeedCollection.cpp new file mode 100644 index 0000000000000..0001c9bb7c7e4 --- /dev/null +++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/SeedCollection.cpp @@ -0,0 +1,98 @@ +//===- SeedCollection.cpp - Seed collection pass --------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/SeedCollection.h" +#include "llvm/Analysis/TargetTransformInfo.h" +#include "llvm/SandboxIR/Module.h" +#include "llvm/SandboxIR/Region.h" +#include "llvm/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizerPassBuilder.h" +#include "llvm/Transforms/Vectorize/SandboxVectorizer/SeedCollector.h" +#include "llvm/Transforms/Vectorize/SandboxVectorizer/VecUtils.h" + +namespace llvm { + +static cl::opt + OverrideVecRegBits("sbvec-vec-reg-bits", cl::init(0), cl::Hidden, + cl::desc("Override the vector register size in bits, " + "which is otherwise found by querying TTI.")); +static cl::opt + AllowNonPow2("sbvec-allow-non-pow2", cl::init(false), cl::Hidden, + cl::desc("Allow non-power-of-2 vectorization.")); + +namespace sandboxir { +SeedCollection::SeedCollection(StringRef Pipeline) + : FunctionPass("seed-collection"), + RPM("rpm", Pipeline, SandboxVectorizerPassBuilder::createRegionPass) {} + +bool SeedCollection::runOnFunction(Function &F, const Analyses &A) { + bool Change = false; + const auto &DL = F.getParent()->getDataLayout(); + unsigned VecRegBits = + OverrideVecRegBits != 0 + ? OverrideVecRegBits + : A.getTTI() + .getRegisterBitWidth(TargetTransformInfo::RGK_FixedWidthVector) + .getFixedValue(); + + // TODO: Start from innermost BBs first + for (auto &BB : F) { + SeedCollector SC(&BB, A.getScalarEvolution()); + for (SeedBundle &Seeds : SC.getStoreSeeds()) { + unsigned ElmBits = + Utils::getNumBits(VecUtils::getElementType(Utils::getExpectedType( + Seeds[Seeds.getFirstUnusedElementIdx()])), + DL); + + auto DivideBy2 = [](unsigned Num) { + auto Floor = VecUtils::getFloorPowerOf2(Num); + if (Floor == Num) + return Floor / 2; + return Floor; + }; + // Try to create the largest vector supported by the target. If it fails + // reduce the vector size by half. + for (unsigned SliceElms = std::min(VecRegBits / ElmBits, + Seeds.getNumUnusedBits() / ElmBits); + SliceElms >= 2u; SliceElms = DivideBy2(SliceElms)) { + if (Seeds.allUsed()) + break; + // Keep trying offsets after FirstUnusedElementIdx, until we vectorize + // the slice. This could be quite expensive, so we enforce a limit. + for (unsigned Offset = Seeds.getFirstUnusedElementIdx(), + OE = Seeds.size(); + Offset + 1 < OE; Offset += 1) { + // Seeds are getting used as we vectorize, so skip them. + if (Seeds.isUsed(Offset)) + continue; + if (Seeds.allUsed()) + break; + + auto SeedSlice = + Seeds.getSlice(Offset, SliceElms * ElmBits, !AllowNonPow2); + if (SeedSlice.empty()) + continue; + + assert(SeedSlice.size() >= 2 && "Should have been rejected!"); + + // Create a region containing the seed slice. + auto &Ctx = F.getContext(); + Region Rgn(Ctx, A.getTTI()); + // TODO: Replace save() with a save pass in the pass pipeline. + Ctx.save(); + Rgn.setAux(SeedSlice); + // Run the region pass pipeline. + Change |= RPM.runOnRegion(Rgn, A); + Rgn.clearAux(); + } + } + } + } + return Change; +} +} // namespace sandboxir +} // namespace llvm diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizer.cpp index b233d35212f94..4f17aa213bbb0 100644 --- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizer.cpp @@ -32,9 +32,11 @@ static cl::opt UserDefinedPassPipeline( SandboxVectorizerPass::SandboxVectorizerPass() : FPM("fpm") { if (UserDefinedPassPipeline == DefaultPipelineMagicStr) { // TODO: Add passes to the default pipeline. It currently contains: - // - the bottom-up-vectorizer pass + // - Seed collection, which creates seed regions and runs the pipeline + // - Bottom-up Vectorizer pass that starts from a seed + // - Accept or revert IR state pass FPM.setPassPipeline( - "bottom-up-vec", + "seed-collection", sandboxir::SandboxVectorizerPassBuilder::createFunctionPass); } else { // Create the user-defined pipeline. diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizerPassBuilder.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizerPassBuilder.cpp index 0c1ab55e91a5c..e552f0570dd9d 100644 --- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizerPassBuilder.cpp +++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizerPassBuilder.cpp @@ -4,6 +4,7 @@ #include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/NullPass.h" #include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/PrintInstructionCount.h" #include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/RegionsFromMetadata.h" +#include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/SeedCollection.h" #include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/TransactionAcceptOrRevert.h" #include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/TransactionAlwaysAccept.h" diff --git a/llvm/test/Transforms/SandboxVectorizer/bottomup_basic.ll b/llvm/test/Transforms/SandboxVectorizer/bottomup_basic.ll index 6b18d4069e0ae..ad23126b681fd 100644 --- a/llvm/test/Transforms/SandboxVectorizer/bottomup_basic.ll +++ b/llvm/test/Transforms/SandboxVectorizer/bottomup_basic.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5 -; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="bottom-up-vec" %s -S | FileCheck %s +; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="seed-collection" %s -S | FileCheck %s define void @store_load(ptr %ptr) { ; CHECK-LABEL: define void @store_load( diff --git a/llvm/test/Transforms/SandboxVectorizer/bottomup_seed_slice.ll b/llvm/test/Transforms/SandboxVectorizer/bottomup_seed_slice.ll index 202b5a6fbd6c9..f4fcc5742f8a7 100644 --- a/llvm/test/Transforms/SandboxVectorizer/bottomup_seed_slice.ll +++ b/llvm/test/Transforms/SandboxVectorizer/bottomup_seed_slice.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5 -; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="bottom-up-vec" %s -S | FileCheck %s +; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="seed-collection" %s -S | FileCheck %s declare void @foo() diff --git a/llvm/test/Transforms/SandboxVectorizer/bottomup_seed_slice_pow2.ll b/llvm/test/Transforms/SandboxVectorizer/bottomup_seed_slice_pow2.ll index 1b189831569f5..4218ca830dccf 100644 --- a/llvm/test/Transforms/SandboxVectorizer/bottomup_seed_slice_pow2.ll +++ b/llvm/test/Transforms/SandboxVectorizer/bottomup_seed_slice_pow2.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5 -; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2=false -sbvec-passes="bottom-up-vec" %s -S | FileCheck %s --check-prefix=POW2 -; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2=true -sbvec-passes="bottom-up-vec" %s -S | FileCheck %s --check-prefix=NON-POW2 +; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2=false -sbvec-passes="seed-collection" %s -S | FileCheck %s --check-prefix=POW2 +; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2=true -sbvec-passes="seed-collection" %s -S | FileCheck %s --check-prefix=NON-POW2 define void @pow2(ptr %ptr, float %val) { ; POW2-LABEL: define void @pow2( diff --git a/llvm/test/Transforms/SandboxVectorizer/cross_bbs.ll b/llvm/test/Transforms/SandboxVectorizer/cross_bbs.ll index ff1604173c317..75d7cc8d9cefe 100644 --- a/llvm/test/Transforms/SandboxVectorizer/cross_bbs.ll +++ b/llvm/test/Transforms/SandboxVectorizer/cross_bbs.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5 -; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="bottom-up-vec" %s -S | FileCheck %s +; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="seed-collection" %s -S | FileCheck %s define void @cross_bbs(ptr %ptr) { ; CHECK-LABEL: define void @cross_bbs( diff --git a/llvm/test/Transforms/SandboxVectorizer/default_pass_pipeline.ll b/llvm/test/Transforms/SandboxVectorizer/default_pass_pipeline.ll index 10de4338caf23..86000da42c799 100644 --- a/llvm/test/Transforms/SandboxVectorizer/default_pass_pipeline.ll +++ b/llvm/test/Transforms/SandboxVectorizer/default_pass_pipeline.ll @@ -5,8 +5,9 @@ ; This checks the default pass pipeline for the sandbox vectorizer. define void @pipeline() { ; CHECK: fpm -; CHECK: bottom-up-vec +; CHECK: seed-collection ; CHECK: rpm +; CHECK: bottom-up-vec ; CHECK: tr-accept-or-revert ; CHECK-EMPTY: ret void diff --git a/llvm/test/Transforms/SandboxVectorizer/pack.ll b/llvm/test/Transforms/SandboxVectorizer/pack.ll index da41036e3a58b..c1e22d3dc2e73 100644 --- a/llvm/test/Transforms/SandboxVectorizer/pack.ll +++ b/llvm/test/Transforms/SandboxVectorizer/pack.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5 -; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="bottom-up-vec" %s -S | FileCheck %s +; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="seed-collection" %s -S | FileCheck %s define void @pack_constants(ptr %ptr) { ; CHECK-LABEL: define void @pack_constants( diff --git a/llvm/test/Transforms/SandboxVectorizer/repeated_instrs.ll b/llvm/test/Transforms/SandboxVectorizer/repeated_instrs.ll index add762ac2d894..3e4ef6787a563 100644 --- a/llvm/test/Transforms/SandboxVectorizer/repeated_instrs.ll +++ b/llvm/test/Transforms/SandboxVectorizer/repeated_instrs.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5 -; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="bottom-up-vec" %s -S | FileCheck %s +; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="seed-collection" %s -S | FileCheck %s define i32 @repeated_splat(ptr %ptr, i32 %v) #0 { ; CHECK-LABEL: define i32 @repeated_splat( diff --git a/llvm/test/Transforms/SandboxVectorizer/scheduler.ll b/llvm/test/Transforms/SandboxVectorizer/scheduler.ll index acbec80db6b06..fd4847da920d1 100644 --- a/llvm/test/Transforms/SandboxVectorizer/scheduler.ll +++ b/llvm/test/Transforms/SandboxVectorizer/scheduler.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5 -; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="bottom-up-vec" %s -S | FileCheck %s +; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="seed-collection" %s -S | FileCheck %s ; This used to crash because the newly added pack instructions would not update ; the DAG and scheduler, leading to def-after-use. diff --git a/llvm/test/Transforms/SandboxVectorizer/special_opcodes.ll b/llvm/test/Transforms/SandboxVectorizer/special_opcodes.ll index e8fe8b4fa88e3..1aca7cf2a8bd8 100644 --- a/llvm/test/Transforms/SandboxVectorizer/special_opcodes.ll +++ b/llvm/test/Transforms/SandboxVectorizer/special_opcodes.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5 -; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="bottom-up-vec" %s -S | FileCheck %s +; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="seed-collection" %s -S | FileCheck %s ; This file includes tests for opcodes that need special checks. diff --git a/llvm/test/Transforms/SandboxVectorizer/user_pass_pipeline.ll b/llvm/test/Transforms/SandboxVectorizer/user_pass_pipeline.ll index b11b55ed96019..2c57a8e7347d2 100644 --- a/llvm/test/Transforms/SandboxVectorizer/user_pass_pipeline.ll +++ b/llvm/test/Transforms/SandboxVectorizer/user_pass_pipeline.ll @@ -1,9 +1,9 @@ ; RUN: opt -passes=sandbox-vectorizer -sbvec-print-pass-pipeline \ -; RUN: -disable-output -sbvec-passes="bottom-up-vec" %s \ +; RUN: -disable-output -sbvec-passes="seed-collection" %s \ ; RUN: | FileCheck %s ; ; RUN: opt -passes=sandbox-vectorizer -sbvec-print-pass-pipeline \ -; RUN: -disable-output -sbvec-passes="bottom-up-vec<>,regions-from-metadata<>" %s \ +; RUN: -disable-output -sbvec-passes="seed-collection<>,regions-from-metadata<>" %s \ ; RUN: | FileCheck --check-prefix CHECK-MULTIPLE-FUNCTION-PASSES %s ; !!!WARNING!!! This won't get updated by update_test_checks.py ! @@ -14,14 +14,14 @@ define void @pipeline() { } ; CHECK: fpm -; CHECK: bottom-up-vec +; CHECK: seed-collection ; CHECK: rpm ; CHECK: null ; CHECK: null ; CHECK-EMPTY: ; CHECK-MULTIPLE-FUNCTION-PASSES: fpm -; CHECK-MULTIPLE-FUNCTION-PASSES: bottom-up-vec +; CHECK-MULTIPLE-FUNCTION-PASSES: seed-collection ; CHECK-MULTIPLE-FUNCTION-PASSES: rpm ; CHECK-MULTIPLE-FUNCTION-PASSES: regions-from-metadata ; CHECK-MULTIPLE-FUNCTION-PASSES: rpm diff --git a/llvm/unittests/SandboxIR/RegionTest.cpp b/llvm/unittests/SandboxIR/RegionTest.cpp index 09c578bcfefaa..992c3f2cbd2ea 100644 --- a/llvm/unittests/SandboxIR/RegionTest.cpp +++ b/llvm/unittests/SandboxIR/RegionTest.cpp @@ -386,6 +386,49 @@ define void @foo(i8 %v) { ".*already.*"); } +// Check that Aux automatically drops instructions that get deleted. +TEST_F(RegionTest, AuxDeleteInstr) { + parseIR(C, R"IR( +define void @foo(i8 %v) { + %Add0 = add i8 %v, 0, !sandboxvec !0, !sandboxaux !1 + %Add1 = add i8 %v, 1, !sandboxvec !0, !sandboxaux !2 + %Add2 = add i8 %v, 2, !sandboxvec !0, !sandboxaux !3 + %Add3 = add i8 %v, 2, !sandboxvec !0, !sandboxaux !4 + ret void +} + +!0 = distinct !{!"sandboxregion"} +!1 = !{i32 0} +!2 = !{i32 1} +!3 = !{i32 2} +!4 = !{i32 3} +)IR"); + llvm::Function *LLVMF = &*M->getFunction("foo"); + sandboxir::Context Ctx(C); + auto *F = Ctx.createFunction(LLVMF); + auto *BB = &*F->begin(); + auto It = BB->begin(); + auto *Add0 = &*It++; + auto *Add1 = &*It++; + auto *Add2 = &*It++; + auto *Add3 = &*It++; + SmallVector> Regions = + sandboxir::Region::createRegionsFromMD(*F, *TTI); + auto &R = *Regions[0]; + EXPECT_THAT(R.getAux(), testing::ElementsAre(Add0, Add1, Add2, Add3)); + // Now delete Add1 and check that Aux contains nullptr instead of Add1. + Add2->eraseFromParent(); + EXPECT_THAT(R.getAux(), testing::ElementsAre(Add0, Add1, Add3)); + { + // Check that metadata have also been updated. + // But first drop Add3 to create a legal Aux vector with no gaps. + Add3->eraseFromParent(); + SmallVector> Regions = + sandboxir::Region::createRegionsFromMD(*F, *TTI); + EXPECT_THAT(Regions[0]->getAux(), testing::ElementsAre(Add0, Add1)); + } +} + TEST_F(RegionTest, AuxRoundTrip) { parseIR(C, R"IR( define i8 @foo(i8 %v0, i8 %v1) {