diff --git a/llvm/include/llvm/ADT/ArrayRef.h b/llvm/include/llvm/ADT/ArrayRef.h index 1c6799f1c56ed..d3d8a9ed6ca03 100644 --- a/llvm/include/llvm/ADT/ArrayRef.h +++ b/llvm/include/llvm/ADT/ArrayRef.h @@ -52,7 +52,7 @@ namespace llvm { using size_type = size_t; using difference_type = ptrdiff_t; - private: + protected: /// The start of the array, in an external buffer. const T *Data = nullptr; diff --git a/llvm/include/llvm/ADT/Matrix.h b/llvm/include/llvm/ADT/Matrix.h new file mode 100644 index 0000000000000..ac9a990c10eac --- /dev/null +++ b/llvm/include/llvm/ADT/Matrix.h @@ -0,0 +1,420 @@ +//===- Matrix.h - Two-dimensional Container with View -----------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_ADT_MATRIX_H +#define LLVM_ADT_MATRIX_H + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/Support/Compiler.h" + +namespace llvm { +template class JaggedArrayView; + +/// Due to the SmallVector infrastructure using SmallVectorAlignmentAndOffset +/// that depends on the exact data layout, no derived classes can have extra +/// members. +template +struct MatrixStorageBase : public SmallVectorImpl, SmallVectorStorage { + LLVM_ATTRIBUTE_ALWAYS_INLINE MatrixStorageBase() : SmallVectorImpl(N) {} + LLVM_ATTRIBUTE_ALWAYS_INLINE MatrixStorageBase(size_t Size) + : SmallVectorImpl(N) { + resize(Size); + } + LLVM_ATTRIBUTE_ALWAYS_INLINE ~MatrixStorageBase() { + destroy_range(this->begin(), this->end()); + } + LLVM_ATTRIBUTE_ALWAYS_INLINE MatrixStorageBase(const MatrixStorageBase &RHS) + : SmallVectorImpl(N) { + if (!RHS.empty()) + SmallVectorImpl::operator=(RHS); + } + LLVM_ATTRIBUTE_ALWAYS_INLINE MatrixStorageBase(MatrixStorageBase &&RHS) + : SmallVectorImpl(N) { + if (!RHS.empty()) + SmallVectorImpl::operator=(::std::move(RHS)); + } + using SmallVectorImpl::size; + using SmallVectorImpl::resize; + using SmallVectorImpl::append; + using SmallVectorImpl::erase; + using SmallVectorImpl::destroy_range; + using SmallVectorImpl::isSafeToReferenceAfterResize; + + LLVM_ATTRIBUTE_ALWAYS_INLINE T *begin() const { + return const_cast(SmallVectorImpl::begin()); + } + LLVM_ATTRIBUTE_ALWAYS_INLINE T *end() const { + return const_cast(SmallVectorImpl::end()); + } +}; + +/// A two-dimensional container storage, whose upper bound on the number of +/// columns should be known ahead of time. Not menat to be used directly: the +/// primary usage API is MatrixView. +template ::value> +class MatrixStorage { +public: + MatrixStorage() = delete; + MatrixStorage(size_t NRows, size_t NCols) + : Base(NRows * NCols), NCols(NCols) {} + MatrixStorage(size_t NCols) : Base(), NCols(NCols) {} + + LLVM_ATTRIBUTE_ALWAYS_INLINE size_t size() const { return Base.size(); } + LLVM_ATTRIBUTE_ALWAYS_INLINE bool empty() const { return !size(); } + LLVM_ATTRIBUTE_ALWAYS_INLINE size_t getNumRows() const { + assert(size() % NCols == 0); + return size() / NCols; + } + LLVM_ATTRIBUTE_ALWAYS_INLINE size_t getNumCols() const { return NCols; } + LLVM_ATTRIBUTE_ALWAYS_INLINE void setNumCols(size_t NCols) { + assert(empty() && "Column-resizing a non-empty MatrixStorage"); + this->NCols = NCols; + } + LLVM_ATTRIBUTE_ALWAYS_INLINE void resize(size_t NRows) { + Base.resize(NCols * NRows); + } + LLVM_ATTRIBUTE_ALWAYS_INLINE void reserve(size_t NRows) { + Base.reserve(NCols * NRows); + } + +protected: + template + friend class JaggedArrayView; + + LLVM_ATTRIBUTE_ALWAYS_INLINE T *begin() const { return Base.begin(); } + LLVM_ATTRIBUTE_ALWAYS_INLINE T *rowFromIdx(size_t RowIdx, + size_t Offset = 0) const { + assert(Offset < NCols); + return begin() + RowIdx * NCols + Offset; + } + LLVM_ATTRIBUTE_ALWAYS_INLINE std::pair + idxFromRow(T *Ptr) const { + assert(Ptr >= begin()); + size_t Offset = (Ptr - begin()) % NCols; + return {(Ptr - begin()) / NCols, Offset}; + } + + // If Arg.size() < NCols, the number of columns won't be changed, and the + // difference is default-constructed. + LLVM_ATTRIBUTE_ALWAYS_INLINE void addRow(const SmallVectorImpl &Arg) { + assert(Arg.size() <= NCols && + "MatrixStorage has insufficient number of columns"); + size_t Diff = NCols - Arg.size(); + Base.append(Arg.begin(), Arg.end()); + Base.append(Diff, T()); + } + + LLVM_ATTRIBUTE_ALWAYS_INLINE void eraseLastRow() { + assert(getNumRows() > 0 && "Non-empty MatrixStorage expected"); + Base.pop_back_n(NCols); + } + + LLVM_ATTRIBUTE_ALWAYS_INLINE bool willReallocateOnAddRow() const { + return Base.capacity() < Base.size() + NCols; + } + +private: + MatrixStorageBase Base; + size_t NCols; +}; + +/// MutableArrayRef with a copy-assign, and extra APIs. +template +struct [[nodiscard]] MutableRowView : public MutableArrayRef { + using pointer = typename MutableArrayRef::pointer; + using iterator = typename MutableArrayRef::iterator; + using const_iterator = typename MutableArrayRef::const_iterator; + + MutableRowView() = delete; + MutableRowView(pointer Data, size_t Length) + : MutableArrayRef(Data, Length) {} + MutableRowView(iterator Begin, iterator End) + : MutableArrayRef(Begin, End) {} + MutableRowView(const_iterator Begin, const_iterator End) + : MutableArrayRef(Begin, End) {} + MutableRowView(MutableArrayRef Other) + : MutableArrayRef(Other.data(), Other.size()) {} + MutableRowView(SmallVectorImpl &Vec) : MutableArrayRef(Vec) {} + + using MutableArrayRef::size; + using MutableArrayRef::data; + using MutableArrayRef::begin; + using MutableArrayRef::end; + + LLVM_ATTRIBUTE_ALWAYS_INLINE T &back() const { + return MutableArrayRef::back(); + } + LLVM_ATTRIBUTE_ALWAYS_INLINE T &front() const { + return MutableArrayRef::front(); + } + LLVM_ATTRIBUTE_ALWAYS_INLINE MutableRowView + drop_back(size_t N = 1) const { // NOLINT + return MutableArrayRef::drop_back(N); + } + LLVM_ATTRIBUTE_ALWAYS_INLINE MutableRowView + drop_front(size_t N = 1) const { // NOLINT + return MutableArrayRef::drop_front(N); + } + // This slice is different from the MutableArrayRef slice, and specifies a + // Begin and End index, instead of a Begin and Length. + LLVM_ATTRIBUTE_ALWAYS_INLINE MutableRowView slice(size_t Begin, + size_t End) { + return MutableArrayRef::slice(Begin, End - Begin); + } + LLVM_ATTRIBUTE_ALWAYS_INLINE void pop_back(size_t N = 1) { // NOLINT + this->Length -= N; + } + LLVM_ATTRIBUTE_ALWAYS_INLINE void pop_front(size_t N = 1) { // NOLINT + this->Data += N; + this->Length -= N; + } + + LLVM_ATTRIBUTE_ALWAYS_INLINE MutableRowView & + operator=(const SmallVectorImpl &Vec) { + copy_assign(Vec.begin(), Vec.end()); + return *this; + } + LLVM_ATTRIBUTE_ALWAYS_INLINE MutableRowView & + operator=(std::initializer_list IL) { + copy_assign(IL.begin(), IL.end()); + return *this; + } + + LLVM_ATTRIBUTE_ALWAYS_INLINE void swap(MutableRowView &Other) { + std::swap(this->Data, Other.Data); + std::swap(this->Length, Other.Length); + } + + // For better cache behavior. + LLVM_ATTRIBUTE_ALWAYS_INLINE void + copy_assign(const MutableRowView &Other) { // NOLINT + copy_assign(Other.begin(), Other.end()); + } + + // For better cache behavior. + LLVM_ATTRIBUTE_ALWAYS_INLINE void + copy_swap(MutableRowView &Other) { // NOLINT + SmallVector Buf{Other}; + Other.copy_assign(begin(), end()); + copy_assign(Buf.begin(), Buf.end()); + } + +protected: + LLVM_ATTRIBUTE_ALWAYS_INLINE void copy_assign(iterator Begin, + iterator End) { // NOLINT + std::uninitialized_copy(Begin, End, data()); + this->Length = End - Begin; + } + LLVM_ATTRIBUTE_ALWAYS_INLINE void copy_assign(const_iterator Begin, + const_iterator End) { // NOLINT + std::uninitialized_copy(Begin, End, data()); + this->Length = End - Begin; + } +}; + +/// The primary usage API of MatrixStorage. Abstracts out indexing-arithmetic, +/// eliminating memory operations on the underlying data. Supports +/// variable-length columns. +template ::value, + size_t NStorageInline = + CalculateSmallVectorDefaultInlinedElements::value> +class [[nodiscard]] JaggedArrayView { +public: + using row_type = MutableRowView; + using container_type = SmallVector; + using iterator = typename container_type::iterator; + using const_iterator = typename container_type::const_iterator; + + LLVM_ATTRIBUTE_ALWAYS_INLINE constexpr JaggedArrayView( + MatrixStorage &Mat, size_t RowSpan, size_t ColSpan) + : Mat(Mat) { + RowView.reserve(RowSpan); + for (size_t RowIdx = 0; RowIdx < RowSpan; ++RowIdx) { + auto RangeBegin = Mat.begin() + RowIdx * ColSpan; + RowView.emplace_back(RangeBegin, RangeBegin + ColSpan); + } + } + + // Constructor with a full View of the underlying MatrixStorage, if + // MatrixStorage has a non-zero number of Columns. Otherwise, creates an empty + // view. + LLVM_ATTRIBUTE_ALWAYS_INLINE constexpr JaggedArrayView( + MatrixStorage &Mat) + : JaggedArrayView(Mat, Mat.getNumRows(), Mat.getNumCols()) {} + + // Obvious copy-construator is deleted, since the underlying storage could + // have changed. + constexpr JaggedArrayView(const JaggedArrayView &) = delete; + + // Copy-assignment operator should not be used when the underlying storage + // changes. + LLVM_ATTRIBUTE_ALWAYS_INLINE constexpr JaggedArrayView & + operator=(const JaggedArrayView &Other) { + assert(Mat.begin() == Other.Mat.begin() && + "Underlying storage has changed: use custom copy-constructor"); + RowView = Other.RowView; + return *this; + } + + // The actual copy-constructor: to be used when the underlying storage is + // copy-constructed. + JaggedArrayView(const JaggedArrayView &OldView, + MatrixStorage &NewMat) + : Mat(NewMat) { + assert(OldView.Mat.size() == Mat.size() && + "Custom copy-constructor called on non-copied storage"); + + // The underlying storage will change. Construct a new RowView by performing + // pointer-arithmetic on the underlying storage of OldView, using pointers + // from OldVie. + for (const auto &R : OldView.RowView) { + auto [StorageIdx, StartOffset] = OldView.Mat.idxFromRow(R.data()); + RowView.emplace_back(Mat.rowFromIdx(StorageIdx, StartOffset), R.size()); + } + } + + void addRow(const SmallVectorImpl &Row) { + // Optimization when we know that the underying storage won't be resized. + if (LLVM_LIKELY(!Mat.willReallocateOnAddRow())) { + Mat.addRow(Row); + RowView.emplace_back(Mat.rowFromIdx(Mat.getNumRows() - 1), Row.size()); + return; + } + + // The underlying storage may be resized, performing reallocations. The + // pointers in RowView will no longer be valid, so save and restore the + // data. Construct RestoreData by performing pointer-arithmetic on the + // underlying storgge. + SmallVector> RestoreData; + RestoreData.reserve(RowView.size()); + for (const auto &R : RowView) { + auto [StorageIdx, StartOffset] = Mat.idxFromRow(R.data()); + RestoreData.emplace_back(StorageIdx, StartOffset, R.size()); + } + + Mat.addRow(Row); + + // Restore the RowView by performing pointer-arithmetic on the + // possibly-reallocated storage, using information from RestoreData. + RowView.clear(); + for (const auto &[StorageIdx, StartOffset, Len] : RestoreData) + RowView.emplace_back(Mat.rowFromIdx(StorageIdx, StartOffset), Len); + + // Finally, add the new row to the VRowView. + RowView.emplace_back(Mat.rowFromIdx(Mat.getNumRows() - 1), Row.size()); + } + + // To support addRow(View[Idx]). + LLVM_ATTRIBUTE_ALWAYS_INLINE void addRow(const row_type &Row) { + addRow(SmallVector{Row}); + } + + LLVM_ATTRIBUTE_ALWAYS_INLINE void addRow(std::initializer_list Row) { + addRow(SmallVector{Row}); + } + + LLVM_ATTRIBUTE_ALWAYS_INLINE constexpr row_type &operator[](size_t RowIdx) { + assert(RowIdx < RowView.size() && "Indexing out of bounds"); + return RowView[RowIdx]; + } + + LLVM_ATTRIBUTE_ALWAYS_INLINE constexpr T *data() const { + assert(!empty() && "Non-empty view expected"); + return RowView.front().data(); + } + size_t size() const { return getRowSpan() * getMaxColSpan(); } + LLVM_ATTRIBUTE_ALWAYS_INLINE bool empty() const { return RowView.empty(); } + LLVM_ATTRIBUTE_ALWAYS_INLINE size_t getRowSpan() const { + return RowView.size(); + } + LLVM_ATTRIBUTE_ALWAYS_INLINE size_t getColSpan(size_t RowIdx) const { + assert(RowIdx < RowView.size() && "Indexing out of bounds"); + return RowView[RowIdx].size(); + } + constexpr size_t getMaxColSpan() const { + return std::max_element(RowView.begin(), RowView.end(), + [](const row_type &RowA, const row_type &RowB) { + return RowA.size() < RowB.size(); + }) + ->size(); + } + + LLVM_ATTRIBUTE_ALWAYS_INLINE iterator begin() { return RowView.begin(); } + LLVM_ATTRIBUTE_ALWAYS_INLINE iterator end() { return RowView.end(); } + LLVM_ATTRIBUTE_ALWAYS_INLINE const_iterator begin() const { + return RowView.begin(); + } + LLVM_ATTRIBUTE_ALWAYS_INLINE const_iterator end() const { + return RowView.end(); + } + + constexpr JaggedArrayView rowSlice(size_t Begin, + size_t End) { + assert(Begin < getRowSpan() && End <= getRowSpan() && + "Indexing out of bounds"); + assert(Begin < End && "Invalid slice"); + container_type NewRowView; + for (size_t RowIdx = Begin; RowIdx < End; ++RowIdx) + NewRowView.emplace_back(RowView[RowIdx]); + return {Mat, std::move(NewRowView)}; + } + + constexpr JaggedArrayView colSlice(size_t Begin, + size_t End) { + assert(Begin < End && "Invalid slice"); + size_t MinColSpan = + std::min_element(RowView.begin(), RowView.end(), + [](const row_type &RowA, const row_type &RowB) { + return RowA.size() < RowB.size(); + }) + ->size(); + assert(Begin < MinColSpan && End <= MinColSpan && "Indexing out of bounds"); + container_type NewRowView; + for (row_type Row : RowView) + NewRowView.emplace_back(Row.slice(Begin, End)); + return {Mat, std::move(NewRowView)}; + } + + LLVM_ATTRIBUTE_ALWAYS_INLINE row_type &lastRow() { + assert(!empty() && "Non-empty view expected"); + return RowView.back(); + } + LLVM_ATTRIBUTE_ALWAYS_INLINE const row_type &lastRow() const { + assert(!empty() && "Non-empty view expected"); + return RowView.back(); + } + LLVM_ATTRIBUTE_ALWAYS_INLINE void dropLastRow() { + assert(!empty() && "Non-empty view expected"); + RowView.pop_back(); + } + + // For better cache behavior. To be used with copy_assign or copy_swap. + LLVM_ATTRIBUTE_ALWAYS_INLINE void eraseLastRow() { + assert(Mat.idxFromRow(lastRow().data()).first == Mat.getNumRows() - 1 && + "Last row does not correspond to last row in storage"); + dropLastRow(); + Mat.eraseLastRow(); + } + +protected: + // Helper constructor. + LLVM_ATTRIBUTE_ALWAYS_INLINE constexpr JaggedArrayView( + MatrixStorage &Mat, + SmallVectorImpl &&RowView) + : Mat(Mat), RowView(std::move(RowView)) {} + +private: + MatrixStorage &Mat; + container_type RowView; +}; +} // namespace llvm + +#endif diff --git a/llvm/include/llvm/Analysis/ConstraintSystem.h b/llvm/include/llvm/Analysis/ConstraintSystem.h index 7b02b618f7cb4..956acf0e6fa1e 100644 --- a/llvm/include/llvm/Analysis/ConstraintSystem.h +++ b/llvm/include/llvm/Analysis/ConstraintSystem.h @@ -9,22 +9,19 @@ #ifndef LLVM_ANALYSIS_CONSTRAINTSYSTEM_H #define LLVM_ANALYSIS_CONSTRAINTSYSTEM_H -#include "llvm/ADT/APInt.h" -#include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/Matrix.h" #include "llvm/Support/MathExtras.h" -#include - namespace llvm { class Value; class ConstraintSystem { struct Entry { - int64_t Coefficient; - uint16_t Id; + int64_t Coefficient = 0; + uint16_t Id = 0; + Entry() = default; Entry(int64_t Coefficient, uint16_t Id) : Coefficient(Coefficient), Id(Id) {} }; @@ -48,7 +45,10 @@ class ConstraintSystem { /// Current linear constraints in the system. /// An entry of the form c0, c1, ... cn represents the following constraint: /// c0 >= v0 * c1 + .... + v{n-1} * cn - SmallVector, 4> Constraints; + MatrixStorage Constraints; + + /// Constraints is only ever manipulated via this View. + JaggedArrayView View; /// A map of variables (IR values) to their corresponding index in the /// constraint system. @@ -64,18 +64,41 @@ class ConstraintSystem { SmallVector getVarNamesList() const; public: - ConstraintSystem() {} - ConstraintSystem(ArrayRef FunctionArgs) { + // The Matrix Constraints should always be initialized with an upper-bound + // number of columns. The default constructor hard-codes an upper-bound of 6, + // as it is only used in unit tests, and not in the actual + // ConstraintElimination Analysis. + ConstraintSystem() : Constraints(6), View(Constraints) {} + + // This constructor is used by ConstraintElimination, inside ConstraintInfo. + // Unfortunately, due to calls to addFact, that adds local variables, it is + // impossible to know how many local variables there are in advance. + // ConstraintElimination has a fixed upper-bound on the number of columns, + // configurable as a cl::opt, so use that number, and don't add the constraint + // if it exceeds that number. + ConstraintSystem(ArrayRef FunctionArgs, size_t NRows, size_t NCols) + : Constraints(NCols), View(Constraints) { + Constraints.reserve(NRows); NumVariables += FunctionArgs.size(); for (auto *Arg : FunctionArgs) { Value2Index.insert({Arg, Value2Index.size() + 1}); } } - ConstraintSystem(const DenseMap &Value2Index) - : NumVariables(Value2Index.size()), Value2Index(Value2Index) {} + + // This constructor is only used by the dump function in + // ConstraintElimination. + ConstraintSystem(const DenseMap &Value2Index, + unsigned NVars) + : NumVariables(Value2Index.size()), + Constraints(std::max(Value2Index.size(), NVars)), View(Constraints), + Value2Index(Value2Index) {} + + ConstraintSystem(const ConstraintSystem &Other) + : NumVariables(Other.NumVariables), Constraints(Other.Constraints), + View(Other.View, Constraints), Value2Index(Other.Value2Index) {} bool addVariableRow(ArrayRef R) { - assert(Constraints.empty() || R.size() == NumVariables); + assert(View.empty() || R.size() == NumVariables); // If all variable coefficients are 0, the constraint does not provide any // usable information. if (all_of(ArrayRef(R).drop_front(1), [](int64_t C) { return C == 0; })) @@ -87,9 +110,15 @@ class ConstraintSystem { continue; NewRow.emplace_back(C, Idx); } - if (Constraints.empty()) + + // There is no correctness issue if we don't add a constraint. + if (NewRow.size() > Constraints.getNumCols()) + return false; + + if (View.empty()) NumVariables = R.size(); - Constraints.push_back(std::move(NewRow)); + + View.addRow(std::move(NewRow)); return true; } @@ -145,21 +174,21 @@ class ConstraintSystem { bool isConditionImplied(SmallVector R) const; SmallVector getLastConstraint() const { - assert(!Constraints.empty() && "Constraint system is empty"); + assert(!View.empty() && "Constraint system is empty"); SmallVector Result(NumVariables, 0); - for (auto &Entry : Constraints.back()) + for (auto &Entry : View.lastRow()) Result[Entry.Id] = Entry.Coefficient; return Result; } - void popLastConstraint() { Constraints.pop_back(); } + void popLastConstraint() { View.dropLastRow(); } void popLastNVariables(unsigned N) { assert(NumVariables > N); NumVariables -= N; } /// Returns the number of rows in the constraint system. - unsigned size() const { return Constraints.size(); } + unsigned size() const { return View.getRowSpan(); } /// Print the constraints in the system. void dump() const; diff --git a/llvm/lib/Analysis/ConstraintSystem.cpp b/llvm/lib/Analysis/ConstraintSystem.cpp index e4c9dcc7544e9..bf478e19ded7f 100644 --- a/llvm/lib/Analysis/ConstraintSystem.cpp +++ b/llvm/lib/Analysis/ConstraintSystem.cpp @@ -7,11 +7,12 @@ //===----------------------------------------------------------------------===// #include "llvm/Analysis/ConstraintSystem.h" +#include "llvm/ADT/Matrix.h" #include "llvm/ADT/SmallVector.h" -#include "llvm/Support/MathExtras.h" #include "llvm/ADT/StringExtras.h" #include "llvm/IR/Value.h" #include "llvm/Support/Debug.h" +#include "llvm/Support/MathExtras.h" #include @@ -26,37 +27,38 @@ bool ConstraintSystem::eliminateUsingFM() { // analysis." // Supercomputing'91: Proceedings of the 1991 ACM/ // IEEE conference on Supercomputing. IEEE, 1991. - assert(!Constraints.empty() && + assert(!View.empty() && "should only be called for non-empty constraint systems"); unsigned LastIdx = NumVariables - 1; // First, either remove the variable in place if it is 0 or add the row to // RemainingRows and remove it from the system. - SmallVector, 4> RemainingRows; - for (unsigned R1 = 0; R1 < Constraints.size();) { - SmallVector &Row1 = Constraints[R1]; + MatrixStorage RemainingRows(View.getMaxColSpan()); + JaggedArrayView RemainingRowsView{RemainingRows}; + for (unsigned R1 = 0; R1 < View.getRowSpan();) { + auto &Row1 = View[R1]; if (getLastCoefficient(Row1, LastIdx) == 0) { if (Row1.size() > 0 && Row1.back().Id == LastIdx) Row1.pop_back(); R1++; } else { - std::swap(Constraints[R1], Constraints.back()); - RemainingRows.push_back(std::move(Constraints.back())); - Constraints.pop_back(); + View[R1].swap(View.lastRow()); + RemainingRowsView.addRow(View.lastRow()); + View.dropLastRow(); } } // Process rows where the variable is != 0. - unsigned NumRemainingConstraints = RemainingRows.size(); + unsigned NumRemainingConstraints = RemainingRowsView.getRowSpan(); for (unsigned R1 = 0; R1 < NumRemainingConstraints; R1++) { // FIXME do not use copy for (unsigned R2 = R1 + 1; R2 < NumRemainingConstraints; R2++) { if (R1 == R2) continue; - int64_t UpperLast = getLastCoefficient(RemainingRows[R2], LastIdx); - int64_t LowerLast = getLastCoefficient(RemainingRows[R1], LastIdx); + int64_t UpperLast = getLastCoefficient(RemainingRowsView[R2], LastIdx); + int64_t LowerLast = getLastCoefficient(RemainingRowsView[R1], LastIdx); assert( UpperLast != 0 && LowerLast != 0 && "RemainingRows should only contain rows where the variable is != 0"); @@ -74,8 +76,8 @@ bool ConstraintSystem::eliminateUsingFM() { SmallVector NR; unsigned IdxUpper = 0; unsigned IdxLower = 0; - auto &LowerRow = RemainingRows[LowerR]; - auto &UpperRow = RemainingRows[UpperR]; + auto &LowerRow = RemainingRowsView[LowerR]; + auto &UpperRow = RemainingRowsView[UpperR]; while (true) { if (IdxUpper >= UpperRow.size() || IdxLower >= LowerRow.size()) break; @@ -112,9 +114,9 @@ bool ConstraintSystem::eliminateUsingFM() { } if (NR.empty()) continue; - Constraints.push_back(std::move(NR)); + View.addRow(std::move(NR)); // Give up if the new system gets too big. - if (Constraints.size() > 500) + if (size() > 500) return false; } } @@ -124,15 +126,15 @@ bool ConstraintSystem::eliminateUsingFM() { } bool ConstraintSystem::mayHaveSolutionImpl() { - while (!Constraints.empty() && NumVariables > 1) { + while (!View.empty() && NumVariables > 1) { if (!eliminateUsingFM()) return true; } - if (Constraints.empty() || NumVariables > 1) + if (View.empty() || NumVariables > 1) return true; - return all_of(Constraints, [](auto &R) { + return all_of(View, [](auto &R) { if (R.empty()) return true; if (R[0].Id == 0) @@ -158,10 +160,10 @@ SmallVector ConstraintSystem::getVarNamesList() const { void ConstraintSystem::dump() const { #ifndef NDEBUG - if (Constraints.empty()) + if (View.empty()) return; SmallVector Names = getVarNamesList(); - for (const auto &Row : Constraints) { + for (const auto &Row : View) { SmallVector Parts; for (const Entry &E : Row) { if (E.Id >= NumVariables) diff --git a/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp b/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp index c31173879af1e..7c7aed742dabb 100644 --- a/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp +++ b/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp @@ -40,7 +40,6 @@ #include "llvm/Transforms/Utils/Cloning.h" #include "llvm/Transforms/Utils/ValueMapper.h" -#include #include #include @@ -57,6 +56,10 @@ static cl::opt MaxRows("constraint-elimination-max-rows", cl::init(500), cl::Hidden, cl::desc("Maximum number of rows to keep in constraint system")); +static cl::opt MaxColumns( + "constraint-elimination-max-cols", cl::init(50), cl::Hidden, + cl::desc("Maximum number of columns to keep in constraint system")); + static cl::opt DumpReproducers( "constraint-elimination-dump-reproducers", cl::init(false), cl::Hidden, cl::desc("Dump IR to reproduce successful transformations.")); @@ -273,8 +276,10 @@ class ConstraintInfo { const DataLayout &DL; public: - ConstraintInfo(const DataLayout &DL, ArrayRef FunctionArgs) - : UnsignedCS(FunctionArgs), SignedCS(FunctionArgs), DL(DL) { + ConstraintInfo(const DataLayout &DL, ArrayRef FunctionArgs, + unsigned MaxRows, unsigned MaxColumns) + : UnsignedCS(FunctionArgs, MaxRows, MaxColumns), + SignedCS(FunctionArgs, MaxRows, MaxColumns), DL(DL) { auto &Value2Index = getValue2Index(false); // Add Arg > -1 constraints to unsigned system for all function arguments. for (Value *Arg : FunctionArgs) { @@ -303,6 +308,7 @@ class ConstraintInfo { void popLastNVariables(bool Signed, unsigned N) { getCS(Signed).popLastNVariables(N); } + const DataLayout &getDataLayout() const { return DL; } bool doesHold(CmpInst::Predicate Pred, Value *A, Value *B) const; @@ -894,7 +900,7 @@ void ConstraintInfo::transferToOtherSystem( static void dumpConstraint(ArrayRef C, const DenseMap &Value2Index) { - ConstraintSystem CS(Value2Index); + ConstraintSystem CS(Value2Index, C.size()); CS.addVariableRowFill(C); CS.dump(); } @@ -1491,7 +1497,7 @@ removeEntryFromStack(const StackEntry &E, ConstraintInfo &Info, /// Check if either the first condition of an AND or OR is implied by the /// (negated in case of OR) second condition or vice versa. static bool checkOrAndOpImpliedByOther( - FactOrCheck &CB, ConstraintInfo &Info, Module *ReproducerModule, + const FactOrCheck &CB, ConstraintInfo &Info, Module *ReproducerModule, SmallVectorImpl &ReproducerCondStack, SmallVectorImpl &DFSInStack) { @@ -1671,18 +1677,92 @@ tryToSimplifyOverflowMath(IntrinsicInst *II, ConstraintInfo &Info, return Changed; } -static bool eliminateConstraints(Function &F, DominatorTree &DT, LoopInfo &LI, - ScalarEvolution &SE, - OptimizationRemarkEmitter &ORE) { - bool Changed = false; +/// Performs a dry run of AddFact, computing a conservative estimate of the +/// number of new variables introduced. +static void dryRunAddFact(CmpInst::Predicate Pred, Value *A, Value *B, + const ConstraintInfo &Info, unsigned &EstimatedRowsA, + unsigned &EstimatedRowsB, + unsigned &EstimatedColumns) { + auto UpdateEstimate = [&Info, &EstimatedRowsA, &EstimatedRowsB, + &EstimatedColumns](CmpInst::Predicate Pred, Value *A, + Value *B) { + SmallVector NewVars; + auto R = Info.getConstraint(Pred, A, B, NewVars); + + // We offset it by 1 due to logic in addFact. + unsigned NewEstimate = + count_if(R.Coefficients, [](int64_t C) { return C != 0; }) + 1; + + EstimatedColumns = std::max(EstimatedColumns, NewEstimate); + if (R.IsSigned) + ++EstimatedRowsA; + else + ++EstimatedRowsB; + }; + + UpdateEstimate(Pred, A, B); + + // What follows is a dry-run of transferToOtherSystem. + auto IsKnownNonNegative = [&Info](Value *V) { + return Info.doesHold(CmpInst::ICMP_SGE, V, + ConstantInt::get(V->getType(), 0)) || + isKnownNonNegative(V, Info.getDataLayout(), + MaxAnalysisRecursionDepth - 1); + }; + + if (!A->getType()->isIntegerTy()) + return; + + switch (Pred) { + default: + break; + case CmpInst::ICMP_ULT: + case CmpInst::ICMP_ULE: + if (IsKnownNonNegative(B)) { + UpdateEstimate(CmpInst::ICMP_SGE, A, ConstantInt::get(B->getType(), 0)); + UpdateEstimate(CmpInst::getSignedPredicate(Pred), A, B); + } + break; + case CmpInst::ICMP_UGE: + case CmpInst::ICMP_UGT: + if (IsKnownNonNegative(A)) { + UpdateEstimate(CmpInst::ICMP_SGE, B, ConstantInt::get(B->getType(), 0)); + UpdateEstimate(CmpInst::getSignedPredicate(Pred), A, B); + } + break; + case CmpInst::ICMP_SLT: + if (IsKnownNonNegative(A)) + UpdateEstimate(CmpInst::ICMP_ULT, A, B); + break; + case CmpInst::ICMP_SGT: + if (Info.doesHold(CmpInst::ICMP_SGE, B, ConstantInt::get(B->getType(), -1))) + UpdateEstimate(CmpInst::ICMP_UGE, A, ConstantInt::get(B->getType(), 0)); + if (IsKnownNonNegative(B)) + UpdateEstimate(CmpInst::ICMP_UGT, A, B); + break; + case CmpInst::ICMP_SGE: + if (IsKnownNonNegative(B)) + UpdateEstimate(CmpInst::ICMP_UGE, A, B); + break; + } +} + +/// Performs a dry run of the transform, computing a conservative estimate of +/// the total number of columns we need in the underlying storage. +static std::tuple +dryRun(Function &F, DominatorTree &DT, LoopInfo &LI, ScalarEvolution &SE) { DT.updateDFSNumbers(); SmallVector FunctionArgs; for (Value &Arg : F.args()) FunctionArgs.push_back(&Arg); - ConstraintInfo Info(F.getDataLayout(), FunctionArgs); State S(DT, LI, SE); - std::unique_ptr ReproducerModule( - DumpReproducers ? new Module(F.getName(), F.getContext()) : nullptr); + unsigned EstimatedColumns = FunctionArgs.size() + 1; + + // EstimatedRowsA corresponds to SignedCS, and EstimatedRowsB corresponds to + // UnsignedCS. + unsigned EstimatedRowsA = 0, EstimatedRowsB = 1; + ConstraintInfo Info(F.getDataLayout(), FunctionArgs, EstimatedRowsB, + EstimatedColumns); // First, collect conditions implied by branches and blocks with their // Dominator DFS in and out numbers. @@ -1725,12 +1805,91 @@ static bool eliminateConstraints(Function &F, DominatorTree &DT, LoopInfo &LI, return A.NumIn < B.NumIn; }); + for (const FactOrCheck &CB : S.WorkList) { + ICmpInst::Predicate Pred; + Value *A, *B; + if (CB.isCheck()) { + // What follows is a dry-run of checkOrAndOpImpliedByOther, without + // assuming that instructions have been simplified, as they would have + // during the course of normal operation. + auto *ContextInst = CB.getContextInst(); + if (auto *Cmp = + dyn_cast_or_null(CB.getInstructionToSimplify())) { + unsigned OtherOpIdx = ContextInst->getOperand(0) == Cmp ? 1 : 0; + if (match(ContextInst, m_LogicalOp()) && + match(ContextInst->getOperand(OtherOpIdx), + m_ICmp(Pred, m_Value(A), m_Value(B)))) { + if (match(ContextInst, m_LogicalOr())) + Pred = CmpInst::getInversePredicate(Pred); + dryRunAddFact(Pred, A, B, Info, EstimatedRowsA, EstimatedRowsB, + EstimatedColumns); + } + } + continue; + } + if (!CB.isConditionFact()) { + Value *X; + if (match(CB.Inst, m_Intrinsic(m_Value(X)))) { + if (cast(CB.Inst->getOperand(1))->isOne()) + dryRunAddFact(CmpInst::ICMP_SGE, CB.Inst, + ConstantInt::get(CB.Inst->getType(), 0), Info, + EstimatedRowsA, EstimatedRowsB, EstimatedColumns); + dryRunAddFact(CmpInst::ICMP_SGE, CB.Inst, X, Info, EstimatedRowsA, + EstimatedRowsB, EstimatedColumns); + continue; + } + + if (auto *MinMax = dyn_cast(CB.Inst)) { + Pred = ICmpInst::getNonStrictPredicate(MinMax->getPredicate()); + dryRunAddFact(Pred, MinMax, MinMax->getLHS(), Info, EstimatedRowsA, + EstimatedRowsB, EstimatedColumns); + dryRunAddFact(Pred, MinMax, MinMax->getRHS(), Info, EstimatedRowsA, + EstimatedRowsB, EstimatedColumns); + continue; + } + } + + if (CB.isConditionFact()) { + Pred = CB.Cond.Pred; + A = CB.Cond.Op0; + B = CB.Cond.Op1; + } else { + bool Matched = match(CB.Inst, m_Intrinsic( + m_ICmp(Pred, m_Value(A), m_Value(B)))); + (void)Matched; + assert(Matched && "Must have an assume intrinsic with a icmp operand"); + } + dryRunAddFact(Pred, A, B, Info, EstimatedRowsA, EstimatedRowsB, + EstimatedColumns); + } + return {S, std::max(EstimatedRowsA, EstimatedRowsB), EstimatedColumns}; +} + +static bool eliminateConstraints(Function &F, DominatorTree &DT, LoopInfo &LI, + ScalarEvolution &SE, + OptimizationRemarkEmitter &ORE) { + bool Changed = false; + const auto &[S, EstimatedRows, EstimatedColumns] = dryRun(F, DT, LI, SE); + + // Fail early if estimates exceed limits. Row estimate could be off by up to + // 40%. + if (EstimatedRows > 1.4 * MaxRows || EstimatedColumns > MaxColumns) + return false; + + SmallVector FunctionArgs; + for (Value &Arg : F.args()) + FunctionArgs.push_back(&Arg); + ConstraintInfo Info(F.getDataLayout(), FunctionArgs, EstimatedRows, + EstimatedColumns); + std::unique_ptr ReproducerModule( + DumpReproducers ? new Module(F.getName(), F.getContext()) : nullptr); + SmallVector ToRemove; // Finally, process ordered worklist and eliminate implied conditions. SmallVector DFSInStack; SmallVector ReproducerCondStack; - for (FactOrCheck &CB : S.WorkList) { + for (const FactOrCheck &CB : S.WorkList) { // First, pop entries from the stack that are out-of-scope for CB. Remove // the corresponding entry from the constraint system. while (!DFSInStack.empty()) { diff --git a/llvm/test/Transforms/ConstraintElimination/max-row-limit.ll b/llvm/test/Transforms/ConstraintElimination/max-row-column-limit.ll similarity index 81% rename from llvm/test/Transforms/ConstraintElimination/max-row-limit.ll rename to llvm/test/Transforms/ConstraintElimination/max-row-column-limit.ll index 0e078109ed663..2f3b62dc5dab7 100644 --- a/llvm/test/Transforms/ConstraintElimination/max-row-limit.ll +++ b/llvm/test/Transforms/ConstraintElimination/max-row-column-limit.ll @@ -1,7 +1,9 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -passes=constraint-elimination -S %s | FileCheck --check-prefixes=COMMON,SIMP %s -; RUN: opt -passes=constraint-elimination -constraint-elimination-max-rows=9 -S %s | FileCheck --check-prefixes=COMMON,SIMP %s -; RUN: opt -passes=constraint-elimination -constraint-elimination-max-rows=8 -S %s | FileCheck --check-prefixes=COMMON,NOSIMP %s +; RUN: opt -passes=constraint-elimination -S %s | FileCheck --check-prefixes=SIMP %s +; RUN: opt -passes=constraint-elimination -constraint-elimination-max-rows=8 -S %s | FileCheck --check-prefixes=SIMP %s +; RUN: opt -passes=constraint-elimination -constraint-elimination-max-cols=6 -S %s | FileCheck --check-prefixes=SIMP %s +; RUN: opt -passes=constraint-elimination -constraint-elimination-max-rows=7 -S %s | FileCheck --check-prefixes=NOSIMP %s +; RUN: opt -passes=constraint-elimination -constraint-elimination-max-cols=5 -S %s | FileCheck --check-prefixes=NOSIMP %s define i1 @test_max_row_limit(i32 %l0, i32 %l1, i32 %l2, i32 %l3, i32 %l4) { @@ -22,7 +24,8 @@ define i1 @test_max_row_limit(i32 %l0, i32 %l1, i32 %l2, i32 %l3, i32 %l4) { ; SIMP-NEXT: [[C4:%.*]] = icmp uge i32 [[L4:%.*]], 100 ; SIMP-NEXT: br i1 [[C4]], label [[BB5:%.*]], label [[EXIT]] ; SIMP: bb5: -; SIMP-NEXT: ret i1 true +; SIMP-NEXT: [[C5:%.*]] = icmp sge i32 [[L4:%.*]], 100 +; SIMP-NEXT: ret i1 [[C5]] ; SIMP: exit: ; SIMP-NEXT: ret i1 false ; @@ -43,7 +46,7 @@ define i1 @test_max_row_limit(i32 %l0, i32 %l1, i32 %l2, i32 %l3, i32 %l4) { ; NOSIMP-NEXT: [[C4:%.*]] = icmp uge i32 [[L4:%.*]], 100 ; NOSIMP-NEXT: br i1 [[C4]], label [[BB5:%.*]], label [[EXIT]] ; NOSIMP: bb5: -; NOSIMP-NEXT: [[C5:%.*]] = icmp uge i32 [[L4]], 100 +; NOSIMP-NEXT: [[C5:%.*]] = icmp sge i32 [[L4]], 100 ; NOSIMP-NEXT: ret i1 [[C5]] ; NOSIMP: exit: ; NOSIMP-NEXT: ret i1 false @@ -69,11 +72,9 @@ bb4: br i1 %c4, label %bb5, label %exit bb5: - %c5 = icmp uge i32 %l4, 100 + %c5 = icmp sge i32 %l4, 100 ret i1 %c5 exit: ret i1 false } -;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: -; COMMON: {{.*}} diff --git a/llvm/unittests/ADT/CMakeLists.txt b/llvm/unittests/ADT/CMakeLists.txt index f48d840a10595..cd04778e86e2d 100644 --- a/llvm/unittests/ADT/CMakeLists.txt +++ b/llvm/unittests/ADT/CMakeLists.txt @@ -54,6 +54,7 @@ add_llvm_unittest(ADTTests LazyAtomicPointerTest.cpp MappedIteratorTest.cpp MapVectorTest.cpp + MatrixTest.cpp PackedVectorTest.cpp PagedVectorTest.cpp PointerEmbeddedIntTest.cpp diff --git a/llvm/unittests/ADT/MatrixTest.cpp b/llvm/unittests/ADT/MatrixTest.cpp new file mode 100644 index 0000000000000..afd9888017798 --- /dev/null +++ b/llvm/unittests/ADT/MatrixTest.cpp @@ -0,0 +1,345 @@ +#include "llvm/ADT/Matrix.h" +#include "llvm/ADT/DynamicAPInt.h" +#include "gtest/gtest.h" + +using namespace llvm; + +template static SmallVector getDummyValues(size_t NValues) { + SmallVector Ret(NValues); + for (size_t Idx = 0; Idx < NValues; ++Idx) + Ret[Idx] = T(Idx + 1); + return Ret; +} + +template class MatrixTest : public testing::Test { +protected: + MatrixTest() + : ColumnInitMatrix(8), SmallMatrix(2, 2), OtherSmall(3, 3), + LargeMatrix(16, 16) {} + MatrixStorage ColumnInitMatrix; + MatrixStorage SmallMatrix; + MatrixStorage OtherSmall; + MatrixStorage LargeMatrix; + SmallVector getDummyRow(size_t NValues) { + return getDummyValues(NValues); + } +}; + +using MatrixTestTypes = ::testing::Types; +TYPED_TEST_SUITE(MatrixTest, MatrixTestTypes, ); + +TYPED_TEST(MatrixTest, Construction) { + auto &E = this->ColumnInitMatrix; + ASSERT_TRUE(E.empty()); + EXPECT_EQ(E.getNumCols(), 8u); + E.setNumCols(3); + EXPECT_EQ(E.getNumCols(), 3u); + EXPECT_TRUE(E.empty()); + EXPECT_EQ(E.getNumRows(), 0u); + auto &M = this->SmallMatrix; + EXPECT_FALSE(M.empty()); + EXPECT_EQ(M.size(), 4u); + EXPECT_EQ(M.getNumRows(), 2u); + EXPECT_EQ(M.getNumCols(), 2u); +} + +TYPED_TEST(MatrixTest, CopyConstruction) { + auto &OldMat = this->SmallMatrix; + auto V = JaggedArrayView{OldMat}; + V[0] = this->getDummyRow(2); + V[0].pop_back(); + V[1] = this->getDummyRow(2); + V[1].pop_front(); + ASSERT_EQ(V.getRowSpan(), 2u); + ASSERT_EQ(V.getColSpan(0), 1u); + ASSERT_EQ(V.getColSpan(1), 1u); + EXPECT_EQ(V[0][0], 1); + EXPECT_EQ(V[1][0], 2); + EXPECT_EQ(JaggedArrayView{OldMat}[0][0], 1); + EXPECT_EQ(JaggedArrayView{OldMat}[0][1], 2); + EXPECT_EQ(JaggedArrayView{OldMat}[1][0], 1); + EXPECT_EQ(JaggedArrayView{OldMat}[1][1], 2); + MatrixStorage NewMat{OldMat}; + JaggedArrayView C{V, NewMat}; + ASSERT_EQ(C.getRowSpan(), 2u); + ASSERT_EQ(C.getColSpan(0), 1u); + ASSERT_EQ(C.getColSpan(1), 1u); + EXPECT_EQ(C[0][0], 1); + EXPECT_EQ(C[1][0], 2); + C.addRow(this->getDummyRow(2)); + EXPECT_EQ(C[2][0], 1); + EXPECT_EQ(C[2][1], 2); +} + +TYPED_TEST(MatrixTest, RowOps) { + auto &M = this->SmallMatrix; + auto &O = this->OtherSmall; + JaggedArrayView V{M}; + ASSERT_EQ(M.getNumRows(), 2u); + ASSERT_EQ(V.getRowSpan(), 2u); + V[0] = this->getDummyRow(2); + V = JaggedArrayView{M}; + EXPECT_EQ(V[0][0], 1); + EXPECT_EQ(V[0][1], 2); + ASSERT_EQ(M.getNumRows(), 2u); + V.addRow({TypeParam(4), TypeParam(5)}); + ASSERT_EQ(M.getNumRows(), 3u); + EXPECT_EQ(V[2][0], 4); + EXPECT_EQ(V[2][1], 5); + ASSERT_EQ(O.getNumRows(), 3u); + JaggedArrayView W{O}; + W.addRow(V[0]); + ASSERT_EQ(O.getNumCols(), 3u); + ASSERT_EQ(O.getNumRows(), 4u); + W[3] = {TypeParam(7), TypeParam(8)}; + auto &WRow3 = W[3]; + WRow3 = WRow3.drop_back(); + ASSERT_EQ(WRow3[0], 7); + WRow3 = WRow3.drop_back(); + EXPECT_TRUE(WRow3.empty()); + ASSERT_EQ(W.getColSpan(3), 0u); + EXPECT_EQ(W[0][2], 0); + EXPECT_EQ(W[1][2], 0); + EXPECT_EQ(W[2][2], 0); + W = JaggedArrayView{O}; + EXPECT_EQ(W[0][2], 0); + EXPECT_EQ(W[1][2], 0); + EXPECT_EQ(W[2][2], 0); + EXPECT_EQ(W[3][2], 0); +} + +TYPED_TEST(MatrixTest, ResizeAssign) { + auto &M = this->SmallMatrix; + M.resize(3); + ASSERT_EQ(M.getNumRows(), 3u); + JaggedArrayView V{M}; + V[0] = this->getDummyRow(2); + V[1].copy_assign(V[0]); + V[2].copy_assign(V[1]); + ASSERT_EQ(M.getNumRows(), 3u); + ASSERT_EQ(V.getRowSpan(), 3u); + EXPECT_EQ(V[0], V[1]); + EXPECT_EQ(V[2], V[1]); + V = JaggedArrayView{M}; + EXPECT_EQ(V[0], V[1]); + EXPECT_EQ(V[2], V[1]); +} + +TYPED_TEST(MatrixTest, ColSlice) { + auto &M = this->OtherSmall; + JaggedArrayView V{M}; + V[0] = {TypeParam(3), TypeParam(7)}; + V[1] = {TypeParam(4), TypeParam(5)}; + V[2] = {TypeParam(8), TypeParam(9)}; + auto W = V.colSlice(1, 2); + ASSERT_EQ(W.getRowSpan(), 3u); + ASSERT_EQ(W.getColSpan(0), 1u); + EXPECT_EQ(V[0][0], 3); + EXPECT_EQ(V[0][1], 7); + EXPECT_EQ(V[1][0], 4); + EXPECT_EQ(V[1][1], 5); + EXPECT_EQ(V[2][0], 8); + EXPECT_EQ(V[2][1], 9); + EXPECT_EQ(W[0][0], 7); + EXPECT_EQ(W[1][0], 5); + EXPECT_EQ(W[2][0], 9); +} + +TYPED_TEST(MatrixTest, RowColSlice) { + auto &M = this->OtherSmall; + JaggedArrayView V{M}; + V[0] = {TypeParam(3), TypeParam(7)}; + V[2] = {TypeParam(4), TypeParam(5)}; + auto W = V.rowSlice(0, 2).colSlice(0, 2); + ASSERT_EQ(W.getRowSpan(), 2u); + ASSERT_EQ(W.getColSpan(0), 2u); + EXPECT_EQ(V[0][0], 3); + EXPECT_EQ(V[0][1], 7); + EXPECT_EQ(V[1][0], 0); + EXPECT_EQ(V[1][1], 0); + EXPECT_EQ(V[2][0], 4); + EXPECT_EQ(V[2][1], 5); + EXPECT_EQ(W[0][0], 3); + EXPECT_EQ(W[0][1], 7); + EXPECT_EQ(W[1][0], 0); + EXPECT_EQ(W[1][1], 0); +} + +TYPED_TEST(MatrixTest, NonWritingSwap) { + auto &M = this->SmallMatrix; + JaggedArrayView V{M}; + V[0] = {TypeParam(3), TypeParam(7)}; + V[1] = {TypeParam(4), TypeParam(5)}; + V[0].swap(V[1]); + EXPECT_EQ(V.lastRow()[0], 3); + EXPECT_EQ(V.lastRow()[1], 7); + EXPECT_EQ(V[0][0], 4); + EXPECT_EQ(V[0][1], 5); + EXPECT_EQ(V[1][0], 3); + EXPECT_EQ(V[1][1], 7); + auto W = JaggedArrayView{M}; + EXPECT_EQ(W[0][0], 3); + EXPECT_EQ(W[0][1], 7); + EXPECT_EQ(W[1][0], 4); + EXPECT_EQ(W[1][1], 5); +} + +TYPED_TEST(MatrixTest, DropLastRow) { + auto &M = this->OtherSmall; + auto V = JaggedArrayView{M}; + ASSERT_EQ(V.getRowSpan(), 3u); + V.dropLastRow(); + ASSERT_EQ(V.getRowSpan(), 2u); + V[0] = {TypeParam(19), TypeParam(7), TypeParam(3)}; + V[1] = {TypeParam(19), TypeParam(7), TypeParam(3)}; + ASSERT_EQ(V.getColSpan(1), 3u); + V.addRow(V.lastRow()); + ASSERT_EQ(V.getRowSpan(), 3u); + ASSERT_EQ(V.getColSpan(2), 3u); + EXPECT_EQ(V[1], V.lastRow()); + V.dropLastRow(); + ASSERT_EQ(V.getRowSpan(), 2u); + EXPECT_EQ(V[0], V.lastRow()); + V.addRow(this->getDummyRow(3)); + V.dropLastRow(); + V.addRow(this->getDummyRow(3)); + V.dropLastRow(); + ASSERT_EQ(V.getRowSpan(), 2u); + EXPECT_EQ(V[0], V.lastRow()); + V.dropLastRow(); + V.dropLastRow(); + ASSERT_TRUE(V.empty()); + V.addRow({TypeParam(9), TypeParam(7), TypeParam(3)}); + V.addRow(this->getDummyRow(3)); + ASSERT_EQ(V.getRowSpan(), 2u); + EXPECT_EQ(V.lastRow()[0], 1); + EXPECT_EQ(V.lastRow()[1], 2); + EXPECT_EQ(V.lastRow()[2], 3); + EXPECT_EQ(V[0][0], 9); + EXPECT_EQ(V[0][1], 7); + EXPECT_EQ(V[0][2], 3); + V.dropLastRow(); + V.addRow({TypeParam(21), TypeParam(22), TypeParam(23)}); + ASSERT_EQ(V.getRowSpan(), 2u); + EXPECT_EQ(V.lastRow()[0], 21); + EXPECT_EQ(V.lastRow()[1], 22); + EXPECT_EQ(V.lastRow()[2], 23); +} + +TYPED_TEST(MatrixTest, EraseLastRow) { + auto &M = this->SmallMatrix; + JaggedArrayView V{M}; + V[0] = {TypeParam(3), TypeParam(7)}; + V[1] = {TypeParam(4), TypeParam(5)}; + V.eraseLastRow(); + ASSERT_EQ(M.size(), 2u); + ASSERT_EQ(V.getRowSpan(), 1u); + auto W = V.lastRow(); + ASSERT_EQ(W.size(), 2u); + EXPECT_EQ(W[0], 3); + EXPECT_EQ(W[1], 7); + V.addRow({TypeParam(1), TypeParam(2)}); + ASSERT_EQ(V.getRowSpan(), 2u); + V[0].copy_swap(V[1]); + EXPECT_EQ(V[0][0], 1); + EXPECT_EQ(V[0][1], 2); + EXPECT_EQ(V.lastRow()[0], 3); + EXPECT_EQ(V.lastRow()[1], 7); + V.eraseLastRow(); + V.addRow({TypeParam(3), TypeParam(7)}); + ASSERT_EQ(V.getRowSpan(), 2u); + EXPECT_EQ(V[0][0], 1); + EXPECT_EQ(V[0][1], 2); + EXPECT_EQ(V[1][0], 3); + EXPECT_EQ(V[1][1], 7); + V.eraseLastRow(); + ASSERT_EQ(V.getRowSpan(), 1u); + EXPECT_EQ(V[0][0], 1); + EXPECT_EQ(V[0][1], 2); + V.eraseLastRow(); + EXPECT_TRUE(V.empty()); + EXPECT_TRUE(M.empty()); +} + +TYPED_TEST(MatrixTest, Iteration) { + auto &M = this->SmallMatrix; + M.resize(2); + JaggedArrayView V{M}; + for (const auto &[RowIdx, Row] : enumerate(V)) { + V[RowIdx] = this->getDummyRow(2); + for (const auto &[ColIdx, Col] : enumerate(Row)) { + EXPECT_GT(V[RowIdx][ColIdx], 0); + } + } +} + +TYPED_TEST(MatrixTest, VariableLengthColumns) { + auto &M = this->ColumnInitMatrix; + JaggedArrayView V{M}; + ASSERT_EQ(V.empty(), true); + size_t NumCols = 6; + size_t NumRows = 3; + SmallVector ColumnVec; + for (size_t Var = 0; Var < NumCols; ++Var) + ColumnVec.emplace_back(Var + 1); + ASSERT_EQ(ColumnVec.size(), NumCols); + for (size_t RowIdx = 0; RowIdx < NumRows; ++RowIdx) + V.addRow(ColumnVec); + ASSERT_EQ(V.getMaxColSpan(), NumCols); + V.addRow({TypeParam(19)}); + ASSERT_EQ(V.getColSpan(NumRows), 1u); + V.dropLastRow(); + V.addRow(V.lastRow()); + ASSERT_EQ(V.getColSpan(NumRows), NumCols); + EXPECT_EQ(V[NumRows - 1], V[NumRows]); + V.dropLastRow(); + ASSERT_EQ(V.getRowSpan(), NumRows); + ASSERT_EQ(M.getNumCols(), 8u); + ASSERT_EQ(M.getNumRows(), NumRows + 2); + V.dropLastRow(); + ASSERT_EQ(V.getRowSpan(), NumRows - 1); + ASSERT_EQ(M.getNumRows(), NumRows + 2); + ASSERT_EQ(V.getColSpan(1), NumCols); + V[1][0] = 19; + std::swap(V[0], V[1]); + V[0].pop_back(); + V[1] = V[1].drop_back().drop_back(); + ASSERT_EQ(V.getColSpan(0), NumCols - 1); + ASSERT_EQ(V.getColSpan(1), NumCols - 2); + EXPECT_EQ(V[0][0], 19); + EXPECT_EQ(V[1][0], 1); + ASSERT_EQ(V.getMaxColSpan(), NumCols - 1); + V = V.colSlice(0, 1).rowSlice(0, 2); + ASSERT_EQ(V.getColSpan(0), 1u); + ASSERT_EQ(V.getColSpan(1), 1u); + EXPECT_EQ(V.getMaxColSpan(), 1u); + ASSERT_EQ(V.getRowSpan(), 2u); + EXPECT_EQ(V[0][0], 19); + EXPECT_EQ(V[1][0], 1); + V.dropLastRow(); + V.dropLastRow(); + EXPECT_TRUE(V.empty()); + EXPECT_EQ(M.size(), 40u); +} + +TYPED_TEST(MatrixTest, LargeMatrixOps) { + auto &M = this->LargeMatrix; + ASSERT_EQ(M.getNumRows(), 16u); + ASSERT_EQ(M.getNumCols(), 16u); + JaggedArrayView V{M}; + V[0] = {TypeParam(1), TypeParam(2), TypeParam(1), TypeParam(4), + TypeParam(5), TypeParam(1), TypeParam(1), TypeParam(8), + TypeParam(9), TypeParam(1), TypeParam(1), TypeParam(12), + TypeParam(13), TypeParam(1), TypeParam(15), TypeParam(1)}; + V[1] = this->getDummyRow(16); + EXPECT_EQ(V[0][14], 15); + EXPECT_EQ(V[0][3], 4); + EXPECT_EQ(std::count(V[0].begin(), V[0].end(), 1), 8); + EXPECT_TRUE( + std::all_of(V[0].begin(), V[0].end(), [](auto &El) { return El > 0; })); + auto W = V.rowSlice(0, 1).colSlice(2, 4); + ASSERT_EQ(W.getRowSpan(), 1u); + ASSERT_EQ(W.getColSpan(0), 2u); + EXPECT_EQ(W[0][0], 1); + EXPECT_EQ(W[0][1], 4); +}