Fix UB in f2reduce for cols == 0 (#5401)

The shift exponent in `uint64_t final_b = (1ull << (cols - 1)) - 1;`
underflows for `cols == 0`. We can skip converting to rref for `cols <=
1` as well to avoid the issue.

Author: chsigg

lib/Tools/LinearLayout.cpp

@@ -117,11 +117,6 @@ std::unique_ptr<uint64_t[]> getMatrix(const LinearLayout &layout) {
 // outDim as columns.  In other words, finds the number of linearly-independent
 // bases for this output dimension.
 int getMatrixRank(std::unique_ptr<uint64_t[]> m, int numRows, int numCols) {
-  // f2reduce underflows if the number of cols is 0, return the rank early in
-  // this case.
-  if (numCols == 0) {
-    return 0;
-  }
   // stride is specified in number of 64-bit words per row, and we pack our
   // matrix so that there's only one uint64_t per row.
   assert(numCols <= 64);

third_party/f2reduce/f2reduce.cpp

@@ -470,8 +470,8 @@ namespace f2reduce {
 
 void inplace_rref_strided(uint64_t *matrix, uint64_t rows, uint64_t cols, uint64_t stride) {
 
-    if (rows <= 1) {
-        // If the matrix has 0 or 1 rows, it must already be in RREF:
+    if (rows <= 1 || cols == 0) {
+        // If the matrix has 0 or 1 rows or 0 columns, it must already be in RREF:
         return;
     }