Modified tree parallelism to use partition of the elimination tree

Author: filikat

highs/ipm/hipo/factorhighs/Analyse.cpp

@@ -1474,13 +1474,27 @@ void Analyse::generateParallelLayer(Int threads) {
   // generate info about subtrees in the layer
   std::vector<Int> first_desc;
   firstDescendant(sn_parent_, first_desc);
-  layerSubtrees_.resize(layerIndex_.size());
+  layerSubtreesInfo_.resize(layerIndex_.size());
   for (auto& subtree : layerIndex_) {
     Int node = subtree.first;
     Int index = subtree.second;
-    layerSubtrees_[index].start = first_desc[node];
-    layerSubtrees_[index].end = node + 1;
-    layerSubtrees_[index].stack = stack_subtree_parallel_[node];
+    layerSubtreesInfo_[index].start = first_desc[node];
+    layerSubtreesInfo_[index].end = node + 1;
+    layerSubtreesInfo_[index].stack = stack_subtree_parallel_[node];
+  }
+
+  smallSubtreesInfo_.resize(smallSubtrees_.size());
+  Int index = 0;
+  for (auto it = smallSubtrees_.begin(); it != smallSubtrees_.end(); ++it) {
+    Int node = *it;
+
+    smallSubtreesInfo_[index].start = first_desc[node];
+    smallSubtreesInfo_[index].end = node + 1;
+
+    // no stack needed for small subtrees
+    smallSubtreesInfo_[index].stack = -1;
+
+    ++index;
   }
 }
 
@@ -1617,7 +1631,8 @@ Int Analyse::run(Symbolic& S) {
   S.consecutive_sums_ = std::move(consecutive_sums_);
   S.clique_block_start_ = std::move(clique_block_start_);
   S.layerIndex_ = std::move(layerIndex_);
-  S.layerSubtrees_ = std::move(layerSubtrees_);
+  S.layerSubtreesInfo_ = std::move(layerSubtreesInfo_);
+  S.smallSubtreesInfo_ = std::move(smallSubtreesInfo_);
   S.aboveLayer_ = std::move(aboveLayer_);
   S.smallSubtrees_ = std::move(smallSubtrees_);
 

highs/ipm/hipo/factorhighs/Analyse.h

@@ -80,7 +80,7 @@ class Analyse {
 
   // Parallel info
   std::map<Int, Int> layerIndex_;
-  std::vector<SubtreeInfo> layerSubtrees_;
+  std::vector<SubtreeInfo> layerSubtreesInfo_, smallSubtreesInfo_;
   std::set<Int> aboveLayer_, smallSubtrees_;
   std::vector<int64_t> stack_subtree_serial_;
   std::vector<int64_t> stack_subtree_parallel_;

highs/ipm/hipo/factorhighs/Factorise.cpp

@@ -58,13 +58,6 @@ Factorise::Factorise(const Symbolic& S, const std::vector<Int>& rowsA,
   // create linked lists of children in supernodal elimination tree
   childrenLinkedList(S_.snParent(), first_child_, next_child_);
 
-  if (S_.parTree()) {
-    // create reverse linked lists of children
-    first_child_reverse_ = first_child_;
-    next_child_reverse_ = next_child_;
-    reverseLinkedList(first_child_reverse_, next_child_reverse_);
-  }
-
   // compute largest diagonal entry in absolute value
   max_diag_ = 0.0;
   min_diag_ = kHighsInf;
@@ -174,19 +167,6 @@ void Factorise::processSupernode(Int sn) {
 
   if (flag_stop_) return;
 
-  if (S_.parTree()) {
-    // spawn children of this supernode in reverse order
-    Int child_to_spawn = first_child_reverse_[sn];
-    while (child_to_spawn != -1) {
-      highs::parallel::spawn([=]() { processSupernode(child_to_spawn); });
-      child_to_spawn = next_child_reverse_[child_to_spawn];
-    }
-
-    // wait for first child to finish, before starting the parent (if there is a
-    // first child)
-    if (first_child_reverse_[sn] != -1) highs::parallel::sync();
-  }
-
 #if HIPO_TIMING_LEVEL >= 2
   Clock clock;
 #endif
@@ -238,17 +218,10 @@ void Factorise::processSupernode(Int sn) {
     // Schur contribution of the current child
     std::vector<double>& child_clique = schur_contribution_[child_sn];
 
-    if (S_.parTree()) {
-      // sync with spawned child, apart from the first one
-      if (child_sn != first_child_[sn]) highs::parallel::sync();
-
-      if (flag_stop_) return;
-
-      if (child_clique.size() == 0) {
-        if (log_) log_->printDevInfo("Missing child supernode contribution\n");
-        flag_stop_ = true;
-        return;
-      }
+    if (child_clique.size() == 0) {
+      if (log_) log_->printDevInfo("Missing child supernode contribution\n");
+      flag_stop_ = true;
+      return;
     }
 
     // determine size of clique of child
@@ -351,6 +324,12 @@ void Factorise::processSupernode(Int sn) {
 #endif
 }
 
+void Factorise::processSupernodes(Int start, Int end) {
+  for (Int sn = start; sn < end; ++sn) {
+    processSupernode(sn);
+  }
+}
+
 bool Factorise::run(Numeric& num) {
 #if HIPO_TIMING_LEVEL >= 1
   Clock clock;
@@ -368,24 +347,33 @@ bool Factorise::run(Numeric& num) {
   sn_columns_.resize(S_.sn());
 
   if (S_.parTree()) {
-    Int spawned_roots{};
-    // spawn tasks for root supernodes
-    for (Int sn = 0; sn < S_.sn(); ++sn) {
-      if (S_.snParent(sn) == -1) {
-        highs::parallel::spawn([=]() { processSupernode(sn); });
-        ++spawned_roots;
-      }
+    // process subtrees in the layer
+    for (Int i = 0; i < S_.layerIndex().size(); ++i) {
+      Int start = S_.layerSubtreeInfo(i).start;
+      Int end = S_.layerSubtreeInfo(i).end;
+      highs::parallel::spawn([=]() { processSupernodes(start, end); });
+    }
+
+    // process small subtrees
+    for (Int i = 0; i < S_.smallSubtrees().size(); ++i) {
+      Int start = S_.smallSubtreeInfo(i).start;
+      Int end = S_.smallSubtreeInfo(i).end;
+      processSupernodes(start, end);
     }
 
-    // sync tasks for root supernodes
-    for (Int root = 0; root < spawned_roots; ++root) {
+    // wait for subtrees in the layer to complete
+    for (Int i = 0; i < S_.layerIndex().size(); ++i) {
       highs::parallel::sync();
     }
+
+    // process nodes above layer
+    for (auto it = S_.aboveLayer().begin(); it != S_.aboveLayer().end(); ++it) {
+      processSupernode(*it);
+    }
+
   } else {
     // go through each supernode serially
-    for (Int sn = 0; sn < S_.sn(); ++sn) {
-      processSupernode(sn);
-    }
+    processSupernodes(0, S_.sn());
   }
 
   if (flag_stop_) return true;

highs/ipm/hipo/factorhighs/Factorise.h

@@ -26,10 +26,6 @@ class Factorise {
   std::vector<Int> first_child_{};
   std::vector<Int> next_child_{};
 
-  // reverse linked lists of chidlren
-  std::vector<Int> first_child_reverse_{};
-  std::vector<Int> next_child_reverse_{};
-
   // generated elements, aka Schur complements.
   std::vector<std::vector<double>> schur_contribution_{};
 
@@ -69,6 +65,7 @@ class Factorise {
  public:
   void permute(const std::vector<Int>& iperm);
   void processSupernode(Int sn);
+  void processSupernodes(Int start, Int end);
 
  public:
   Factorise(const Symbolic& S, const std::vector<Int>& rowsA,

highs/ipm/hipo/factorhighs/HybridHybridFormatHandler.cpp

@@ -154,6 +154,7 @@ void HybridHybridFormatHandler::assembleClique(const std::vector<double>& child,
 }
 
 void HybridHybridFormatHandler::extremeEntries() {
+#ifdef HIPO_COLLECT_EXPENSIVE_DATA
   double minD = 1e100;
   double maxD = 0.0;
   double minoffD = 1e100;
@@ -199,6 +200,7 @@ void HybridHybridFormatHandler::extremeEntries() {
   }
 
   data_.setExtremeEntries(minD, maxD, minoffD, maxoffD);
+#endif
 }
 
 }  // namespace hipo

highs/ipm/hipo/factorhighs/Symbolic.cpp

@@ -44,6 +44,15 @@ const std::vector<Int>& Symbolic::iperm() const { return iperm_; }
 const std::vector<Int>& Symbolic::snParent() const { return sn_parent_; }
 const std::vector<Int>& Symbolic::snStart() const { return sn_start_; }
 const std::vector<Int>& Symbolic::pivotSign() const { return pivot_sign_; }
+const SubtreeInfo& Symbolic::layerSubtreeInfo(Int i) const {
+  return layerSubtreesInfo_[i];
+}
+const SubtreeInfo& Symbolic::smallSubtreeInfo(Int i) const {
+  return smallSubtreesInfo_[i];
+}
+const std::set<Int>& Symbolic::aboveLayer() const { return aboveLayer_; }
+const std::set<Int>& Symbolic::smallSubtrees() const { return smallSubtrees_; }
+const std::map<Int, Int>& Symbolic::layerIndex() const { return layerIndex_; }
 
 std::string memoryString(double mem) {
   std::stringstream ss;
@@ -68,10 +77,10 @@ void Symbolic::print(const Log& log, bool verbose) const {
   log_stream << textline("Fill-in:") << fix(fillin_, 0, 2) << '\n';
   log_stream << textline("Flops:") << sci(flops_, 0, 2) << '\n';
   if (verbose) {
-    log_stream << textline("serial stack entries:") << sci(serial_stack_size_, 0, 1)
-               << '\n';
-    log_stream << textline("parallel stacks entries:") << sci(parallel_stack_size_, 0, 1)
-               << '\n';
+    log_stream << textline("serial stack entries:")
+               << sci(serial_stack_size_, 0, 1) << '\n';
+    log_stream << textline("parallel stacks entries:")
+               << sci(parallel_stack_size_, 0, 1) << '\n';
     log_stream << textline("factor entries:")
                << sci(factors_total_entries_, 0, 1) << '\n';
     log_stream << textline("serial memory:")

highs/ipm/hipo/factorhighs/Symbolic.h

@@ -2,8 +2,8 @@
 #define FACTORHIGHS_SYMBOLIC_H
 
 #include <map>
-#include <vector>
 #include <set>
+#include <vector>
 
 #include "ipm/hipo/auxiliary/IntConfig.h"
 #include "ipm/hipo/auxiliary/Log.h"
@@ -107,13 +107,13 @@ class Symbolic {
   std::map<Int, Int> layerIndex_;
 
   // Information about subtrees in the layer
-  // - layerSubtrees_[j] contains the following information about the j-th
-  //   subtree
-  //   in the layer (according to the numbering in layerIndex_):
+  // - layerSubtreesInfo_[j] contains the following information about the j-th
+  //   subtree in the layer (according to the numbering in layerIndex_):
   //   . start: first node in the subtree
   //   . end: first later node not in the subtree
   //   . stack: minimum amount of stack space required to process the subtree
-  std::vector<SubtreeInfo> layerSubtrees_;
+  std::vector<SubtreeInfo> layerSubtreesInfo_;
+  std::vector<SubtreeInfo> smallSubtreesInfo_;
 
   // Set containing the nodes that appear above the parallel layer
   std::set<Int> aboveLayer_;
@@ -156,6 +156,11 @@ class Symbolic {
   const std::vector<Int>& snParent() const;
   const std::vector<Int>& snStart() const;
   const std::vector<Int>& pivotSign() const;
+  const SubtreeInfo& layerSubtreeInfo(Int i) const;
+  const SubtreeInfo& smallSubtreeInfo(Int i) const;
+  const std::set<Int>& aboveLayer() const;
+  const std::set<Int>& smallSubtrees() const;
+  const std::map<Int, Int>& layerIndex() const;
 
   void print(const Log& log, bool verbose = false) const;
 };