Merge branch 'master' into SarkarBufferMerges

Author: raphael-s-steiner

apps/CMakeLists.txt

@@ -50,6 +50,7 @@ endif()
 
 if (COPT_FOUND)
 _add_executable( ilp_bsp_scheduler )
+_add_executable( ilp_hypergraph_partitioner )
 endif()
 
 endif()

apps/ilp_hypergraph_partitioner.cpp

@@ -0,0 +1,165 @@
+/*
+Copyright 2024 Huawei Technologies Co., Ltd.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+@author Toni Boehnlein, Benjamin Lozes, Pal Andras Papp, Raphael S. Steiner
+*/
+
+#include <filesystem>
+#include <fstream>
+#include <iostream>
+#include <set>
+#include <string>
+#include <vector>
+
+#include "osp/auxiliary/misc.hpp"
+#include "osp/graph_algorithms/directed_graph_path_util.hpp"
+#include "osp/auxiliary/io/general_file_reader.hpp"
+#include "osp/partitioning/model/hypergraph_utility.hpp"
+#include "osp/partitioning/partitioners/generic_FM.hpp"
+#include "osp/partitioning/partitioners/partitioning_ILP.hpp"
+#include "osp/partitioning/partitioners/partitioning_ILP_replication.hpp"
+#include "osp/graph_implementations/adj_list_impl/computational_dag_vector_impl.hpp"
+#include "osp/auxiliary/io/hdag_graph_file_reader.hpp"
+#include "osp/auxiliary/io/mtx_hypergraph_file_reader.hpp"
+#include "osp/auxiliary/io/partitioning_file_writer.hpp"
+    
+
+using namespace osp;
+
+using graph = computational_dag_vector_impl_def_int_t;
+using hypergraph = Hypergraph_def_t;
+
+int main(int argc, char *argv[]) {
+    if (argc < 4) {
+        std::cerr << "Usage: " << argv[0] << " <input_file> <nr_parts> <imbalance> <optional:part_repl|full_repl>"
+                  << std::endl;
+        return 1;
+    }
+
+    std::string filename_hgraph = argv[1];
+    std::string name_hgraph = filename_hgraph.substr(0, filename_hgraph.rfind("."));
+    std::string file_ending = filename_hgraph.substr(filename_hgraph.rfind(".") + 1);
+    if (!file_reader::isPathSafe(filename_hgraph)) {
+        std::cerr << "Error: Unsafe file path (possible traversal or invalid type).\n";
+        return 1;
+    }
+
+    std::cout << name_hgraph << std::endl;
+
+    int nr_parts = std::stoi(argv[2]);
+    if (nr_parts < 2 || nr_parts > 32) {
+        std::cerr << "Argument nr_parts must be an integer between 2 and 32: " << nr_parts << std::endl;
+        return 1;
+    }
+
+    float imbalance = std::stof(argv[3]);
+    if (imbalance < 0.01 || imbalance > .99) {
+        std::cerr << "Argument imbalance must be a float between 0.01 and 0.99: " << imbalance << std::endl;
+        return 1;
+    }
+
+    unsigned replicate = 0;
+
+    if (argc > 4 && std::string(argv[4]) == "part_repl") {
+        replicate = 1;
+    } else if (argc > 4 && std::string(argv[4]) == "full_repl") {
+        replicate = 2;
+    } else if (argc > 4) {
+        std::cerr << "Unknown argument: " << argv[4] << ". Expected 'part_repl' or 'full_repl' for replication." << std::endl;
+        return 1;
+    }
+
+    PartitioningProblem<hypergraph> instance;
+
+    bool file_status = true;    
+    if (file_ending == "hdag") {
+        graph dag;
+        file_status = file_reader::readComputationalDagHyperdagFormatDB(filename_hgraph, dag);
+        if(file_status)
+            instance.getHypergraph() = convert_from_cdag_as_hyperdag<hypergraph, graph>(dag);
+    } else if (file_ending == "mtx") {
+        file_status = file_reader::readHypergraphMartixMarketFormat(filename_hgraph, instance.getHypergraph());
+    } else {
+        std::cout << "Unknown file extension." << std::endl;
+        return 1;
+    }
+    if (!file_status) {
+
+        std::cout << "Reading input file failed." << std::endl;
+        return 1;
+    }
+
+    instance.setNumberOfPartitions(static_cast<unsigned>(nr_parts));
+    instance.setMaxWorkWeightViaImbalanceFactor(imbalance);
+
+    Partitioning<hypergraph> initial_partition(instance);
+    GenericFM<hypergraph> fm;
+    for(size_t node = 0; node < instance.getHypergraph().num_vertices(); ++node)
+        initial_partition.setAssignedPartition(node, static_cast<unsigned>(node % static_cast<size_t>(nr_parts)));
+    if(nr_parts == 2)
+        fm.ImprovePartitioning(initial_partition);
+    if(nr_parts == 4 || nr_parts == 8 || nr_parts == 16 || nr_parts == 32)
+        fm.RecursiveFM(initial_partition);
+
+    if (replicate > 0) {
+
+        PartitioningWithReplication<hypergraph> partition(instance);
+        HypergraphPartitioningILPWithReplication<hypergraph> partitioner;
+
+        for(size_t node = 0; node < instance.getHypergraph().num_vertices(); ++node)
+            partition.setAssignedPartitions(node, {initial_partition.assignedPartition(node)});
+        if(partition.satisfiesBalanceConstraint())
+            partitioner.setUseInitialSolution(true);
+
+        partitioner.setTimeLimitSeconds(600);
+        if(replicate == 2)
+            partitioner.setReplicationModel(HypergraphPartitioningILPWithReplication<hypergraph>::REPLICATION_MODEL_IN_ILP::GENERAL);
+
+        auto solve_status = partitioner.computePartitioning(partition);
+
+        if (solve_status == RETURN_STATUS::OSP_SUCCESS || solve_status == RETURN_STATUS::BEST_FOUND) {
+            file_writer::write_txt(name_hgraph + "_" + std::to_string(nr_parts) + "_" + std::to_string(imbalance) +
+                "_ILP_rep" + std::to_string(replicate) + ".txt", partition);
+            std::cout << "Partitioning (with replicaiton) computed with costs: " << partition.computeConnectivityCost() << std::endl;
+        } else {
+            std::cout << "Computing partition failed." << std::endl;
+            return 1;
+        }
+
+    } else {
+
+        Partitioning<hypergraph> partition(instance);
+        HypergraphPartitioningILP<hypergraph> partitioner;
+
+        for(size_t node = 0; node < instance.getHypergraph().num_vertices(); ++node)
+            partition.setAssignedPartition(node, initial_partition.assignedPartition(node));
+        if(partition.satisfiesBalanceConstraint())
+            partitioner.setUseInitialSolution(true);
+
+        partitioner.setTimeLimitSeconds(600);
+
+        auto solve_status = partitioner.computePartitioning(partition);
+
+        if (solve_status == RETURN_STATUS::OSP_SUCCESS || solve_status == RETURN_STATUS::BEST_FOUND) {
+            file_writer::write_txt(name_hgraph + "_" + std::to_string(nr_parts) + "_" + std::to_string(imbalance) +
+                "_ILP_rep" + std::to_string(replicate) + ".txt", partition);
+            std::cout << "Partitioning computed with costs: " << partition.computeConnectivityCost() << std::endl;
+        } else {
+            std::cout << "Computing partition failed." << std::endl;
+            return 1;
+        }
+    }
+    return 0;
+}

include/osp/auxiliary/hash_util.hpp

@@ -28,6 +28,15 @@ struct uniform_node_hash_func {
     result_type operator()(const VertexType& ) { return defautlVal; }
 };
 
+
+template<typename VertexType>
+struct vector_node_hash_func {
+    const std::vector<std::size_t>& node_hashes_;
+    vector_node_hash_func(const std::vector<std::size_t>& node_hashes) : node_hashes_(node_hashes) {}
+    using result_type = std::size_t;
+    result_type operator()(const VertexType& v) const { return node_hashes_[v]; }
+};
+
 template<class T>
 void hash_combine(std::size_t &seed, const T &v) {
     std::hash<T> hasher;

include/osp/auxiliary/io/hdag_graph_file_reader.hpp

@@ -252,12 +252,17 @@ bool readComputationalDagHyperdagFormatDB(std::ifstream& infile, Graph_t& graph)
             graph.set_vertex_comm_weight(static_cast<vertex_idx_t<Graph_t>>(node), hyperedge_comm_weights[edgeIdx]);
             graph.set_vertex_mem_weight(static_cast<vertex_idx_t<Graph_t>>(node), hyperedge_mem_weights[edgeIdx]);
         } else {
-            auto edge = graph.add_edge(static_cast<vertex_idx_t<Graph_t>>(edgeSource[edgeIdx]),
+            if constexpr (is_modifiable_cdag_comm_edge_v<Graph_t>) {
+
+                auto edge = graph.add_edge(static_cast<vertex_idx_t<Graph_t>>(edgeSource[edgeIdx]),
                                     static_cast<vertex_idx_t<Graph_t>>(nodeIdx));
 
-            if constexpr (is_modifiable_cdag_comm_edge_v<Graph_t>) {
                 graph.set_edge_comm_weight(edge.first,
                     static_cast<e_commw_t<Graph_t>>(hyperedge_comm_weights[edgeIdx]));
+
+            } else {
+                graph.add_edge(static_cast<vertex_idx_t<Graph_t>>(edgeSource[edgeIdx]),
+                                    static_cast<vertex_idx_t<Graph_t>>(nodeIdx));
             }
         }
     }

include/osp/auxiliary/io/mtx_graph_file_reader.hpp

@@ -69,12 +69,12 @@ bool readComputationalDagMartixMarketFormat(std::ifstream& infile, Graph_t& grap
         return false;
     }
 
-    const vertex_t num_nodes = static_cast<vertex_t>(M_row);
-    if (num_nodes > std::numeric_limits<vertex_t>::max()) {
+    if (static_cast<unsigned long long>(M_row) > std::numeric_limits<vertex_t>::max()) {
         std::cerr << "Error: Matrix dimension too large for vertex type.\n";
         return false;
     }
 
+    const vertex_t num_nodes = static_cast<vertex_t>(M_row);
     std::vector<int> node_work_wts(num_nodes, 0);
     std::vector<int> node_comm_wts(num_nodes, 1);