Hypergraph partitioning enters OSP

apps/CMakeLists.txt

@@ -50,6 +50,7 @@ endif()
 
 if (COPT_FOUND)
 _add_executable( ilp_bsp_scheduler )
+_add_executable( ilp_hypergraph_partitioner )
 endif()
 
 endif()

apps/ilp_hypergraph_partitioner.cpp

@@ -0,0 +1,165 @@
+/*
+Copyright 2024 Huawei Technologies Co., Ltd.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+@author Toni Boehnlein, Benjamin Lozes, Pal Andras Papp, Raphael S. Steiner
+*/
+
+#include <filesystem>
+#include <fstream>
+#include <iostream>
+#include <set>
+#include <string>
+#include <vector>
+
+#include "osp/auxiliary/misc.hpp"
+#include "osp/graph_algorithms/directed_graph_path_util.hpp"
+#include "osp/auxiliary/io/general_file_reader.hpp"
+#include "osp/partitioning/model/hypergraph_utility.hpp"
+#include "osp/partitioning/partitioners/generic_FM.hpp"
+#include "osp/partitioning/partitioners/partitioning_ILP.hpp"
+#include "osp/partitioning/partitioners/partitioning_ILP_replication.hpp"
+#include "osp/graph_implementations/adj_list_impl/computational_dag_vector_impl.hpp"
+#include "osp/auxiliary/io/hdag_graph_file_reader.hpp"
+#include "osp/auxiliary/io/mtx_hypergraph_file_reader.hpp"
+#include "osp/auxiliary/io/partitioning_file_writer.hpp"
+
+
+using namespace osp;
+
+using graph = computational_dag_vector_impl_def_int_t;
+using hypergraph = Hypergraph_def_t;
+
+int main(int argc, char *argv[]) {
+    if (argc < 4) {
+        std::cerr << "Usage: " << argv[0] << " <input_file> <nr_parts> <imbalance> <optional:part_repl|full_repl>"
+                  << std::endl;
+        return 1;
+    }
+
+    std::string filename_hgraph = argv[1];
+    std::string name_hgraph = filename_hgraph.substr(0, filename_hgraph.rfind("."));
+    std::string file_ending = filename_hgraph.substr(filename_hgraph.rfind(".") + 1);
+    if (!file_reader::isPathSafe(filename_hgraph)) {
+        std::cerr << "Error: Unsafe file path (possible traversal or invalid type).\n";
+        return 1;
+    }
+
+    std::cout << name_hgraph << std::endl;
+
+    int nr_parts = std::stoi(argv[2]);
+    if (nr_parts < 2 || nr_parts > 32) {
+        std::cerr << "Argument nr_parts must be an integer between 2 and 32: " << nr_parts << std::endl;
+        return 1;
+    }
+
+    float imbalance = std::stof(argv[3]);
+    if (imbalance < 0.01 || imbalance > .99) {
+        std::cerr << "Argument imbalance must be a float between 0.01 and 0.99: " << imbalance << std::endl;
+        return 1;
+    }
+
+    unsigned replicate = 0;
+
+    if (argc > 4 && std::string(argv[4]) == "part_repl") {
+        replicate = 1;
+    } else if (argc > 4 && std::string(argv[4]) == "full_repl") {
+        replicate = 2;
+    } else if (argc > 4) {
+        std::cerr << "Unknown argument: " << argv[4] << ". Expected 'part_repl' or 'full_repl' for replication." << std::endl;
+        return 1;
+    }
+
+    PartitioningProblem<hypergraph> instance;
+
+    bool file_status = true;    
+    if (file_ending == "hdag") {
+        graph dag;
+        file_status = file_reader::readComputationalDagHyperdagFormatDB(filename_hgraph, dag);
+        if(file_status)
+            instance.getHypergraph() = convert_from_cdag_as_hyperdag<hypergraph, graph>(dag);
+    } else if (file_ending == "mtx") {
+        file_status = file_reader::readHypergraphMartixMarketFormat(filename_hgraph, instance.getHypergraph());
+    } else {
+        std::cout << "Unknown file extension." << std::endl;
+        return 1;
+    }
+    if (!file_status) {
+
+        std::cout << "Reading input file failed." << std::endl;
+        return 1;
+    }
+
+    instance.setNumberOfPartitions(static_cast<unsigned>(nr_parts));
+    instance.setMaxWorkWeightViaImbalanceFactor(imbalance);
+
+    Partitioning<hypergraph> initial_partition(instance);
+    GenericFM<hypergraph> fm;
+    for(size_t node = 0; node < instance.getHypergraph().num_vertices(); ++node)
+        initial_partition.setAssignedPartition(node, static_cast<unsigned>(node % static_cast<size_t>(nr_parts)));
+    if(nr_parts == 2)
+        fm.ImprovePartitioning(initial_partition);
+    if(nr_parts == 4 || nr_parts == 8 || nr_parts == 16 || nr_parts == 32)
+        fm.RecursiveFM(initial_partition);
+
+    if (replicate > 0) {
+
+        PartitioningWithReplication<hypergraph> partition(instance);
+        HypergraphPartitioningILPWithReplication<hypergraph> partitioner;
+
+        for(size_t node = 0; node < instance.getHypergraph().num_vertices(); ++node)
+            partition.setAssignedPartitions(node, {initial_partition.assignedPartition(node)});
+        if(partition.satisfiesBalanceConstraint())
+            partitioner.setUseInitialSolution(true);
+
+        partitioner.setTimeLimitSeconds(600);
+        if(replicate == 2)
+            partitioner.setReplicationModel(HypergraphPartitioningILPWithReplication<hypergraph>::REPLICATION_MODEL_IN_ILP::GENERAL);
+
+        auto solve_status = partitioner.computePartitioning(partition);
+
+        if (solve_status == RETURN_STATUS::OSP_SUCCESS || solve_status == RETURN_STATUS::BEST_FOUND) {
+            file_writer::write_txt(name_hgraph + "_" + std::to_string(nr_parts) + "_" + std::to_string(imbalance) +
+                "_ILP_rep" + std::to_string(replicate) + ".txt", partition);
+            std::cout << "Partitioning (with replicaiton) computed with costs: " << partition.computeConnectivityCost() << std::endl;
+        } else {
+            std::cout << "Computing partition failed." << std::endl;
+            return 1;
+        }
+
+    } else {
+
+        Partitioning<hypergraph> partition(instance);
+        HypergraphPartitioningILP<hypergraph> partitioner;
+
+        for(size_t node = 0; node < instance.getHypergraph().num_vertices(); ++node)
+            partition.setAssignedPartition(node, initial_partition.assignedPartition(node));
+        if(partition.satisfiesBalanceConstraint())
+            partitioner.setUseInitialSolution(true);
+
+        partitioner.setTimeLimitSeconds(600);
+
+        auto solve_status = partitioner.computePartitioning(partition);
+
+        if (solve_status == RETURN_STATUS::OSP_SUCCESS || solve_status == RETURN_STATUS::BEST_FOUND) {
+            file_writer::write_txt(name_hgraph + "_" + std::to_string(nr_parts) + "_" + std::to_string(imbalance) +
+                "_ILP_rep" + std::to_string(replicate) + ".txt", partition);
+            std::cout << "Partitioning computed with costs: " << partition.computeConnectivityCost() << std::endl;
+        } else {
+            std::cout << "Computing partition failed." << std::endl;
+            return 1;
+        }
+    }
+    return 0;
+}

include/osp/auxiliary/io/mtx_hypergraph_file_reader.hpp

@@ -0,0 +1,175 @@
+/*
+Copyright 2024 Huawei Technologies Co., Ltd.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+@author Toni Boehnlein, Christos Matzoros, Benjamin Lozes, Pal Andras Papp, Raphael S. Steiner
+*/
+
+#pragma once
+
+#include <fstream>
+#include <iostream>
+#include <sstream>
+#include <string>
+#include <vector>
+#include <limits>
+#include <filesystem>
+
+#include "osp/partitioning/model/hypergraph.hpp"
+#include "osp/auxiliary/io/filepath_checker.hpp"
+
+namespace osp {
+namespace file_reader {
+
+// reads a matrix into Hypergraph format, where nonzeros are vertices, and rows/columns are hyperedges
+template<typename index_type, typename workw_type, typename memw_type, typename commw_type>
+bool readHypergraphMartixMarketFormat(std::ifstream& infile, Hypergraph<index_type, workw_type, memw_type, commw_type>& hgraph) {
+
+    std::string line;
+
+    // Skip comments or empty lines (robustly)
+    while (std::getline(infile, line)) {
+        if (line.empty() || line[0] == '%') continue;
+
+        // Null byte check
+        if (line.find('\0') != std::string::npos) {
+            std::cerr << "Error: Null byte detected in header line.\n";
+            return false;
+        }
+
+        if (line.size() > MAX_LINE_LENGTH) {
+            std::cerr << "Error: Line too long, possible malformed or malicious file.\n";
+            return false;
+        }
+        break; // We found the actual header line
+    }
+
+    if (infile.eof()) {
+        std::cerr << "Error: Unexpected end of file while reading header.\n";
+        return false;
+    }
+
+    int M_row = 0, M_col = 0, nEntries = 0;
+
+    std::istringstream header_stream(line);
+    if (!(header_stream >> M_row >> M_col >> nEntries) ||
+        M_row <= 0 || M_col <= 0) {
+        std::cerr << "Error: Invalid header.\n";
+        return false;
+    }
+
+    const index_type num_nodes = static_cast<index_type>(nEntries);
+
+    hgraph.reset(num_nodes, 0);
+    for (index_type node = 0; node < num_nodes; ++node) {
+        hgraph.set_vertex_work_weight(node, static_cast<workw_type>(1));
+        hgraph.set_vertex_memory_weight(node, static_cast<memw_type>(1));
+    }
+
+    std::vector<std::vector<index_type>> row_hyperedges(static_cast<index_type>(M_row));
+    std::vector<std::vector<index_type>> column_hyperedges(static_cast<index_type>(M_col));
+
+    int entries_read = 0;
+    while (entries_read < nEntries && std::getline(infile, line)) {
+        if (line.empty() || line[0] == '%') continue;
+        if (line.size() > MAX_LINE_LENGTH) {
+            std::cerr << "Error: Line too long.\n";
+            return false;
+        }
+
+        std::istringstream entry_stream(line);
+        int row = -1, col = -1;
+        double val = 0.0;
+
+        if (!(entry_stream >> row >> col >> val)) {
+            std::cerr << "Error: Malformed matrix entry.\n";
+            return false;
+        }
+
+        row -= 1; col -= 1; // Convert to 0-based
+
+        if (row < 0 || col < 0 || row >= M_row || col >= M_col) {
+            std::cerr << "Error: Matrix entry out of bounds.\n";
+            return false;
+        }
+
+        if (static_cast<index_type>(row) >= num_nodes || static_cast<index_type>(col) >= num_nodes) {
+            std::cerr << "Error: Index exceeds vertex type limit.\n";
+            return false;
+        }
+
+        row_hyperedges[static_cast<index_type>(row)].push_back(static_cast<index_type>(entries_read));
+        column_hyperedges[static_cast<index_type>(col)].push_back(static_cast<index_type>(entries_read));
+
+        ++entries_read;
+    }
+
+    if (entries_read != nEntries) {
+        std::cerr << "Error: Incomplete matrix entries.\n";
+        return false;
+    }
+
+    while (std::getline(infile, line)) {
+        if (!line.empty() && line[0] != '%') {
+            std::cerr << "Error: Extra data after matrix content.\n";
+            return false;
+        }
+    }
+
+    for(index_type row = 0; row < static_cast<index_type>(M_row); ++row)
+        if(!row_hyperedges[row].empty())
+            hgraph.add_hyperedge(row_hyperedges[row]);
+
+    for(index_type col = 0; col < static_cast<index_type>(M_col); ++col)
+        if(!column_hyperedges[col].empty())
+            hgraph.add_hyperedge(column_hyperedges[col]);
+
+    return true;
+}
+
+template<typename index_type, typename workw_type, typename memw_type, typename commw_type>
+bool readHypergraphMartixMarketFormat(const std::string& filename, Hypergraph<index_type, workw_type, memw_type, commw_type>& hgraph) {
+    // Ensure the file is .mtx format
+    if (std::filesystem::path(filename).extension() != ".mtx") {
+        std::cerr << "Error: Only .mtx files are accepted.\n";
+        return false;
+    }
+
+    if (!isPathSafe(filename)) {
+        std::cerr << "Error: Unsafe file path (potential traversal attack).\n";
+        return false;
+    }
+
+    if (std::filesystem::is_symlink(filename)) {
+        std::cerr << "Error: Symbolic links are not allowed.\n";
+        return false;
+    }
+
+    if (!std::filesystem::is_regular_file(filename)) {
+        std::cerr << "Error: Input is not a regular file.\n";
+        return false;
+    }
+
+    std::ifstream infile(filename);
+    if (!infile.is_open()) {
+        std::cerr << "Error: Failed to open file.\n";
+        return false;
+    }
+
+    return readHypergraphMartixMarketFormat(infile, hgraph);
+}
+
+} // namespace FileReader
+
+} // namespace osp