Skip to content

Commit 9a42108

Browse files
papp-pal-andrastonibohnlein
papp-pal-andras
and
tonibohnlein
authored
Hypergraph partitioning enters OSP (#16)
* hypergraph first draft * hypergraphs and partitioning, next version * two kinds of vertex weights on hypergraphs * templating the partitioning part * hypergraph-partitioning-ILP app added * FM fixes and recursive FM * initialization in Hgraph partitioning app * moving hypergraph utility functions into separate file * added hyperdag template arguments --------- Co-authored-by: tonibohnlein <[email protected]>
1 parent 6ee2a23 commit 9a42108

16 files changed

+2525
-0
lines changed

‎apps/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ endif()
5050

5151
if (COPT_FOUND)
5252
_add_executable( ilp_bsp_scheduler )
53+
_add_executable( ilp_hypergraph_partitioner )
5354
endif()
5455

5556
endif()

‎apps/ilp_hypergraph_partitioner.cpp‎

Lines changed: 165 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,165 @@
1+
/*
2+
Copyright 2024 Huawei Technologies Co., Ltd.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
16+
@author Toni Boehnlein, Benjamin Lozes, Pal Andras Papp, Raphael S. Steiner
17+
*/
18+
19+
#include <filesystem>
20+
#include <fstream>
21+
#include <iostream>
22+
#include <set>
23+
#include <string>
24+
#include <vector>
25+
26+
#include "osp/auxiliary/misc.hpp"
27+
#include "osp/graph_algorithms/directed_graph_path_util.hpp"
28+
#include "osp/auxiliary/io/general_file_reader.hpp"
29+
#include "osp/partitioning/model/hypergraph_utility.hpp"
30+
#include "osp/partitioning/partitioners/generic_FM.hpp"
31+
#include "osp/partitioning/partitioners/partitioning_ILP.hpp"
32+
#include "osp/partitioning/partitioners/partitioning_ILP_replication.hpp"
33+
#include "osp/graph_implementations/adj_list_impl/computational_dag_vector_impl.hpp"
34+
#include "osp/auxiliary/io/hdag_graph_file_reader.hpp"
35+
#include "osp/auxiliary/io/mtx_hypergraph_file_reader.hpp"
36+
#include "osp/auxiliary/io/partitioning_file_writer.hpp"
37+
38+
39+
using namespace osp;
40+
41+
using graph = computational_dag_vector_impl_def_int_t;
42+
using hypergraph = Hypergraph_def_t;
43+
44+
int main(int argc, char *argv[]) {
45+
if (argc < 4) {
46+
std::cerr << "Usage: " << argv[0] << " <input_file> <nr_parts> <imbalance> <optional:part_repl|full_repl>"
47+
<< std::endl;
48+
return 1;
49+
}
50+
51+
std::string filename_hgraph = argv[1];
52+
std::string name_hgraph = filename_hgraph.substr(0, filename_hgraph.rfind("."));
53+
std::string file_ending = filename_hgraph.substr(filename_hgraph.rfind(".") + 1);
54+
if (!file_reader::isPathSafe(filename_hgraph)) {
55+
std::cerr << "Error: Unsafe file path (possible traversal or invalid type).\n";
56+
return 1;
57+
}
58+
59+
std::cout << name_hgraph << std::endl;
60+
61+
int nr_parts = std::stoi(argv[2]);
62+
if (nr_parts < 2 || nr_parts > 32) {
63+
std::cerr << "Argument nr_parts must be an integer between 2 and 32: " << nr_parts << std::endl;
64+
return 1;
65+
}
66+
67+
float imbalance = std::stof(argv[3]);
68+
if (imbalance < 0.01 || imbalance > .99) {
69+
std::cerr << "Argument imbalance must be a float between 0.01 and 0.99: " << imbalance << std::endl;
70+
return 1;
71+
}
72+
73+
unsigned replicate = 0;
74+
75+
if (argc > 4 && std::string(argv[4]) == "part_repl") {
76+
replicate = 1;
77+
} else if (argc > 4 && std::string(argv[4]) == "full_repl") {
78+
replicate = 2;
79+
} else if (argc > 4) {
80+
std::cerr << "Unknown argument: " << argv[4] << ". Expected 'part_repl' or 'full_repl' for replication." << std::endl;
81+
return 1;
82+
}
83+
84+
PartitioningProblem<hypergraph> instance;
85+
86+
bool file_status = true;
87+
if (file_ending == "hdag") {
88+
graph dag;
89+
file_status = file_reader::readComputationalDagHyperdagFormatDB(filename_hgraph, dag);
90+
if(file_status)
91+
instance.getHypergraph() = convert_from_cdag_as_hyperdag<hypergraph, graph>(dag);
92+
} else if (file_ending == "mtx") {
93+
file_status = file_reader::readHypergraphMartixMarketFormat(filename_hgraph, instance.getHypergraph());
94+
} else {
95+
std::cout << "Unknown file extension." << std::endl;
96+
return 1;
97+
}
98+
if (!file_status) {
99+
100+
std::cout << "Reading input file failed." << std::endl;
101+
return 1;
102+
}
103+
104+
instance.setNumberOfPartitions(static_cast<unsigned>(nr_parts));
105+
instance.setMaxWorkWeightViaImbalanceFactor(imbalance);
106+
107+
Partitioning<hypergraph> initial_partition(instance);
108+
GenericFM<hypergraph> fm;
109+
for(size_t node = 0; node < instance.getHypergraph().num_vertices(); ++node)
110+
initial_partition.setAssignedPartition(node, static_cast<unsigned>(node % static_cast<size_t>(nr_parts)));
111+
if(nr_parts == 2)
112+
fm.ImprovePartitioning(initial_partition);
113+
if(nr_parts == 4 || nr_parts == 8 || nr_parts == 16 || nr_parts == 32)
114+
fm.RecursiveFM(initial_partition);
115+
116+
if (replicate > 0) {
117+
118+
PartitioningWithReplication<hypergraph> partition(instance);
119+
HypergraphPartitioningILPWithReplication<hypergraph> partitioner;
120+
121+
for(size_t node = 0; node < instance.getHypergraph().num_vertices(); ++node)
122+
partition.setAssignedPartitions(node, {initial_partition.assignedPartition(node)});
123+
if(partition.satisfiesBalanceConstraint())
124+
partitioner.setUseInitialSolution(true);
125+
126+
partitioner.setTimeLimitSeconds(600);
127+
if(replicate == 2)
128+
partitioner.setReplicationModel(HypergraphPartitioningILPWithReplication<hypergraph>::REPLICATION_MODEL_IN_ILP::GENERAL);
129+
130+
auto solve_status = partitioner.computePartitioning(partition);
131+
132+
if (solve_status == RETURN_STATUS::OSP_SUCCESS || solve_status == RETURN_STATUS::BEST_FOUND) {
133+
file_writer::write_txt(name_hgraph + "_" + std::to_string(nr_parts) + "_" + std::to_string(imbalance) +
134+
"_ILP_rep" + std::to_string(replicate) + ".txt", partition);
135+
std::cout << "Partitioning (with replicaiton) computed with costs: " << partition.computeConnectivityCost() << std::endl;
136+
} else {
137+
std::cout << "Computing partition failed." << std::endl;
138+
return 1;
139+
}
140+
141+
} else {
142+
143+
Partitioning<hypergraph> partition(instance);
144+
HypergraphPartitioningILP<hypergraph> partitioner;
145+
146+
for(size_t node = 0; node < instance.getHypergraph().num_vertices(); ++node)
147+
partition.setAssignedPartition(node, initial_partition.assignedPartition(node));
148+
if(partition.satisfiesBalanceConstraint())
149+
partitioner.setUseInitialSolution(true);
150+
151+
partitioner.setTimeLimitSeconds(600);
152+
153+
auto solve_status = partitioner.computePartitioning(partition);
154+
155+
if (solve_status == RETURN_STATUS::OSP_SUCCESS || solve_status == RETURN_STATUS::BEST_FOUND) {
156+
file_writer::write_txt(name_hgraph + "_" + std::to_string(nr_parts) + "_" + std::to_string(imbalance) +
157+
"_ILP_rep" + std::to_string(replicate) + ".txt", partition);
158+
std::cout << "Partitioning computed with costs: " << partition.computeConnectivityCost() << std::endl;
159+
} else {
160+
std::cout << "Computing partition failed." << std::endl;
161+
return 1;
162+
}
163+
}
164+
return 0;
165+
}

‎include/osp/auxiliary/io/mtx_hypergraph_file_reader.hpp‎

Lines changed: 175 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,175 @@
1+
/*
2+
Copyright 2024 Huawei Technologies Co., Ltd.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
16+
@author Toni Boehnlein, Christos Matzoros, Benjamin Lozes, Pal Andras Papp, Raphael S. Steiner
17+
*/
18+
19+
#pragma once
20+
21+
#include <fstream>
22+
#include <iostream>
23+
#include <sstream>
24+
#include <string>
25+
#include <vector>
26+
#include <limits>
27+
#include <filesystem>
28+
29+
#include "osp/partitioning/model/hypergraph.hpp"
30+
#include "osp/auxiliary/io/filepath_checker.hpp"
31+
32+
namespace osp {
33+
namespace file_reader {
34+
35+
// reads a matrix into Hypergraph format, where nonzeros are vertices, and rows/columns are hyperedges
36+
template<typename index_type, typename workw_type, typename memw_type, typename commw_type>
37+
bool readHypergraphMartixMarketFormat(std::ifstream& infile, Hypergraph<index_type, workw_type, memw_type, commw_type>& hgraph) {
38+
39+
std::string line;
40+
41+
// Skip comments or empty lines (robustly)
42+
while (std::getline(infile, line)) {
43+
if (line.empty() || line[0] == '%') continue;
44+
45+
// Null byte check
46+
if (line.find('\0') != std::string::npos) {
47+
std::cerr << "Error: Null byte detected in header line.\n";
48+
return false;
49+
}
50+
51+
if (line.size() > MAX_LINE_LENGTH) {
52+
std::cerr << "Error: Line too long, possible malformed or malicious file.\n";
53+
return false;
54+
}
55+
break; // We found the actual header line
56+
}
57+
58+
if (infile.eof()) {
59+
std::cerr << "Error: Unexpected end of file while reading header.\n";
60+
return false;
61+
}
62+
63+
int M_row = 0, M_col = 0, nEntries = 0;
64+
65+
std::istringstream header_stream(line);
66+
if (!(header_stream >> M_row >> M_col >> nEntries) ||
67+
M_row <= 0 || M_col <= 0) {
68+
std::cerr << "Error: Invalid header.\n";
69+
return false;
70+
}
71+
72+
const index_type num_nodes = static_cast<index_type>(nEntries);
73+
74+
hgraph.reset(num_nodes, 0);
75+
for (index_type node = 0; node < num_nodes; ++node) {
76+
hgraph.set_vertex_work_weight(node, static_cast<workw_type>(1));
77+
hgraph.set_vertex_memory_weight(node, static_cast<memw_type>(1));
78+
}
79+
80+
std::vector<std::vector<index_type>> row_hyperedges(static_cast<index_type>(M_row));
81+
std::vector<std::vector<index_type>> column_hyperedges(static_cast<index_type>(M_col));
82+
83+
int entries_read = 0;
84+
while (entries_read < nEntries && std::getline(infile, line)) {
85+
if (line.empty() || line[0] == '%') continue;
86+
if (line.size() > MAX_LINE_LENGTH) {
87+
std::cerr << "Error: Line too long.\n";
88+
return false;
89+
}
90+
91+
std::istringstream entry_stream(line);
92+
int row = -1, col = -1;
93+
double val = 0.0;
94+
95+
if (!(entry_stream >> row >> col >> val)) {
96+
std::cerr << "Error: Malformed matrix entry.\n";
97+
return false;
98+
}
99+
100+
row -= 1; col -= 1; // Convert to 0-based
101+
102+
if (row < 0 || col < 0 || row >= M_row || col >= M_col) {
103+
std::cerr << "Error: Matrix entry out of bounds.\n";
104+
return false;
105+
}
106+
107+
if (static_cast<index_type>(row) >= num_nodes || static_cast<index_type>(col) >= num_nodes) {
108+
std::cerr << "Error: Index exceeds vertex type limit.\n";
109+
return false;
110+
}
111+
112+
row_hyperedges[static_cast<index_type>(row)].push_back(static_cast<index_type>(entries_read));
113+
column_hyperedges[static_cast<index_type>(col)].push_back(static_cast<index_type>(entries_read));
114+
115+
++entries_read;
116+
}
117+
118+
if (entries_read != nEntries) {
119+
std::cerr << "Error: Incomplete matrix entries.\n";
120+
return false;
121+
}
122+
123+
while (std::getline(infile, line)) {
124+
if (!line.empty() && line[0] != '%') {
125+
std::cerr << "Error: Extra data after matrix content.\n";
126+
return false;
127+
}
128+
}
129+
130+
for(index_type row = 0; row < static_cast<index_type>(M_row); ++row)
131+
if(!row_hyperedges[row].empty())
132+
hgraph.add_hyperedge(row_hyperedges[row]);
133+
134+
for(index_type col = 0; col < static_cast<index_type>(M_col); ++col)
135+
if(!column_hyperedges[col].empty())
136+
hgraph.add_hyperedge(column_hyperedges[col]);
137+
138+
return true;
139+
}
140+
141+
template<typename index_type, typename workw_type, typename memw_type, typename commw_type>
142+
bool readHypergraphMartixMarketFormat(const std::string& filename, Hypergraph<index_type, workw_type, memw_type, commw_type>& hgraph) {
143+
// Ensure the file is .mtx format
144+
if (std::filesystem::path(filename).extension() != ".mtx") {
145+
std::cerr << "Error: Only .mtx files are accepted.\n";
146+
return false;
147+
}
148+
149+
if (!isPathSafe(filename)) {
150+
std::cerr << "Error: Unsafe file path (potential traversal attack).\n";
151+
return false;
152+
}
153+
154+
if (std::filesystem::is_symlink(filename)) {
155+
std::cerr << "Error: Symbolic links are not allowed.\n";
156+
return false;
157+
}
158+
159+
if (!std::filesystem::is_regular_file(filename)) {
160+
std::cerr << "Error: Input is not a regular file.\n";
161+
return false;
162+
}
163+
164+
std::ifstream infile(filename);
165+
if (!infile.is_open()) {
166+
std::cerr << "Error: Failed to open file.\n";
167+
return false;
168+
}
169+
170+
return readHypergraphMartixMarketFormat(infile, hgraph);
171+
}
172+
173+
} // namespace FileReader
174+
175+
} // namespace osp

0 commit comments

Comments
 (0)