Skip to content
This repository was archived by the owner on Mar 4, 2025. It is now read-only.

Commit 64b7835

Browse files
PogozhelskayaPogozhelskaya
committed
Memory benchmarks
1 parent 062f1d8 commit 64b7835

File tree

1 file changed

+122
-57
lines changed

1 file changed

+122
-57
lines changed

‎src/benchmark/Neo4jBenchmark.java‎

Lines changed: 122 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88

99
import static java.lang.System.gc;
1010
import static java.util.Arrays.asList;
11+
1112
import com.google.common.collect.Lists;
1213

1314
import iguana.utils.input.GraphInput;
@@ -72,8 +73,8 @@ public class Neo4jBenchmark {
7273
public static void main(String[] args) throws IOException {
7374

7475
loadGraph(args[6], Integer.parseInt(args[1]), args[4], args[0]);
75-
benchmark(args[0], Integer.parseInt(args[1]), Integer.parseInt(args[2]), Integer.parseInt(args[3]), args[5], args[6], args[7], args[4]);
76-
//benchmarkReachabilities(args[0], Integer.parseInt(args[1]), Integer.parseInt(args[2]), Integer.parseInt(args[3]), args[5], args[6], args[7], args[4]);
76+
//benchmark(args[0], Integer.parseInt(args[1]), Integer.parseInt(args[2]), Integer.parseInt(args[3]), args[5], args[6], args[7], args[4]);
77+
benchmarkReachabilities(args[0], Integer.parseInt(args[1]), Integer.parseInt(args[2]), Integer.parseInt(args[3]), args[5], args[6], args[7], args[4]);
7778
removeData();
7879
managementService.shutdown();
7980
}
@@ -165,16 +166,16 @@ public static void registerProcedure(GraphDatabaseService graphDb, List<Class<?>
165166
}
166167
}
167168

168-
public static void loadGraph(String dataset, int nodeNumber, String pathToDataset, String typeOfRelationships) throws IOException {
169+
public static void loadGraph(String dataset, int nodeNumber, String pathToDataset, String typeOfRelationships) throws IOException {
169170
FileUtils.deleteRecursively(databaseDirectory);
170171
managementService =
171172
new DatabaseManagementServiceBuilder(databaseDirectory)
172173
//.setConfig(GraphDatabaseSettings.pagecache_memory, "100G")
173174
.setConfig(GraphDatabaseSettings.tx_state_max_off_heap_memory, java.lang.Long.parseLong("24000000000"))
174175
.setConfig(GraphDatabaseSettings.pagecache_warmup_enabled, true)
175-
.setConfig(GraphDatabaseSettings.procedure_whitelist, List.of("gds.*","apoc.*", "apoc.load.*"))
176+
.setConfig(GraphDatabaseSettings.procedure_whitelist, List.of("gds.*", "apoc.*", "apoc.load.*"))
176177
.setConfig(GraphDatabaseSettings.procedure_unrestricted, List.of("gds.*", "apoc.*"))
177-
.setConfig(GraphDatabaseSettings.default_allowed,"gds.*,apoc.*")
178+
.setConfig(GraphDatabaseSettings.default_allowed, "gds.*,apoc.*")
178179
.setConfig(BoltConnector.enabled, true)
179180
.setConfig(ApocSettings.apoc_import_file_enabled, true)
180181
.setConfig(ApocSettings.apoc_import_file_use__neo4j__config, false)
@@ -189,7 +190,7 @@ public static void loadGraph(String dataset, int nodeNumber, String pathToDatas
189190
));
190191

191192
try (Stream<String> inputNodes = Files.lines(Paths.get("/" + pathToDataset + dataset + "_all_nodes.csv"))) {
192-
try (Transaction tx = graphDb.beginTx()){
193+
try (Transaction tx = graphDb.beginTx()) {
193194
inputNodes.forEach(node -> {
194195
String s = String.format("CREATE (:Node {name: '%d'});", Integer.parseInt(node));
195196
tx.execute(s);
@@ -198,12 +199,12 @@ public static void loadGraph(String dataset, int nodeNumber, String pathToDatas
198199
}
199200
}
200201
// try (Transaction tx = graphDb.beginTx()) {
201-
// for (int i = 0; i < nodeNumber; ++i) {
202-
// String s = String.format("CREATE (:Node {name: '%d'});", i);
203-
// tx.execute(s);
204-
// }
205-
// tx.commit();
206-
// }
202+
// for (int i = 0; i < nodeNumber; ++i) {
203+
// String s = String.format("CREATE (:Node {name: '%d'});", i);
204+
// tx.execute(s);
205+
// }
206+
// tx.commit();
207+
// }
207208

208209
try (Transaction tx = graphDb.beginTx()) {
209210
tx.execute("CREATE CONSTRAINT node_unique_name ON (n:Node) ASSERT n.name IS UNIQUE");
@@ -214,47 +215,47 @@ public static void loadGraph(String dataset, int nodeNumber, String pathToDatas
214215
if ("bt".equals(typeOfRelationships)) {
215216
try (Transaction tx = graphDb.beginTx()) {
216217
tx.execute(" CALL apoc.periodic.iterate(\n" +
217-
" \"CALL apoc.load.csv('FILE:///" + pathToDataset + dataset + "_broaderTransitive.csv') YIELD map AS row RETURN row\",\n" +
218-
" \"MATCH (f:Node {name: row.from}), (t:Node {name: row.to})\n" +
219-
" CREATE (f)-[:broaderTransitive]->(t)\",\n" +
220-
" {batchSize:10000, parallel:false}\n" +
221-
" )\n" +
222-
" YIELD batches, total;\n");
218+
" \"CALL apoc.load.csv('FILE:///" + pathToDataset + dataset + "_broaderTransitive.csv') YIELD map AS row RETURN row\",\n" +
219+
" \"MATCH (f:Node {name: row.from}), (t:Node {name: row.to})\n" +
220+
" CREATE (f)-[:broaderTransitive]->(t)\",\n" +
221+
" {batchSize:10000, parallel:false}\n" +
222+
" )\n" +
223+
" YIELD batches, total;\n");
223224
tx.execute(" CALL apoc.periodic.iterate(\n" +
224-
" \"CALL apoc.load.csv('FILE:///" + pathToDataset + dataset + "_other.csv') YIELD map AS row RETURN row\",\n" +
225-
" \"MATCH (f:Node {name: row.from}), (t:Node {name: row.to})\n" +
226-
" CREATE (f)-[:other]->(t)\",\n" +
227-
" {batchSize:100000, parallel:false}\n" +
228-
" )\n" +
229-
" YIELD batches, total;\n");
225+
" \"CALL apoc.load.csv('FILE:///" + pathToDataset + dataset + "_other.csv') YIELD map AS row RETURN row\",\n" +
226+
" \"MATCH (f:Node {name: row.from}), (t:Node {name: row.to})\n" +
227+
" CREATE (f)-[:other]->(t)\",\n" +
228+
" {batchSize:100000, parallel:false}\n" +
229+
" )\n" +
230+
" YIELD batches, total;\n");
230231
tx.commit();
231232
}
232-
} else if ("st".equals(typeOfRelationships)){
233+
} else if ("st".equals(typeOfRelationships)) {
233234
try (Transaction tx = graphDb.beginTx()) {
234235
tx.execute(" CALL apoc.periodic.iterate(\n" +
235-
" \"CALL apoc.load.csv('FILE:///" + pathToDataset + dataset + "_subClassOf.csv') YIELD map AS row RETURN row\",\n" +
236-
" \"MATCH (f:Node {name: row.from}), (t:Node {name: row.to})\n" +
237-
" CREATE (f)-[:subClassOf]->(t)\",\n" +
238-
" {batchSize:10000, parallel:false}\n" +
239-
" )\n" +
240-
" YIELD batches, total;\n");
236+
" \"CALL apoc.load.csv('FILE:///" + pathToDataset + dataset + "_subClassOf.csv') YIELD map AS row RETURN row\",\n" +
237+
" \"MATCH (f:Node {name: row.from}), (t:Node {name: row.to})\n" +
238+
" CREATE (f)-[:subClassOf]->(t)\",\n" +
239+
" {batchSize:10000, parallel:false}\n" +
240+
" )\n" +
241+
" YIELD batches, total;\n");
241242
tx.execute(" CALL apoc.periodic.iterate(\n" +
242-
" \"CALL apoc.load.csv('FILE:///" + pathToDataset + dataset + "_type.csv') YIELD map AS row RETURN row\",\n" +
243-
" \"MATCH (f:Node {name: row.from}), (t:Node {name: row.to})\n" +
244-
" CREATE (f)-[:type]->(t)\",\n" +
245-
" {batchSize:10000, parallel:false}\n" +
246-
" )\n" +
247-
" YIELD batches, total;\n");
243+
" \"CALL apoc.load.csv('FILE:///" + pathToDataset + dataset + "_type.csv') YIELD map AS row RETURN row\",\n" +
244+
" \"MATCH (f:Node {name: row.from}), (t:Node {name: row.to})\n" +
245+
" CREATE (f)-[:type]->(t)\",\n" +
246+
" {batchSize:10000, parallel:false}\n" +
247+
" )\n" +
248+
" YIELD batches, total;\n");
248249
tx.execute(" CALL apoc.periodic.iterate(\n" +
249-
" \"CALL apoc.load.csv('FILE:///" + pathToDataset + dataset + "_other.csv') YIELD map AS row RETURN row\",\n" +
250-
" \"MATCH (f:Node {name: row.from}), (t:Node {name: row.to})\n" +
251-
" CREATE (f)-[:other]->(t)\",\n" +
252-
" {batchSize:100000, parallel:false}\n" +
253-
" )\n" +
254-
" YIELD batches, total;\n");
250+
" \"CALL apoc.load.csv('FILE:///" + pathToDataset + dataset + "_other.csv') YIELD map AS row RETURN row\",\n" +
251+
" \"MATCH (f:Node {name: row.from}), (t:Node {name: row.to})\n" +
252+
" CREATE (f)-[:other]->(t)\",\n" +
253+
" {batchSize:100000, parallel:false}\n" +
254+
" )\n" +
255+
" YIELD batches, total;\n");
255256
tx.commit();
256257
}
257-
} else if ("ad".equals(typeOfRelationships)){
258+
} else if ("ad".equals(typeOfRelationships)) {
258259
try (Transaction tx = graphDb.beginTx()) {
259260
tx.execute(" CALL apoc.periodic.iterate(\n" +
260261
" \"CALL apoc.load.csv('FILE:///" + pathToDataset + dataset + "_D.csv') YIELD map AS row RETURN row\",\n" +
@@ -277,6 +278,70 @@ public static void loadGraph(String dataset, int nodeNumber, String pathToDatas
277278
}
278279

279280
public static void benchmarkReachabilities(String relType, int nodeNumber, int warmUp, int maxIter, String pathToGrammar, String dataset, String grammarName, String pathToDataset) throws IOException {
281+
BiFunction<Relationship, Direction, String> f = getFunction(relType);
282+
283+
Map<String, List<Integer>> vertexToTime = new HashMap<>();
284+
Grammar grammar;
285+
try {
286+
grammar = Grammar.load(pathToGrammar, "json");
287+
} catch (FileNotFoundException e) {
288+
throw new RuntimeException("No grammar.json file is present");
289+
}
290+
Runtime r = Runtime.getRuntime();
291+
PrintWriter outStatsTime = new PrintWriter("results/" + dataset + "_" + relType + "_time_reachabilities.csv");
292+
outStatsTime.append("chunk_size, time");
293+
outStatsTime.append("\n");
294+
295+
//List<Integer> chunkSize = Arrays.asList(10000);
296+
Integer chunkSize = nodeNumber;
297+
List<Integer> vertices = Interval.zeroTo(nodeNumber - 1);
298+
List<List<Integer>> verticesPartitioned = Lists.partition(vertices, chunkSize);
299+
long t1 = System.nanoTime();
300+
r.gc();
301+
long m1 = r.totalMemory() - r.freeMemory();
302+
List<Integer> sumResult = new ArrayList<>();
303+
verticesPartitioned.forEach(chunk -> {
304+
for (int iter = 0; iter < maxIter; ++iter) {
305+
IguanaParser parser = new IguanaParser(grammar);
306+
//System.out.println("iter 0" + " chunkSize " + sz);
307+
GraphInput input = new Neo4jBenchmarkInput(graphDb, f, chunk.stream(), nodeNumber);
308+
long t1_local = System.nanoTime();
309+
Stream<Pair> parseResults = parser.getReachabilities(input,
310+
new ParseOptions.Builder().setAmbiguous(false).build());
311+
//sumResult.add(parseResults.getSecond());
312+
long m2 = r.totalMemory() - r.freeMemory();
313+
long curM = (m2 - m1);
314+
long t2_local = System.nanoTime();
315+
long stepTime = t2_local - t1_local;
316+
//curT[0] += stepTime;a
317+
//System.out.println("My id " + myId + "; full time is " + curT[0] + "; step time is " + stepTime);
318+
System.out.println("Used memory is " + curM);
319+
r.gc();
320+
((Neo4jBenchmarkInput) input).close();
321+
/*if (parseResults != null) {
322+
vertexToTime.putIfAbsent(sz.toString(), new ArrayList<>());
323+
vertexToTime.get(sz.toString()).add((int) curT[0]);
324+
}*/
325+
if (parseResults != null)
326+
System.out.println(parseResults.count());
327+
// if (parseResults != null) {
328+
// return parseResults.count();
329+
// } else {
330+
// return 0;
331+
// }
332+
}
333+
});
334+
335+
final long[] curT = {0};
336+
337+
Integer i = 0;
338+
339+
340+
long t2 = System.nanoTime();
341+
// System.out.println("Total answer: " + finalResultCount);
342+
//System.out.println("Total memory: " + curM);
343+
344+
outStatsTime.close();
280345
}
281346

282347
public static void benchmark(String relType, int nodeNumber, int warmUp, int maxIter, String pathToGrammar, String dataset, String grammarName, String pathToDataset) throws IOException {
@@ -298,11 +363,11 @@ public static void benchmark(String relType, int nodeNumber, int warmUp, int max
298363
}
299364

300365
List<Tuple<Integer, Integer>> chunkSize = Arrays.asList(
301-
// new Tuple<>(nodeNumber, 100)
366+
new Tuple<>(1, 100)
302367
// new Tuple<>(1, 100)
303368
// , new Tuple<>(10, 100)
304369
// , new Tuple<>(50, 100)
305-
new Tuple<>(100, 100)
370+
// , new Tuple<>(100, 100)
306371
// , new Tuple<>(500, 100)
307372
// , new Tuple<>(1000, 100)
308373
// , new Tuple<>(5000, 100)
@@ -324,18 +389,18 @@ public static void benchmark(String relType, int nodeNumber, int warmUp, int max
324389
verticesPartitioned.parallelStream().forEach(chunk -> {
325390
GraphInput input = new Neo4jBenchmarkInput(graphDb, f, chunk.stream(), nodeNumber);
326391
System.out.println("iteration: " + finalIter + " first vertex: " + chunk.get(0));
327-
long t1_local = System.nanoTime();
392+
long t1_local = System.nanoTime();
328393
// Map<Pair, ParseTreeNode> parseResults = parser.getParserTree(input,
329394
// new ParseOptions.Builder().setAmbiguous(false).build());
330-
Map<Pair, NonterminalNode> parseResults = parser.getSPPF(input);
331-
long t2_local = System.nanoTime();
332-
long stepTime = t2_local - t1_local;
333-
if (finalIter >= warmUp) {
334-
resulTimePerChunk.println(stepTime);
335-
}
336-
if (parseResults != null) {
337-
System.out.println(" time: " + stepTime + "\n" + "ans:" + parseResults.size());
338-
}
395+
Map<Pair, NonterminalNode> parseResults = parser.getSPPF(input);
396+
long t2_local = System.nanoTime();
397+
long stepTime = t2_local - t1_local;
398+
if (finalIter >= warmUp) {
399+
resulTimePerChunk.println(stepTime);
400+
}
401+
if (parseResults != null) {
402+
System.out.println(" time: " + stepTime + "\n" + "ans:" + parseResults.size());
403+
}
339404
((Neo4jBenchmarkInput) input).close();
340405
});
341406
long t2 = System.nanoTime();
@@ -359,4 +424,4 @@ private static void removeData() {
359424
tx.commit();
360425
}
361426
}
362-
}
427+
}

0 commit comments

Comments
 (0)