Thrown and captured exceptions upon a job failing

Author: ArnauPrat

src/main/java/ldbc/snb/datagen/generator/LDBCDatagen.java

@@ -237,8 +237,9 @@ public int runGenerateJob(Configuration conf) throws Exception {
         return 0;
     }
 
-    public static void main(String[] args) throws Exception {
+    public static void main(String[] args) /*throws Exception*/ {
 
+        try {
         Configuration conf = ConfigParser.initialize();
         ConfigParser.readConfig(conf,"./src/main/resources/params.ini");
         //ConfigParser.readConfig(conf,args[0]);
@@ -257,6 +258,11 @@ public static void main(String[] args) throws Exception {
         // Create input text file in HDFS
         LDBCDatagen datagen = new LDBCDatagen();
         LDBCDatagen.init(conf);
-        datagen.runGenerateJob(conf);
+            datagen.runGenerateJob(conf);
+        }catch(Exception e ) {
+            System.err.println("Error during execution");
+            System.err.println(e.getMessage());
+            System.exit(1);
+        }
     }
 }

src/main/java/ldbc/snb/datagen/hadoop/HadoopFileKeyChanger.java

@@ -78,6 +78,8 @@ public void run( String inputFileName, String outputFileName ) throws Exception
         job.setJarByClass(V);
         job.setInputFormatClass(SequenceFileInputFormat.class);
         job.setOutputFormatClass(SequenceFileOutputFormat.class);
-        job.waitForCompletion(true);
+        if(!job.waitForCompletion(true)){
+            throw new Exception();
+        }
     }
 }

src/main/java/ldbc/snb/datagen/hadoop/HadoopFileRanker.java

@@ -155,7 +155,9 @@ public void run( String inputFileName, String outputFileName ) throws Exception
         TotalOrderPartitioner.setPartitionFile(jobSort.getConfiguration(), new Path(inputFileName + "_partition.lst"));
         InputSampler.writePartitionFile(jobSort, sampler);
         jobSort.setPartitionerClass(TotalOrderPartitioner.class);
-        jobSort.waitForCompletion(true);
+        if(!jobSort.waitForCompletion(true)){
+            throw new Exception();
+        }
 
         /** Second Job to assign the rank to each element.**/
         Job jobRank = Job.getInstance(conf, "Sorting "+inputFileName);
@@ -173,7 +175,9 @@ public void run( String inputFileName, String outputFileName ) throws Exception
         jobRank.setInputFormatClass(SequenceFileInputFormat.class);
         jobRank.setOutputFormatClass(SequenceFileOutputFormat.class);
         jobRank.setPartitionerClass(HadoopFileRankerPartitioner.class);
-        jobRank.waitForCompletion(true);
+        if(!jobRank.waitForCompletion(true)) {
+            throw new Exception();
+        }
 
         try{
             FileSystem fs = FileSystem.get(conf);

src/main/java/ldbc/snb/datagen/hadoop/HadoopFileSorter.java

@@ -58,6 +58,8 @@ public void run( String inputFileName, String outputFileName ) throws Exception
         TotalOrderPartitioner.setPartitionFile(job.getConfiguration(),new Path(inputFileName+"_partition.lst"));
         InputSampler.writePartitionFile(job, sampler);
         job.setPartitionerClass(TotalOrderPartitioner.class);
-        job.waitForCompletion(true);
+        if(!job.waitForCompletion(true)) {
+            throw new Exception();
+        }
     }
 }

src/main/java/ldbc/snb/datagen/hadoop/HadoopKnowsGenerator.java

@@ -113,7 +113,9 @@ public void run( String inputFileName, String outputFileName ) throws Exception
 
         System.out.println("Generating knows relations");
         start = System.currentTimeMillis();
-        job.waitForCompletion(true);
+        if(!job.waitForCompletion(true) ){
+            throw new Exception();
+        }
         System.out.println("... Time to generate knows relations: "+ (System.currentTimeMillis() - start)+" ms");
 
         fs.delete(new Path(rankedFileName), true);

src/main/java/ldbc/snb/datagen/hadoop/HadoopPersonActivityGenerator.java

@@ -132,7 +132,9 @@ public void run( String inputFileName ) throws Exception {
 
         FileInputFormat.setInputPaths(job, new Path(rankedFileName));
         FileOutputFormat.setOutputPath(job, new Path(conf.get("ldbc.snb.datagen.serializer.hadoopDir")+"/aux"));
-        job.waitForCompletion(true);
+        if(!job.waitForCompletion(true)) {
+            throw new Exception();
+        }
 
         try{
             fs.delete(new Path(rankedFileName), true);

src/main/java/ldbc/snb/datagen/hadoop/HadoopPersonGenerator.java

@@ -133,6 +133,8 @@ public void run( String outputFileName, String postKeySetterName ) throws Except
         job.setOutputFormatClass(SequenceFileOutputFormat.class);
         FileInputFormat.setInputPaths(job, new Path(tempFile));
         FileOutputFormat.setOutputPath(job, new Path(outputFileName));
-        job.waitForCompletion(true);
+        if(!job.waitForCompletion(true)) {
+            throw new Exception();
+        }
     }
 }

src/main/java/ldbc/snb/datagen/hadoop/HadoopPersonSerializer.java

@@ -103,7 +103,9 @@ public void run( String inputFileName ) throws Exception {
 
 		FileInputFormat.setInputPaths(job, new Path(rankedFileName));
 		FileOutputFormat.setOutputPath(job, new Path(conf.get("ldbc.snb.datagen.serializer.hadoopDir")+"/aux"));
-		job.waitForCompletion(true);
+		if(!job.waitForCompletion(true)) {
+            throw new Exception();
+        }
 
 
 		try{

src/main/java/ldbc/snb/datagen/hadoop/HadoopUpdateStreamSerializer.java

@@ -92,7 +92,9 @@ public void run( String inputFileName, int reducer, int partition, String type )
 
 		FileInputFormat.setInputPaths(job, new Path(inputFileName));
 		FileOutputFormat.setOutputPath(job, new Path(conf.get("ldbc.snb.datagen.serializer.hadoopDir")+"/aux"));
-		job.waitForCompletion(true);
+		if(!job.waitForCompletion(true)) {
+            throw new Exception();
+        }
 
 
 		try{

tools/extractDegrees.py

@@ -17,7 +17,12 @@
             edgesPerPerson[int(edge[0])]+=1
         else:
             edgesPerPerson[int(edge[0])]=1
-        numEdges+=1
+
+        if int(edge[1]) in edgesPerPerson:
+            edgesPerPerson[int(edge[1])]+=1
+        else:
+            edgesPerPerson[int(edge[1])]=1
+        numEdges+=2
     index+=1   
 knowsFile.close()