Fixed critical bug that made the generator crash when serializing with CSV serializers

Author: ArnauPrat

src/main/java/ldbc/snb/datagen/generator/distribution/AltmannDistribution.java

@@ -27,7 +27,8 @@ public ArrayList<Bucket> getBuckets() {
 
     @Override
     public void initialize(Configuration conf) {
-        POPULATION_ = DatagenParams.numPersons;
+        throw new UnsupportedOperationException("Distribution not implemented");
+        /*POPULATION_ = DatagenParams.numPersons;
         for( int i = 1; i <= POPULATION_; ++i ) {
             normalization_factor_+= Math.pow(i,-GAMMA_)*Math.exp(-DELTA_*i);
         }
@@ -42,5 +43,6 @@ public void initialize(Configuration conf) {
             e.max(e.max()*scale_factor);
             e.min(e.min()*scale_factor);
         }
+        */
     }
 }

src/main/java/ldbc/snb/datagen/generator/distribution/WeibullDistribution.java

@@ -0,0 +1,37 @@
+package ldbc.snb.datagen.generator.distribution;
+
+import ldbc.snb.datagen.generator.DatagenParams;
+import ldbc.snb.datagen.generator.distribution.utils.Bucket;
+import ldbc.snb.datagen.generator.distribution.utils.BucketedDistribution;
+import org.apache.commons.math3.distribution.*;
+import org.apache.hadoop.conf.Configuration;
+
+import java.util.ArrayList;
+
+/**
+ * Created by aprat on 5/03/15.
+ */
+public class WeibullDistribution implements BucketedDistribution {
+
+    private ArrayList<Bucket> buckets_;
+    private org.apache.commons.math3.distribution.WeibullDistribution weibull_;
+    private double LAMBDA_ = 0.3881;
+    private double K_ = 0.2622;
+
+    @Override
+    public ArrayList<Bucket> getBuckets() {
+        return buckets_;
+    }
+
+    @Override
+    public void initialize(Configuration conf) {
+        weibull_ = new org.apache.commons.math3.distribution.WeibullDistribution(LAMBDA_,K_);
+
+        ArrayList<Double> histogram = new ArrayList<Double>();
+        for( int i = 1; i <= DatagenParams.numPersons; ++i ) {
+            histogram.add(DatagenParams.numPersons * weibull_.probability(i));
+        }
+
+        buckets_ = Bucket.bucketizeHistogram(histogram,100);
+    }
+}

src/main/java/ldbc/snb/datagen/serializer/HDFSCSVWriter.java

@@ -19,6 +19,7 @@ public class HDFSCSVWriter extends HDFSWriter {
     public HDFSCSVWriter( String outputDir, String prefix, int numPartitions, boolean compressed, String separator )  {
        super(outputDir, prefix, numPartitions, compressed, "csv" );
         this.separator = separator;
+        buffer = new StringBuffer(2048);
 
     }
 

src/main/java/ldbc/snb/datagen/serializer/snb/interactive/CSVMergeForeignPersonSerializer.java

@@ -183,6 +183,8 @@ protected void serialize( Person p, Knows knows) {
         arguments.add(dateString);
         writers[FileNames.PERSON_KNOWS_PERSON.ordinal()].writeEntry(arguments);
     }
+
+    @Override
     public void reset() {
 
     }

src/main/java/ldbc/snb/datagen/serializer/snb/interactive/CSVPersonSerializer.java

@@ -228,6 +228,7 @@ protected void serialize(Person p, Knows knows) {
         writers[FileNames.PERSON_KNOWS_PERSON.ordinal()].writeEntry(arguments);
     }
 
+    @Override
     public void reset() {
 
     }