Scale option (#32)

* Added scale option

* Improved performance of Zipf distribution at expenses of accuracy

* Fixed bug in serializers when numPartitions was larger than 1

* Added .travis.yml file

* Improved tests

Author: ArnauPrat

.travis.yml

@@ -3,6 +3,8 @@ sudo: required
 language: java
 jdk:
   - openjdk7
+python:
+  - "2.7"
 install:
   - curl -s http://www.eu.apache.org/dist/hadoop/common/hadoop-2.6.5/hadoop-2.6.5.tar.gz | tar -xz
 before_script:

CHANGELOG.txt

@@ -1,4 +1,12 @@
 
+v0.2.7 CHANGELOG
+
+- Added gscale option which allows specifying the size of the generated dataset
+based on the graphalytics scaling metric.
+
+- Fixed a bug which caused wrong serialization of data once the numPartitions
+parameter was larger than 1.
+
 v0.2.6 CHANGELOG
 Bi parameter generation
 Added testing

pom.xml

@@ -6,7 +6,7 @@
 
   <groupId>ldbc.snb.datagen</groupId>
   <artifactId>ldbc_snb_datagen</artifactId>
-  <version>0.2.5</version>
+  <version>0.2.7</version>
   <packaging>jar</packaging>
 
   <properties>

src/main/java/ldbc/snb/datagen/dictionary/Dictionaries.java

@@ -49,7 +49,7 @@ public static void loadDictionaries(Configuration conf) {
 
 		emails = new EmailDictionary();
 
-		places = new PlaceDictionary(DatagenParams.numPersons);
+		places = new PlaceDictionary();
 
 		ips = new IPAddressDictionary(  places,
 			DatagenParams.probDiffIPinTravelSeason,

src/main/java/ldbc/snb/datagen/dictionary/PlaceDictionary.java

@@ -100,10 +100,9 @@ public int compareTo(PlaceZOrder obj) {
     }
 
     /**
-     * @param numUsers: The total number of users.
      * @brief Creator.
      */
-    public PlaceDictionary(int numUsers) {
+    public PlaceDictionary() {
         this.cumulativeDistribution = new ArrayList<Float>();
         this.countryNames = new HashMap<String, Integer>();
         this.cityNames = new HashMap<String, Integer>();

src/main/java/ldbc/snb/datagen/generator/DatagenParams.java

@@ -38,16 +38,9 @@
 
 package ldbc.snb.datagen.generator;
 
-import ldbc.snb.datagen.util.ScaleFactor;
+import ldbc.snb.datagen.generator.distribution.DegreeDistribution;
+import ldbc.snb.datagen.generator.distribution.utils.Algorithms;
 import org.apache.hadoop.conf.Configuration;
-import org.w3c.dom.Document;
-import org.w3c.dom.Element;
-import org.w3c.dom.Node;
-import org.w3c.dom.NodeList;
-
-import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
-import java.util.TreeMap;
 
 public class DatagenParams {
 
@@ -217,12 +210,12 @@ public String toString() {
     public static final double alpha                       = 0.4;
 
 
-    public static String outputDir                         = "./";
+    public static String outputDir                          = "./";
     public static String hadoopDir                         = "./";
     public static String socialNetworkDir                  = "./";
     public static int    numThreads                        = 1;
     public static int    deltaTime                         = 10000;
-    public static int    numPersons                        = 10000;
+    public static long   numPersons                         = 10000;
     public static int    startYear                         = 2010;
     public static int    endYear                           = 2013;
     public static int    numYears                          = 3;
@@ -307,7 +300,7 @@ public static void readConf( Configuration conf ) {
         }
 
         try {
-            numPersons = Integer.parseInt(conf.get("ldbc.snb.datagen.generator.numPersons"));
+            numPersons = Long.parseLong(conf.get("ldbc.snb.datagen.generator.numPersons"));
             startYear = Integer.parseInt(conf.get("ldbc.snb.datagen.generator.startYear"));
             numYears = Integer.parseInt(conf.get("ldbc.snb.datagen.generator.numYears"));
             endYear = startYear + numYears;
@@ -320,6 +313,13 @@ public static void readConf( Configuration conf ) {
             outputDir = conf.get("ldbc.snb.datagen.serializer.outputDir");
             hadoopDir = outputDir+"/hadoop";
             socialNetworkDir = outputDir+"social_network";
+            if(conf.get("ldbc.snb.datagen.generator.gscale") != null) {
+                double scale = conf.getDouble("ldbc.snb.datagen.generator.gscale", 6.0);
+                String degreeDistributionName = conf.get("ldbc.snb.datagen.generator.distribution.degreeDistribution");
+                DegreeDistribution  degreeDistribution = (DegreeDistribution)Class.forName(degreeDistributionName).newInstance();
+                degreeDistribution.initialize(conf);
+                numPersons = Algorithms.findNumPersonsFromGraphalyticsScale(degreeDistribution,scale);
+            }
             System.out.println(" ... Num Persons " + numPersons);
             System.out.println(" ... Start Year " + startYear);
             System.out.println(" ... Num Years " + numYears);

src/main/java/ldbc/snb/datagen/generator/LDBCDatagen.java

@@ -39,15 +39,12 @@
 import ldbc.snb.datagen.dictionary.Dictionaries;
 import ldbc.snb.datagen.hadoop.*;
 import ldbc.snb.datagen.objects.Person;
-import ldbc.snb.datagen.objects.similarity.GeoDistanceSimilarity;
 import ldbc.snb.datagen.util.ConfigParser;
 import ldbc.snb.datagen.vocabulary.SN;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
 
 
 import java.io.File;
@@ -59,7 +56,7 @@
 public class LDBCDatagen {
 
     static boolean initialized = false;
-    public static synchronized void init (Configuration conf) {
+    public static synchronized void initializeContext(Configuration conf) {
         if(!initialized) {
             DatagenParams.readConf(conf);
             Dictionaries.loadDictionaries(conf);
@@ -352,26 +349,31 @@ public int runGenerateJob(Configuration conf) throws Exception {
         return 0;
     }
 
-    public static void main(String[] args) /*throws Exception*/ {
+    public static void prepareConfiguration(Configuration conf) throws Exception {
 
-        try {
-        Configuration conf = ConfigParser.initialize();
-        ConfigParser.readConfig(conf, args[0]);
-        ConfigParser.readConfig(conf, LDBCDatagen.class.getResourceAsStream("/params.ini"));
         conf.set("ldbc.snb.datagen.serializer.hadoopDir",conf.get("ldbc.snb.datagen.serializer.outputDir")+"/hadoop");
         conf.set("ldbc.snb.datagen.serializer.socialNetworkDir",conf.get("ldbc.snb.datagen.serializer.outputDir")+"/social_network");
-        ConfigParser.printConfig(conf);
-//        conf.setBoolean("mapreduce.map.output.compress", true);
-//       conf.setBoolean("mapreduce.output.fileoutputformat.compress", false);
 
         // Deleting existing files
         FileSystem dfs = FileSystem.get(conf);
         dfs.delete(new Path(conf.get("ldbc.snb.datagen.serializer.hadoopDir")), true);
         dfs.delete(new Path(conf.get("ldbc.snb.datagen.serializer.socialNetworkDir")), true);
 
-        // Create input text file in HDFS
-        LDBCDatagen datagen = new LDBCDatagen();
-        LDBCDatagen.init(conf);
+        ConfigParser.printConfig(conf);
+
+    }
+
+    public static void main(String[] args) /*throws Exception*/ {
+
+        try {
+            Configuration conf = ConfigParser.initialize();
+            ConfigParser.readConfig(conf, args[0]);
+            ConfigParser.readConfig(conf, LDBCDatagen.class.getResourceAsStream("/params.ini"));
+
+            // Create input text file in HDFS
+            LDBCDatagen.prepareConfiguration(conf);
+            LDBCDatagen.initializeContext(conf);
+            LDBCDatagen datagen = new LDBCDatagen();
             datagen.runGenerateJob(conf);
         }catch(AssertionError e ) {
             System.err.println("Error during execution");

src/main/java/ldbc/snb/datagen/generator/distribution/AltmannDistribution.java

@@ -12,7 +12,7 @@
  */
 public class AltmannDistribution extends CumulativeBasedDegreeDistribution {
 
-    private int POPULATION_ = 10000;
+    private long POPULATION_ = 10000;
     private double normalization_factor_ = 0.0;
     private double ALPHA_ = 0.4577;
     private double BETA_ = 0.0162;

src/main/java/ldbc/snb/datagen/generator/distribution/CumulativeBasedDegreeDistribution.java

@@ -8,7 +8,7 @@
 /**
  * Created by aprat on 12/05/15.
  */
-public abstract class CumulativeBasedDegreeDistribution implements DegreeDistribution{
+public abstract class CumulativeBasedDegreeDistribution extends DegreeDistribution {
 
     public class CumulativeEntry {
         double prob_;

src/main/java/ldbc/snb/datagen/generator/distribution/DegreeDistribution.java

@@ -5,22 +5,21 @@
  */
 package ldbc.snb.datagen.generator.distribution;
 
-import ldbc.snb.datagen.generator.distribution.utils.Bucket;
-import ldbc.snb.datagen.generator.distribution.utils.BucketedDistribution;
 import org.apache.hadoop.conf.Configuration;
 
-import java.util.ArrayList;
-import java.util.Random;
-
 /**
  *
  * @author aprat
  */
-public interface DegreeDistribution {
+public abstract class DegreeDistribution {
+
+	public abstract void initialize( Configuration conf );
 
-	public void initialize( Configuration conf );
+    public abstract void reset (long seed);
 
-    public void reset (long seed);
+	public abstract long nextDegree();
 
-	public long nextDegree();
+	public double mean(long numPersons) {
+		return -1;
+	}
 }