Merge branch 'master' of https://github.com/ldbc/ldbc_snb_datagen

Author: Hadoop user

run.sh

@@ -1,8 +1,6 @@
 #!/bin/bash
-DEFAULT_HADOOP_HOME=/usr/local/
-DEFAULT_LDBC_SNB_DATAGEN_HOME=/Users/alexaverbuch/IdeaProjects/ldbc_snb_datagen
-#DEFAULT_HADOOP_HOME=/home/user/hadoop-2.6.0 #change to your hadoop folder
-#DEFAULT_LDBC_SNB_DATAGEN_HOME=/home/user/ldbc_snb_datagen_0.2 #change to your ldbc_socialnet_dbgen folder
+DEFAULT_HADOOP_HOME=/home/user/hadoop-2.6.0 #change to your hadoop folder
+DEFAULT_LDBC_SNB_DATAGEN_HOME=/home/user/ldbc_snb_datagen_0.2 #change to your ldbc_socialnet_dbgen folder
 PARAM_GENERATION=1 #param generation
 
 # allow overriding configuration from outside via environment variables
@@ -14,17 +12,13 @@ LDBC_SNB_DATAGEN_HOME=${LDBC_SNB_DATAGEN_HOME:-$DEFAULT_LDBC_SNB_DATAGEN_HOME}
 
 export HADOOP_HOME
 export LDBC_SNB_DATAGEN_HOME
-export HADOOP_HEAPSIZE=6144
-export HADOOP_CLIENT_OPTS="-Xmx6g $HADOOP_CLIENT_OPTS"
 
 mvn clean
 mvn -DskipTests assembly:assembly 
 
 cp $LDBC_SNB_DATAGEN_HOME/target/ldbc_snb_datagen.jar $LDBC_SNB_DATAGEN_HOME/
 rm $LDBC_SNB_DATAGEN_HOME/target/ldbc_snb_datagen.jar
 
-zip -d $LDBC_SNB_DATAGEN_HOME/ldbc_snb_datagen.jar META-INF/LICENSE
-
 $HADOOP_HOME/bin/hadoop jar $LDBC_SNB_DATAGEN_HOME/ldbc_snb_datagen.jar $LDBC_SNB_DATAGEN_HOME/params.ini
 
 if [ $PARAM_GENERATION -eq 1 ]
@@ -38,4 +32,4 @@ then
   rm -f .m*activityFactors*
   rm -f m0friendList*
   rm -f .m0friendList*
-fi
+fi

src/main/java/ldbc/snb/datagen/generator/PersonActivityGenerator.java

@@ -33,12 +33,11 @@ public class PersonActivityGenerator {
     private PersonActivityExporter exporter_;
 
 	public PersonActivityGenerator( PersonActivitySerializer serializer, UpdateEventSerializer updateSerializer ) {
+		randomFarm_ = new RandomGeneratorFarm();
 		personActivitySerializer_ = serializer;
 		updateSerializer_ = updateSerializer;
-		randomFarm_ = new RandomGeneratorFarm();
 		forumGenerator_ = new ForumGenerator();
-		Random random = new Random();
-		TextGenerator generator = new LdbcSnbTextGenerator(random, Dictionaries.tags);
+		TextGenerator generator = new LdbcSnbTextGenerator(randomFarm_.get(RandomGeneratorFarm.Aspect.LARGE_TEXT), Dictionaries.tags);
         likeGenerator_ = new LikeGenerator();
         commentGenerator_ = new CommentGenerator(generator, likeGenerator_);
 		uniformPostGenerator_ = new UniformPostGenerator(generator, commentGenerator_, likeGenerator_);

src/main/java/ldbc/snb/datagen/generator/distribution/ZipfDistribution.java

@@ -15,7 +15,7 @@ public class ZipfDistribution implements DegreeDistribution {
 
     public void initialize(Configuration conf) {
         ALPHA_ = conf.getDouble("ldbc.snb.datagen.generator.distribution.ZipfDistribution.alpha",ALPHA_);
-        zipf_ = new org.apache.commons.math3.distribution.ZipfDistribution(10000, ALPHA_);
+        zipf_ = new org.apache.commons.math3.distribution.ZipfDistribution(1000, ALPHA_);
     }
 
     public void reset (long seed) {

src/main/resources/scale_factors.xml

@@ -374,4 +374,30 @@
             <value>false</value>
         </property>
     </scale_factor>
+    <scale_factor name="graphalytics.30000" >
+        <property>
+            <name>ldbc.snb.datagen.generator.numPersons</name>
+            <value>250000000</value>
+        </property>
+        <property>
+            <name>ldbc.snb.datagen.generator.startYear</name>
+            <value>2010</value>
+        </property>
+        <property>
+            <name>ldbc.snb.datagen.generator.numYears</name>
+            <value>1</value>
+        </property>
+        <property>
+            <name>ldbc.snb.datagen.generator.distribution.degreeDistribution</name>
+            <value>ldbc.snb.datagen.generator.distribution.FacebookDegreeDistribution</value>
+        </property>
+        <property>
+            <name>ldbc.snb.datagen.generator.activity</name>
+            <value>false</value>
+        </property>
+        <property>
+            <name>ldbc.snb.datagen.serializer.updateStreams</name>
+            <value>false</value>
+        </property>
+    </scale_factor>
 </scale_factors>