Merge remote-tracking branch 'origin/date_serializing' into bi

Author: ArnauPrat

.gitignore

@@ -4,6 +4,7 @@
 *.jar
 *.war
 *.ear
+*.pyc
 
 # IntelliJ stuff
 .idea
@@ -13,6 +14,7 @@ ldbc_socialnet_dbgen.iml
 target/
 .m0factors.txt.crc
 .m0friendList0.csv.crc
-paramgenerator/.pyc
 substitution_parameters/
 target/
+scripts/
+*.iml

:

@@ -0,0 +1,7 @@
+Merge branch 'bi' of github.com:ldbc/ldbc_snb_datagen into bi
+
+# Please enter a commit message to explain why this merge is necessary,
+# especially if it merges an updated upstream into a topic branch.
+#
+# Lines starting with '#' will be ignored, and an empty message aborts
+# the commit.

paramgenerator/discoverparams.pyc

@@ -1,15 +1,7 @@
-#ldbc.snb.datagen.generator.scaleFactor:snb.interactive.1
-ldbc.snb.datagen.generator.numPersons:1000
-ldbc.snb.datagen.generator.numYears:1
-ldbc.snb.datagen.generator.startYear:2010
-
+ ldbc.snb.datagen.generator.scaleFactor:snb.interactive.1
 
 ldbc.snb.datagen.serializer.compressed:false
-ldbc.snb.datagen.serializer.endlineSeparator:true
-ldbc.snb.datagen.serializer.numUpdatePartitions:8
 
 ldbc.snb.datagen.serializer.personSerializer:ldbc.snb.datagen.serializer.snb.interactive.CSVPersonSerializer
 ldbc.snb.datagen.serializer.invariantSerializer:ldbc.snb.datagen.serializer.snb.interactive.CSVInvariantSerializer
 ldbc.snb.datagen.serializer.personActivitySerializer:ldbc.snb.datagen.serializer.snb.interactive.CSVPersonActivitySerializer
-
-ldbc.snb.datagen.serializer.dateFormatter:ldbc.snb.datagen.serializer.formatter.LongDateFormatter

params.ini

@@ -1,6 +1,8 @@
 #!/bin/bash
-DEFAULT_HADOOP_HOME=/home/user/hadoop-2.6.0 #change to your hadoop folder
-DEFAULT_LDBC_SNB_DATAGEN_HOME=/home/user/ldbc_snb_datagen #change to your ldbc_socialnet_dbgen folder
+DEFAULT_HADOOP_HOME=/usr/local/
+DEFAULT_LDBC_SNB_DATAGEN_HOME=/Users/alexaverbuch/IdeaProjects/ldbc_snb_datagen
+#DEFAULT_HADOOP_HOME=/home/user/hadoop-2.6.0 #change to your hadoop folder
+#DEFAULT_LDBC_SNB_DATAGEN_HOME=/home/user/ldbc_snb_datagen_0.2 #change to your ldbc_socialnet_dbgen folder
 PARAM_GENERATION=1 #param generation
 
 # allow overriding configuration from outside via environment variables
@@ -12,15 +14,18 @@ LDBC_SNB_DATAGEN_HOME=${LDBC_SNB_DATAGEN_HOME:-$DEFAULT_LDBC_SNB_DATAGEN_HOME}
 
 export HADOOP_HOME
 export LDBC_SNB_DATAGEN_HOME
+export HADOOP_HEAPSIZE=6144
+export HADOOP_CLIENT_OPTS="-Xmx6g $HADOOP_CLIENT_OPTS"
 
 mvn clean
 mvn -DskipTests assembly:assembly 
 
 cp $LDBC_SNB_DATAGEN_HOME/target/ldbc_snb_datagen.jar $LDBC_SNB_DATAGEN_HOME/
 rm $LDBC_SNB_DATAGEN_HOME/target/ldbc_snb_datagen.jar
 
-$HADOOP_HOME/bin/hadoop  jar $LDBC_SNB_DATAGEN_HOME/ldbc_snb_datagen.jar $LDBC_SNB_DATAGEN_HOME/params.ini
+zip -d $LDBC_SNB_DATAGEN_HOME/ldbc_snb_datagen.jar META-INF/LICENSE
 
+$HADOOP_HOME/bin/hadoop jar $LDBC_SNB_DATAGEN_HOME/ldbc_snb_datagen.jar $LDBC_SNB_DATAGEN_HOME/params.ini
 
 if [ $PARAM_GENERATION -eq 1 ]
 then
@@ -33,4 +38,4 @@ then
   rm -f .m*activityFactors*
   rm -f m0friendList*
   rm -f .m0friendList*
-fi
+fi

run.sh

@@ -11,23 +11,27 @@
  * Created by aprat on 14/01/16.
  */
 public class LongDateFormatter implements DateFormatter {
-    private Date date_;
     private GregorianCalendar calendar_;
+    private int minHour;
+    private int minMinute;
+    private int minSecond;
+    private int minMillisecond;
+
     public void initialize(Configuration config) {
-        date_ = new Date();
         calendar_ = new GregorianCalendar(TimeZone.getTimeZone("GMT"));
-
+        minHour = calendar_.getActualMinimum( Calendar.HOUR );
+        minMinute = calendar_.getActualMinimum( Calendar.MINUTE );
+        minSecond = calendar_.getActualMinimum( Calendar.SECOND );
+        minMillisecond = calendar_.getActualMinimum( Calendar.MILLISECOND );
     }
 
     public String formatDate(long date) {
-        date_.setTime(date);
-        calendar_.setTime(date_);
-        int year = calendar_.get(Calendar.YEAR);
-        int month = calendar_.get(Calendar.MONTH);
-        int day = calendar_.get(Calendar.DAY_OF_MONTH);
-        calendar_.clear();
-        calendar_.set(year, month, day,0,0,0);
-        return Long.toString(calendar_.getTime().getTime());
+        calendar_.setTimeInMillis(date);
+        calendar_.set( Calendar.HOUR,minHour );
+        calendar_.set( Calendar.MINUTE,minMinute );
+        calendar_.set( Calendar.SECOND,minSecond );
+        calendar_.set( Calendar.MILLISECOND,minMillisecond);
+        return Long.toString(calendar_.getTimeInMillis());
     }
 
     public String formatDateTime(long date) {

src/main/java/ldbc/snb/datagen/serializer/formatter/LongDateFormatter.java

@@ -19,7 +19,6 @@ public class StringDateFormatter implements DateFormatter{
 
     private SimpleDateFormat gmtDateTimeFormatter_;
     private SimpleDateFormat gmtDateFormatter_;
-    private GregorianCalendar calendar_;
     private Date date_;
     public void initialize(Configuration conf) {