Made like generation more memory efficient

Author: ArnauPrat

src/main/java/ldbc/snb/datagen/generator/CommentGenerator.java

@@ -29,9 +29,9 @@ public class CommentGenerator {
 	public CommentGenerator(TextGenerator generator, LikeGenerator likeGenerator){
 		this.generator = generator;
 		this.likeGenerator_ = likeGenerator;
-	}	
+	}
 
-	public long createComments(RandomGeneratorFarm randomFarm, Forum forum, Post post, long numComments, long startId, PersonActivityExporter exporter){
+	public long createComments(RandomGeneratorFarm randomFarm, final Forum forum, final Post post, long numComments, long startId, PersonActivityExporter exporter){
 		long nextId = startId;
 		ArrayList<Message> replyCandidates = new ArrayList<Message>();
 		replyCandidates.add(post);
@@ -98,6 +98,7 @@ public long createComments(RandomGeneratorFarm randomFarm, Forum forum, Post pos
 				}
 			}
 		}
+		replyCandidates.clear();
 		return nextId;
 	}
 

src/main/java/ldbc/snb/datagen/generator/FlashmobPostGenerator.java

@@ -100,7 +100,7 @@ private void populateForumFlashmobTags( Random randomNumPost, Forum forum ) {
 	    }
     }
 
-    protected PostGenerator.PostInfo generatePostInfo( Random randomTag, Random randomDate, Forum forum, ForumMembership membership ) {
+    protected PostGenerator.PostInfo generatePostInfo( Random randomTag, Random randomDate, final Forum forum, final ForumMembership membership ) {
 	    if( currentForum != forum.id() ) {
 		    populateForumFlashmobTags(randomTag,forum);
 		    currentForum = forum.id();

src/main/java/ldbc/snb/datagen/generator/LikeGenerator.java

@@ -23,14 +23,15 @@
 public class LikeGenerator {
 	private double maxNumberOfLikes;
 	private final PowerDistGenerator likesGenerator_;
+	private Like like;
 
-	
 
 	public LikeGenerator() {
 		likesGenerator_ = new PowerDistGenerator(1,DatagenParams.maxNumLike,0.07);
+		this.like = new Like();
 	}
 
-	public void generateLikes(Random random, Forum forum, Message message, LikeType type, PersonActivityExporter exporter) {
+	public void generateLikes(Random random, final Forum forum, final Message message, LikeType type, PersonActivityExporter exporter) {
 		int numMembers = forum.memberships().size();
 		int numLikes = likesGenerator_.getValue(random);
 		numLikes = numLikes >= numMembers ?  numMembers : numLikes;
@@ -44,7 +45,6 @@ public void generateLikes(Random random, Forum forum, Message message, LikeType
 			long minDate = message.creationDate() > memberships.get(startIndex+i).creationDate() ? message.creationDate() : membership.creationDate();
 			long date = Math.max(Dictionaries.dates.randomSevenDays(random),DatagenParams.deltaTime) + minDate;
 			if( date <= Dictionaries.dates.getEndDateTime() ) {
-				Like like = new Like();
 				like.user = membership.person().accountId();
 				like.userCreationDate = membership.person().creationDate();
 				like.messageId = message.messageId();

src/main/java/ldbc/snb/datagen/generator/PersonActivityGenerator.java

@@ -49,8 +49,11 @@ public PersonActivityGenerator( PersonActivitySerializer serializer, UpdateEvent
 	}
 
 	private void generateActivity( Person person, ArrayList<Person> block ) {
+        System.out.println("Generating wall");
 		generateWall(person, block);
+        System.out.println("Generating groups");
 		generateGroups(person, block);
+        System.out.println("Generating albums");
 		generateAlbums(person, block);
         if(person.creationDate() < Dictionaries.dates.getUpdateThreshold() || !DatagenParams.updateStreams ) {
             factorTable_.extractFactors(person);

src/main/java/ldbc/snb/datagen/generator/PhotoGenerator.java

@@ -27,7 +27,7 @@ public class PhotoGenerator {
 	public PhotoGenerator(LikeGenerator likeGenerator) {
 		this.likeGenerator_ = likeGenerator;
 	}
-	public long createPhotos(RandomGeneratorFarm randomFarm, Forum album, ArrayList<ForumMembership> memberships, long numPhotos, long startId, PersonActivityExporter exporter){
+	public long createPhotos(RandomGeneratorFarm randomFarm, final Forum album, final ArrayList<ForumMembership> memberships, long numPhotos, long startId, PersonActivityExporter exporter){
 		long nextId = startId;
 		ArrayList<Photo> photos = new ArrayList<Photo>();
 		int numPopularPlaces = randomFarm.get(RandomGeneratorFarm.Aspect.NUM_POPULAR).nextInt(DatagenParams.maxNumPopularPlaces + 1);

src/main/java/ldbc/snb/datagen/generator/PostGenerator.java

@@ -78,7 +78,7 @@ public void initialize() {
 	}
 
 
-	public long createPosts(RandomGeneratorFarm randomFarm, Forum forum, ArrayList<ForumMembership> memberships, long numPosts, long startId, PersonActivityExporter exporter){
+	public long createPosts(RandomGeneratorFarm randomFarm, final Forum forum, final ArrayList<ForumMembership> memberships, long numPosts, long startId, PersonActivityExporter exporter){
 		long postId = startId;
 		Properties prop = new Properties();
 		prop.setProperty("type","post");
@@ -123,5 +123,5 @@ public long createPosts(RandomGeneratorFarm randomFarm, Forum forum, ArrayList<F
 		return postId;
 	}
 
-	protected abstract PostInfo generatePostInfo( Random randomTag, Random randomDate, Forum forum, ForumMembership membership );
+	protected abstract PostInfo generatePostInfo( Random randomTag, Random randomDate, final Forum forum, final ForumMembership membership );
 }

src/main/java/ldbc/snb/datagen/generator/UniformPostGenerator.java

@@ -25,7 +25,7 @@ public UniformPostGenerator(TextGenerator generator, CommentGenerator commentGen
 	}
 
 	@Override
-    protected PostInfo generatePostInfo( Random randomTag, Random randomDate, Forum forum, ForumMembership membership ) {
+    protected PostInfo generatePostInfo( Random randomTag, Random randomDate, final Forum forum, final ForumMembership membership ) {
 	    PostInfo postInfo = new PostInfo();
 	    postInfo.tags = new TreeSet<Integer>();
 	    Iterator<Integer> it = forum.tags().iterator();

src/main/java/ldbc/snb/datagen/hadoop/HadoopPersonActivityGenerator.java

@@ -143,10 +143,10 @@ public void run( String inputFileName ) throws Exception {
         job.setPartitionerClass(HadoopBlockPartitioner.class);
 
         /** PROFILING OPTIONS **/
-        job.setProfileEnabled(true);
-        job.setProfileParams("-agentlib:hprof=cpu=samples,heap=sites,depth=4,thread=y,format=b,file=%s");
-        job.setProfileTaskRange(true,"0-1");
-        job.setProfileTaskRange(false,"0-1");
+        //job.setProfileEnabled(true);
+        //job.setProfileParams("-agentlib:hprof=cpu=samples,heap=sites,depth=4,thread=y,format=b,file=%s");
+        //job.setProfileTaskRange(true,"0-1");
+        //job.setProfileTaskRange(false,"0-1");
         /****/
 
         FileInputFormat.setInputPaths(job, new Path(rankedFileName));

src/main/java/ldbc/snb/datagen/objects/Comment.java

@@ -46,6 +46,10 @@ public class Comment extends Message {
     private long postId_;
     private long replyOf_;
 
+    public Comment() {
+        super();
+    }
+
     public Comment(long commentId,
                    long creationDate,
                    PersonSummary author,
@@ -63,6 +67,21 @@ public Comment(long commentId,
         replyOf_ = replyOf;
     }
 
+    public void initialize(long commentId,
+                   long creationDate,
+                   PersonSummary author,
+                   long forumId,
+                   String content,
+                   TreeSet<Integer> tags,
+                   IP ipAddress,
+                   int browserId,
+                   long postId,
+                   long replyOf) {
+        super.initialize(commentId, creationDate, author, forumId, content, tags, ipAddress, browserId);
+        postId_ = postId;
+        replyOf_ = replyOf;
+    }
+
     public long postId() {
 	    return postId_;
     }

src/main/java/ldbc/snb/datagen/objects/Message.java

@@ -53,6 +53,11 @@ abstract public class Message {
     private int browserId_;
     private int countryId_;
 
+    public Message() {
+        tags_ = new TreeSet<Integer>();
+        ipAddress_ = new IP();
+    }
+
     public Message(long messageId,
                    long creationDate,
                    PersonSummary author,
@@ -73,6 +78,27 @@ public Message(long messageId,
         countryId_ = Dictionaries.ips.getLocation(ipAddress);
     }
 
+    public void initialize(long messageId,
+                   long creationDate,
+                   PersonSummary author,
+                   long forumId,
+                   String content,
+                   TreeSet<Integer> tags,
+                   IP ipAddress,
+                   int browserId
+                   ) {
+        messageId_ = messageId;
+        creationDate_ = creationDate;
+        author_ = new PersonSummary(author);
+        forumId_ = forumId;
+        content_ = content;
+        tags_.clear();
+        tags_.addAll(tags);
+        ipAddress_.copy(ipAddress);
+        browserId_ = browserId;
+        countryId_ = Dictionaries.ips.getLocation(ipAddress);
+    }
+
     public long messageId() {
 	    return messageId_;
     }