Skip to content

Commit 299bbfe

Browse files
ArnauPratArnauPrat
committed
Changed person activity architecture
1 parent f90fed4 commit 299bbfe

File tree

8 files changed

+152
-181
lines changed

8 files changed

+152
-181
lines changed

‎src/main/java/ldbc/snb/datagen/generator/CommentGenerator.java

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77

88
import ldbc.snb.datagen.dictionary.Dictionaries;
99
import ldbc.snb.datagen.objects.*;
10+
import ldbc.snb.datagen.serializer.PersonActivityExporter;
1011
import ldbc.snb.datagen.util.RandomGeneratorFarm;
1112
import ldbc.snb.datagen.vocabulary.SN;
1213

@@ -22,15 +23,16 @@
2223
public class CommentGenerator {
2324
private String[] shortComments_ = {"ok", "good", "great", "cool", "thx", "fine", "LOL", "roflol", "no way!", "I see", "right", "yes", "no", "duh", "thanks", "maybe"};
2425
private TextGenerator generator;
26+
private LikeGenerator likeGenerator_;
2527
/* A set of random number generator for different purposes.*/
2628

27-
public CommentGenerator(TextGenerator generator){
29+
public CommentGenerator(TextGenerator generator, LikeGenerator likeGenerator){
2830
this.generator = generator;
31+
this.likeGenerator_ = likeGenerator;
2932
}
3033

31-
public ArrayList<Comment> createComments(RandomGeneratorFarm randomFarm, Forum forum, Post post, long numComments, long startId ){
34+
public long createComments(RandomGeneratorFarm randomFarm, Forum forum, Post post, long numComments, long startId, PersonActivityExporter exporter){
3235
long nextId = startId;
33-
ArrayList<Comment> result = new ArrayList<Comment>();
3436
ArrayList<Message> replyCandidates = new ArrayList<Message>();
3537
replyCandidates.add(post);
3638

@@ -46,7 +48,7 @@ public ArrayList<Comment> createComments(RandomGeneratorFarm randomFarm, Forum f
4648
}
4749
}
4850
if (validMemberships.size() == 0) {
49-
return result;
51+
return nextId;
5052
}
5153
ForumMembership member = validMemberships.get(randomFarm.get(RandomGeneratorFarm.Aspect.MEMBERSHIP_INDEX).nextInt(validMemberships.size()));
5254
TreeSet<Integer> tags = new TreeSet<Integer>();
@@ -89,11 +91,15 @@ public ArrayList<Comment> createComments(RandomGeneratorFarm randomFarm, Forum f
8991
Dictionaries.browsers.getPostBrowserId(randomFarm.get(RandomGeneratorFarm.Aspect.DIFF_BROWSER), randomFarm.get(RandomGeneratorFarm.Aspect.BROWSER), member.person().browserId()),
9092
post.messageId(),
9193
replyTo.messageId());
92-
result.add(comment);
9394
if(!isShort) replyCandidates.add(comment);
95+
96+
exporter.export(comment);
97+
if( comment.content().length() > 10 && randomFarm.get(RandomGeneratorFarm.Aspect.NUM_LIKE).nextDouble() <= 0.1 ) {
98+
likeGenerator_.generateLikes(randomFarm.get(RandomGeneratorFarm.Aspect.NUM_LIKE), forum, comment, Like.LikeType.COMMENT, exporter);
99+
}
94100
}
95101
}
96-
return result;
102+
return nextId;
97103
}
98104

99105
}

‎src/main/java/ldbc/snb/datagen/generator/FlashmobPostGenerator.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,8 @@ public class FlashmobPostGenerator extends PostGenerator{
2525
private long hoursToMillis_;
2626
private long flashmobSpan_;
2727
private long currentForum = -1;
28-
public FlashmobPostGenerator(TextGenerator generator) {
29-
super(generator);
28+
public FlashmobPostGenerator(TextGenerator generator, CommentGenerator commentGenerator, LikeGenerator likeGenerator) {
29+
super(generator, commentGenerator, likeGenerator);
3030
dateDistribution_ = new Distribution(DatagenParams.flashmobDistFile);
3131
hoursToMillis_ = 60*60*1000;
3232
flashmobSpan_ = 72 * hoursToMillis_;

‎src/main/java/ldbc/snb/datagen/generator/LikeGenerator.java

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
import ldbc.snb.datagen.objects.Like;
1212
import ldbc.snb.datagen.objects.Like.LikeType;
1313
import ldbc.snb.datagen.objects.Message;
14+
import ldbc.snb.datagen.serializer.PersonActivityExporter;
1415

1516
import java.util.ArrayList;
1617
import java.util.Random;
@@ -29,8 +30,7 @@ public LikeGenerator() {
2930
likesGenerator_ = new PowerDistGenerator(1,DatagenParams.maxNumLike,0.07);
3031
}
3132

32-
public ArrayList<Like> generateLikes( Random random, Forum forum, Message message, LikeType type ) {
33-
ArrayList<Like> likes = new ArrayList<Like>();
33+
public void generateLikes(Random random, Forum forum, Message message, LikeType type, PersonActivityExporter exporter) {
3434
int numMembers = forum.memberships().size();
3535
int numLikes = likesGenerator_.getValue(random);
3636
numLikes = numLikes >= numMembers ? numMembers : numLikes;
@@ -50,9 +50,9 @@ public ArrayList<Like> generateLikes( Random random, Forum forum, Message messag
5050
like.messageId = message.messageId();
5151
like.date = date;
5252
like.type = type;
53-
likes.add(like);
53+
exporter.export(like);
5454
}
5555
}
56-
return likes;
56+
return;
5757
}
5858
}

‎src/main/java/ldbc/snb/datagen/generator/PersonActivityGenerator.java

Lines changed: 19 additions & 136 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33

44
import ldbc.snb.datagen.dictionary.Dictionaries;
55
import ldbc.snb.datagen.objects.*;
6+
import ldbc.snb.datagen.serializer.PersonActivityExporter;
67
import ldbc.snb.datagen.serializer.PersonActivitySerializer;
78
import ldbc.snb.datagen.serializer.UpdateEventSerializer;
89
import ldbc.snb.datagen.util.FactorTable;
@@ -29,23 +30,22 @@ public class PersonActivityGenerator {
2930
private long forumId = 0;
3031
private long messageId = 0;
3132
private FactorTable factorTable_;
33+
private PersonActivityExporter exporter_;
3234

3335
public PersonActivityGenerator( PersonActivitySerializer serializer, UpdateEventSerializer updateSerializer ) {
3436
personActivitySerializer_ = serializer;
3537
updateSerializer_ = updateSerializer;
3638
randomFarm_ = new RandomGeneratorFarm();
37-
// load generators
3839
forumGenerator_ = new ForumGenerator();
39-
//david
4040
Random random = new Random();
4141
TextGenerator generator = new LdbcSnbTextGenerator(random, Dictionaries.tags);
42-
//d: end
43-
uniformPostGenerator_ = new UniformPostGenerator(generator);
44-
flashmobPostGenerator_ = new FlashmobPostGenerator(generator);
45-
photoGenerator_ = new PhotoGenerator();
46-
commentGenerator_ = new CommentGenerator(generator);
47-
likeGenerator_ = new LikeGenerator();
42+
likeGenerator_ = new LikeGenerator();
43+
commentGenerator_ = new CommentGenerator(generator, likeGenerator_);
44+
uniformPostGenerator_ = new UniformPostGenerator(generator, commentGenerator_, likeGenerator_);
45+
flashmobPostGenerator_ = new FlashmobPostGenerator(generator, commentGenerator_, likeGenerator_);
46+
photoGenerator_ = new PhotoGenerator(likeGenerator_);
4847
factorTable_ = new FactorTable();
48+
exporter_ = new PersonActivityExporter(personActivitySerializer_, updateSerializer_, factorTable_);
4949
}
5050

5151
private void generateActivity( Person person, ArrayList<Person> block ) {
@@ -70,9 +70,9 @@ public void reset() {
7070
private void generateWall( Person person, ArrayList<Person> block ) {
7171
// generate wall
7272
Forum wall = forumGenerator_.createWall(randomFarm_, forumId++, person);
73-
export(wall);
73+
exporter_.export(wall);
7474
for( ForumMembership fm : wall.memberships()) {
75-
export(fm);
75+
exporter_.export(fm);
7676
}
7777

7878
// generate wall posts
@@ -81,36 +81,8 @@ private void generateWall( Person person, ArrayList<Person> block ) {
8181
);
8282
ArrayList<ForumMembership> fakeMembers = new ArrayList<ForumMembership>();
8383
fakeMembers.add(personMembership);
84-
ArrayList<Post> wallPosts = uniformPostGenerator_.createPosts(randomFarm_, wall, fakeMembers , numPostsPerGroup(randomFarm_, wall, DatagenParams.maxNumPostPerMonth, DatagenParams.maxNumFriends), messageId);
85-
long aux = messageId + wallPosts.size();
86-
wallPosts.addAll(flashmobPostGenerator_.createPosts(randomFarm_, wall, fakeMembers, numPostsPerGroup(randomFarm_, wall, DatagenParams.maxNumFlashmobPostPerMonth, DatagenParams.maxNumFriends), aux ));
87-
messageId+=wallPosts.size();
88-
89-
for( Post p : wallPosts ) {
90-
export(p);
91-
// generate likes to post
92-
if( randomFarm_.get(RandomGeneratorFarm.Aspect.NUM_LIKE).nextDouble() <= 0.1 ) {
93-
ArrayList<Like> postLikes = likeGenerator_.generateLikes(randomFarm_.get(RandomGeneratorFarm.Aspect.NUM_LIKE), wall, p, Like.LikeType.POST);
94-
for( Like l : postLikes ) {
95-
export(l);
96-
}
97-
}
98-
//// generate comments
99-
int numComments = randomFarm_.get(RandomGeneratorFarm.Aspect.NUM_COMMENT).nextInt(DatagenParams.maxNumComments+1);
100-
ArrayList<Comment> comments = commentGenerator_.createComments(randomFarm_, wall, p, numComments, messageId);
101-
messageId+=comments.size();
102-
for( Comment c : comments ) {
103-
export(c);
104-
// generate likes to comments
105-
if( c.content().length() > 10 && randomFarm_.get(RandomGeneratorFarm.Aspect.NUM_LIKE).nextDouble() <= 0.1 ) {
106-
ArrayList<Like> commentLikes = likeGenerator_.generateLikes(randomFarm_.get(RandomGeneratorFarm.Aspect.NUM_LIKE), wall, c, Like.LikeType.COMMENT);
107-
for( Like l : commentLikes ) {
108-
export(l);
109-
}
110-
}
111-
}
112-
}
113-
84+
messageId = uniformPostGenerator_.createPosts(randomFarm_, wall, fakeMembers , numPostsPerGroup(randomFarm_, wall, DatagenParams.maxNumPostPerMonth, DatagenParams.maxNumFriends), messageId, exporter_);
85+
messageId = flashmobPostGenerator_.createPosts(randomFarm_, wall, fakeMembers, numPostsPerGroup(randomFarm_, wall, DatagenParams.maxNumFlashmobPostPerMonth, DatagenParams.maxNumFriends), messageId, exporter_ );
11486
}
11587

11688
private void generateGroups( Person person, ArrayList<Person> block ) {
@@ -120,41 +92,15 @@ private void generateGroups( Person person, ArrayList<Person> block ) {
12092
int numGroup = randomFarm_.get(RandomGeneratorFarm.Aspect.NUM_FORUM).nextInt(DatagenParams.maxNumGroupCreatedPerUser)+1;
12193
for (int j = 0; j < numGroup; j++) {
12294
Forum group = forumGenerator_.createGroup(randomFarm_, forumId++, person, block);
123-
export(group);
95+
exporter_.export(group);
12496

12597
for( ForumMembership fm : group.memberships()) {
126-
export(fm);
98+
exporter_.export(fm);
12799
}
128100

129101
// generate uniform posts/comments
130-
ArrayList<Post> groupPosts = uniformPostGenerator_.createPosts(randomFarm_, group, group.memberships(), numPostsPerGroup(randomFarm_, group, DatagenParams.maxNumGroupPostPerMonth, DatagenParams.maxNumMemberGroup), messageId);
131-
long aux = messageId+groupPosts.size();
132-
groupPosts.addAll(flashmobPostGenerator_.createPosts(randomFarm_, group, group.memberships(), numPostsPerGroup(randomFarm_, group, DatagenParams.maxNumGroupFlashmobPostPerMonth, DatagenParams.maxNumMemberGroup),aux));
133-
messageId += groupPosts.size();
134-
for( Post p : groupPosts ) {
135-
export(p);
136-
// generate likes to post
137-
if( randomFarm_.get(RandomGeneratorFarm.Aspect.NUM_LIKE).nextDouble() <= 0.1 ) {
138-
ArrayList<Like> postLikes = likeGenerator_.generateLikes(randomFarm_.get(RandomGeneratorFarm.Aspect.NUM_LIKE), group, p, Like.LikeType.POST);
139-
for( Like l : postLikes ) {
140-
export(l);
141-
}
142-
}
143-
// generate comments
144-
int numComments = randomFarm_.get(RandomGeneratorFarm.Aspect.NUM_COMMENT).nextInt(DatagenParams.maxNumComments+1);
145-
ArrayList<Comment> comments = commentGenerator_.createComments(randomFarm_, group, p, numComments, messageId);
146-
messageId+=comments.size();
147-
for( Comment c : comments ) {
148-
export(c);
149-
// generate likes to comments
150-
if( c.content().length() > 10 && randomFarm_.get(RandomGeneratorFarm.Aspect.NUM_LIKE).nextDouble() <= 0.1 ) {
151-
ArrayList<Like> commentLikes = likeGenerator_.generateLikes(randomFarm_.get(RandomGeneratorFarm.Aspect.NUM_LIKE), group, c, Like.LikeType.COMMENT);
152-
for( Like l : commentLikes ) {
153-
export(l);
154-
}
155-
}
156-
}
157-
}
102+
messageId = uniformPostGenerator_.createPosts(randomFarm_, group, group.memberships(), numPostsPerGroup(randomFarm_, group, DatagenParams.maxNumGroupPostPerMonth, DatagenParams.maxNumMemberGroup), messageId, exporter_);
103+
messageId = flashmobPostGenerator_.createPosts(randomFarm_, group, group.memberships(), numPostsPerGroup(randomFarm_, group, DatagenParams.maxNumGroupFlashmobPostPerMonth, DatagenParams.maxNumMemberGroup),messageId, exporter_);
158104
}
159105
}
160106

@@ -169,10 +115,10 @@ private void generateAlbums(Person person, ArrayList<Person> block ) {
169115
}
170116
for (int i = 0; i < numPhotoAlbums; i++) {
171117
Forum album = forumGenerator_.createAlbum(randomFarm_, forumId++, person, i);
172-
export(album);
118+
exporter_.export(album);
173119

174120
for( ForumMembership fm : album.memberships()) {
175-
export(fm);
121+
exporter_.export(fm);
176122
}
177123

178124
ForumMembership personMembership = new ForumMembership(album.id(),
@@ -181,18 +127,7 @@ private void generateAlbums(Person person, ArrayList<Person> block ) {
181127
ArrayList<ForumMembership> fakeMembers = new ArrayList<ForumMembership>();
182128
fakeMembers.add(personMembership);
183129
int numPhotos = randomFarm_.get(RandomGeneratorFarm.Aspect.NUM_PHOTO).nextInt(DatagenParams.maxNumPhotoPerAlbums+1);
184-
ArrayList<Photo> photos = photoGenerator_.createPhotos(randomFarm_, album, fakeMembers, numPhotos, messageId);
185-
messageId+=photos.size();
186-
for( Photo p : photos ) {
187-
export(p);
188-
// generate likes
189-
if( randomFarm_.get(RandomGeneratorFarm.Aspect.NUM_LIKE).nextDouble() <= 0.1 ) {
190-
ArrayList<Like> photoLikes = likeGenerator_.generateLikes(randomFarm_.get(RandomGeneratorFarm.Aspect.NUM_LIKE), album, p, Like.LikeType.PHOTO);
191-
for( Like l : photoLikes ) {
192-
export(l);
193-
}
194-
}
195-
}
130+
messageId = photoGenerator_.createPhotos(randomFarm_, album, fakeMembers, numPhotos, messageId, exporter_);
196131
}
197132
}
198133

@@ -237,58 +172,6 @@ public void generateActivityForBlock( int seed, ArrayList<Person> block, Context
237172
System.out.println("Average person activity generation time "+personGenerationTime / (float)block.size());
238173
}
239174

240-
private void export(Forum forum) {
241-
if(forum.creationDate() < Dictionaries.dates.getUpdateThreshold() || !DatagenParams.updateStreams ) {
242-
personActivitySerializer_.export(forum);
243-
} else {
244-
updateSerializer_.export(forum);
245-
}
246-
}
247-
248-
private void export(Post post) {
249-
if(post.creationDate() < Dictionaries.dates.getUpdateThreshold() || !DatagenParams.updateStreams ) {
250-
personActivitySerializer_.export(post);
251-
factorTable_.extractFactors(post);
252-
} else {
253-
updateSerializer_.export(post);
254-
}
255-
}
256-
257-
private void export(Comment comment) {
258-
if(comment.creationDate() < Dictionaries.dates.getUpdateThreshold() || !DatagenParams.updateStreams ) {
259-
personActivitySerializer_.export(comment);
260-
factorTable_.extractFactors(comment);
261-
} else {
262-
updateSerializer_.export(comment);
263-
}
264-
}
265-
266-
private void export(Photo photo) {
267-
if(photo.creationDate() < Dictionaries.dates.getUpdateThreshold() || !DatagenParams.updateStreams ) {
268-
personActivitySerializer_.export(photo);
269-
factorTable_.extractFactors(photo);
270-
} else {
271-
updateSerializer_.export(photo);
272-
}
273-
}
274-
275-
private void export(ForumMembership member) {
276-
if(member.creationDate() < Dictionaries.dates.getUpdateThreshold() || !DatagenParams.updateStreams ) {
277-
personActivitySerializer_.export(member);
278-
factorTable_.extractFactors(member);
279-
} else {
280-
updateSerializer_.export(member);
281-
}
282-
}
283-
284-
private void export(Like like) {
285-
if(like.date < Dictionaries.dates.getUpdateThreshold() || !DatagenParams.updateStreams ) {
286-
personActivitySerializer_.export(like);
287-
factorTable_.extractFactors(like);
288-
} else {
289-
updateSerializer_.export(like);
290-
}
291-
}
292175

293176
public void writeFactors( OutputStream writer) {
294177
factorTable_.write(writer);

‎src/main/java/ldbc/snb/datagen/generator/PhotoGenerator.java

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,8 @@
66
package ldbc.snb.datagen.generator;
77

88
import ldbc.snb.datagen.dictionary.Dictionaries;
9-
import ldbc.snb.datagen.objects.Forum;
10-
import ldbc.snb.datagen.objects.ForumMembership;
11-
import ldbc.snb.datagen.objects.Photo;
12-
import ldbc.snb.datagen.objects.PopularPlace;
9+
import ldbc.snb.datagen.objects.*;
10+
import ldbc.snb.datagen.serializer.PersonActivityExporter;
1311
import ldbc.snb.datagen.util.RandomGeneratorFarm;
1412
import ldbc.snb.datagen.vocabulary.SN;
1513

@@ -22,11 +20,14 @@
2220
*/
2321
public class PhotoGenerator {
2422
private long postId = 0;
23+
private LikeGenerator likeGenerator_;
24+
2525
private static final String SEPARATOR = " ";
2626

27-
public PhotoGenerator() {
27+
public PhotoGenerator(LikeGenerator likeGenerator) {
28+
this.likeGenerator_ = likeGenerator;
2829
}
29-
public ArrayList<Photo> createPhotos(RandomGeneratorFarm randomFarm, Forum album, ArrayList<ForumMembership> memberships, long numPhotos, long startId ){
30+
public long createPhotos(RandomGeneratorFarm randomFarm, Forum album, ArrayList<ForumMembership> memberships, long numPhotos, long startId, PersonActivityExporter exporter){
3031
long nextId = startId;
3132
ArrayList<Photo> photos = new ArrayList<Photo>();
3233
int numPopularPlaces = randomFarm.get(RandomGeneratorFarm.Aspect.NUM_POPULAR).nextInt(DatagenParams.maxNumPopularPlaces + 1);
@@ -77,10 +78,13 @@ public ArrayList<Photo> createPhotos(RandomGeneratorFarm randomFarm, Forum album
7778
if( date <= Dictionaries.dates.getEndDateTime() ) {
7879
long id = SN.formId(SN.composeId(nextId++,date));
7980
Photo photo = new Photo(id,date,album.moderator(), album.id(), "photo"+id+".jpg",tags,album.moderator().ipAddress(),album.moderator().browserId(),latt,longt);
80-
photos.add(photo);
81+
exporter.export(photo);
82+
if( randomFarm.get(RandomGeneratorFarm.Aspect.NUM_LIKE).nextDouble() <= 0.1 ) {
83+
likeGenerator_.generateLikes(randomFarm.get(RandomGeneratorFarm.Aspect.NUM_LIKE), album, photo, Like.LikeType.PHOTO, exporter);
84+
}
8185
}
8286
}
83-
return photos;
87+
return nextId;
8488
}
8589

8690
}

0 commit comments

Comments
 (0)