Skip to content

Commit 7c892e0

Browse files
committed
Fixed bug with parameter generation that was gnerating parameters not
belonging to the static part
1 parent eab8358 commit 7c892e0

File tree

2 files changed

+42
-1
lines changed

2 files changed

+42
-1
lines changed

src/main/java/ldbc/socialnet/dbgen/generator/MRGenerateUsers.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -540,7 +540,7 @@ public int runGenerateJob(Configuration conf) throws Exception {
540540
}
541541

542542
/// --------------- Sixth job: Materialize the friends lists ----------------
543-
Job job6 = new Job(conf,"Dump the friends lists");
543+
/* Job job6 = new Job(conf,"Dump the friends lists");
544544
job6.setMapOutputKeyClass(ComposedKey.class);
545545
job6.setMapOutputValueClass(ReducedUserProfile.class);
546546
job6.setOutputKeyClass(ComposedKey.class);
@@ -561,6 +561,7 @@ public int runGenerateJob(Configuration conf) throws Exception {
561561
printProgress("Starting: Materialize friends for substitution parameters");
562562
int resMaterializeFriends = job6.waitForCompletion(true) ? 0 : 1;
563563
fs.delete(new Path(hadoopDir + "/sibSorting3"),true);
564+
*/
564565

565566

566567
long end = System.currentTimeMillis();

src/main/java/ldbc/socialnet/dbgen/serializer/DataExporter.java

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,12 @@
4242
import ldbc.socialnet.dbgen.generator.ScalableGenerator;
4343
import ldbc.socialnet.dbgen.objects.*;
4444
import ldbc.socialnet.dbgen.serializer.CSVSerializer.CSVSerializer;
45+
import org.apache.hadoop.conf.Configuration;
46+
import org.apache.hadoop.fs.FSDataOutputStream;
47+
import org.apache.hadoop.fs.FileSystem;
48+
import org.apache.hadoop.fs.Path;
4549

50+
import java.io.IOException;
4651
import java.util.*;
4752

4853
/**
@@ -71,6 +76,7 @@ public enum DataFormat {
7176
private int startMonth, startYear;
7277
private int reducerId = 0;
7378
private long deltaTime = 0;
79+
FSDataOutputStream out;
7480
GregorianCalendar c;
7581

7682
public DataExporter( DataFormat format,
@@ -117,6 +123,14 @@ public DataExporter( DataFormat format,
117123
staticSerializer = new EmptySerializer();
118124
}
119125
updateStreamSerializer = new UpdateEventSerializer(directory,"temp_updateStream_"+reducerId,exportText, numPartitions,tagDic,browsers,languageDic,ipDic, statistics);
126+
127+
try {
128+
FileSystem fs = FileSystem.get(new Configuration());
129+
Path outFile = new Path(directory + "/m0friendList" + reducerId + ".csv");
130+
out = fs.create(outFile);
131+
} catch(IOException e ) {
132+
System.out.println(e.getMessage());
133+
}
120134
exportCommonEntities();
121135
}
122136

@@ -125,6 +139,12 @@ public void changePartition(){
125139
}
126140

127141
public void close() {
142+
try {
143+
out.flush();
144+
out.close();
145+
} catch ( IOException e ) {
146+
System.out.println(e.getMessage());
147+
}
128148
staticSerializer.close();
129149
updateStreamSerializer.close();
130150
}
@@ -224,6 +244,9 @@ public void exportTags() {
224244

225245
public void export( UserInfo userInfo ) {
226246
long creationDate = userInfo.user.getCreationDate();
247+
if(userInfo.user.getAccountId() == 18691698848287L) {
248+
System.out.println("creation date "+creationDate+" "+dateThreshold);
249+
}
227250
if( creationDate <= dateThreshold ) {
228251
staticSerializer.serialize(userInfo);
229252
} else {
@@ -263,21 +286,38 @@ public void export( UserInfo userInfo ) {
263286
}
264287
}
265288

289+
StringBuffer strbuf = new StringBuffer();
290+
strbuf.append(userInfo.user.getAccountId());
266291
Friend friends[] = userInfo.user.getFriendList();
267292
int numFriends = friends.length;
268293
for( int i = 0; i < numFriends; ++i ) {
269294
if (friends[i] != null && friends[i].getCreatedTime() != -1) {
270295
if( friends[i].getCreatedTime() <= dateThreshold ) {
296+
297+
if(userInfo.user.getAccountId() == 18691698848287L) {
298+
System.out.println("ENTRA friend creation date "+friends[i].getCreatedTime());
299+
}
271300
staticSerializer.serialize(friends[i]);
272301
if (!factorTable.containsKey(userInfo.user.getAccountId()))
273302
factorTable.put(userInfo.user.getAccountId(), new ReducedUserProfile.Counts());
274303
factorTable.get(userInfo.user.getAccountId()).numberOfFriends++;
304+
strbuf.append(",");
305+
strbuf.append(friends[i].getFriendAcc());
275306
} else {
276307
updateStreamSerializer.setCurrentDependantDate(friends[i].dependantDate);
277308
updateStreamSerializer.serialize(friends[i]);
278309
}
279310
}
280311
}
312+
strbuf.append("\n");
313+
314+
try {
315+
if(creationDate <= dateThreshold) {
316+
out.write(strbuf.toString().getBytes());
317+
}
318+
}catch(IOException e ) {
319+
System.out.println(e.getMessage());
320+
}
281321
}
282322

283323
public void export(Post post, long dependantDate) {

0 commit comments

Comments
 (0)