Skip to content

Commit cff73f8

Browse files
miratepuffinmiratepuffin
committed
Refactored CSVCompositeDynamicPersonSerializer
1 parent f70408d commit cff73f8

File tree

2 files changed

+31
-126
lines changed

2 files changed

+31
-126
lines changed

‎src/main/java/ldbc/snb/datagen/serializer/DynamicPersonSerializer.java

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,6 @@ Linked Data Benchmark Council (http://www.ldbcouncil.org)
4141
import ldbc.snb.datagen.entities.dynamic.relations.StudyAt;
4242
import ldbc.snb.datagen.entities.dynamic.relations.WorkAt;
4343
import org.apache.hadoop.conf.Configuration;
44-
4544
import java.io.IOException;
4645
import java.util.Iterator;
4746

‎src/main/java/ldbc/snb/datagen/serializer/snb/csv/dynamicserializer/person/CSVCompositeDynamicPersonSerializer.java

Lines changed: 31 additions & 125 deletions
Original file line numberDiff line numberDiff line change
@@ -37,127 +37,53 @@ Linked Data Benchmark Council (http://www.ldbcouncil.org)
3737

3838
package ldbc.snb.datagen.serializer.snb.csv.dynamicserializer.person;
3939

40+
import com.google.common.collect.ImmutableList;
4041
import ldbc.snb.datagen.dictionary.Dictionaries;
4142
import ldbc.snb.datagen.entities.dynamic.relations.Knows;
4243
import ldbc.snb.datagen.entities.dynamic.person.Person;
4344
import ldbc.snb.datagen.entities.dynamic.relations.StudyAt;
4445
import ldbc.snb.datagen.entities.dynamic.relations.WorkAt;
4546
import ldbc.snb.datagen.hadoop.writer.HDFSCSVWriter;
4647
import ldbc.snb.datagen.serializer.DynamicPersonSerializer;
48+
import ldbc.snb.datagen.serializer.snb.csv.FileName;
4749
import org.apache.hadoop.conf.Configuration;
48-
50+
import static ldbc.snb.datagen.serializer.snb.csv.FileName.*;
51+
import ldbc.snb.datagen.serializer.snb.csv.FileName;
4952
import java.io.IOException;
5053
import java.util.ArrayList;
54+
import java.util.Arrays;
5155
import java.util.Iterator;
56+
import java.util.List;
5257

5358
public class CSVCompositeDynamicPersonSerializer extends DynamicPersonSerializer {
5459

55-
private HDFSCSVWriter[] writers;
56-
57-
private enum FileNames {
58-
PERSON("person"),
59-
PERSON_LOCATED_IN_PLACE("person_isLocatedIn_place"),
60-
PERSON_HAS_INTEREST_TAG("person_hasInterest_tag"),
61-
PERSON_WORK_AT("person_workAt_organisation"),
62-
PERSON_STUDY_AT("person_studyAt_organisation"),
63-
PERSON_KNOWS_PERSON("person_knows_person");
64-
65-
private final String name;
66-
67-
private FileNames(String name) {
68-
this.name = name;
69-
}
70-
71-
public String toString() {
72-
return name;
73-
}
74-
}
7560

7661
@Override
77-
public void initialize(Configuration conf, int reducerId) throws IOException {
78-
int numFiles = FileNames.values().length;
79-
writers = new HDFSCSVWriter[numFiles];
80-
for (int i = 0; i < numFiles; ++i) {
81-
writers[i] = new HDFSCSVWriter(conf.get("ldbc.snb.datagen.serializer.socialNetworkDir")+"/dynamic/", FileNames
82-
.values()[i].toString() + "_" + reducerId, conf
83-
.getInt("ldbc.snb.datagen.serializer.numPartitions", 1), conf
84-
.getBoolean("ldbc.snb.datagen.serializer.compressed", false), "|", conf
85-
.getBoolean("ldbc.snb.datagen.serializer.endlineSeparator", false));
86-
}
87-
88-
ArrayList<String> arguments = new ArrayList<String>();
89-
arguments.add("id");
90-
arguments.add("firstName");
91-
arguments.add("lastName");
92-
arguments.add("gender");
93-
arguments.add("birthday");
94-
arguments.add("creationDate");
95-
arguments.add("locationIP");
96-
arguments.add("browserUsed");
97-
arguments.add("language");
98-
arguments.add("email");
99-
writers[FileNames.PERSON.ordinal()].writeHeader(arguments);
100-
101-
arguments.clear();
102-
arguments.add("Person.id");
103-
arguments.add("Place.id");
104-
writers[FileNames.PERSON_LOCATED_IN_PLACE.ordinal()].writeHeader(arguments);
105-
106-
arguments.clear();
107-
arguments.add("Person.id");
108-
arguments.add("Tag.id");
109-
writers[FileNames.PERSON_HAS_INTEREST_TAG.ordinal()].writeHeader(arguments);
110-
111-
arguments.clear();
112-
arguments.add("Person.id");
113-
arguments.add("Organisation.id");
114-
arguments.add("workFrom");
115-
writers[FileNames.PERSON_WORK_AT.ordinal()].writeHeader(arguments);
116-
117-
arguments.clear();
118-
arguments.add("Person.id");
119-
arguments.add("Organisation.id");
120-
arguments.add("classYear");
121-
writers[FileNames.PERSON_STUDY_AT.ordinal()].writeHeader(arguments);
122-
123-
arguments.clear();
124-
arguments.add("Person.id");
125-
arguments.add("Person.id");
126-
arguments.add("creationDate");
127-
writers[FileNames.PERSON_KNOWS_PERSON.ordinal()].writeHeader(arguments);
128-
62+
public List<FileName> getFileNames() {
63+
return Arrays.asList(PERSON,PERSON_LOCATED_IN_PLACE,PERSON_HAS_INTEREST_TAG,
64+
PERSON_WORK_AT,PERSON_STUDY_AT,PERSON_KNOWS_PERSON)
12965
}
13066

13167
@Override
132-
public void close() {
133-
int numFiles = FileNames.values().length;
134-
for (int i = 0; i < numFiles; ++i) {
135-
writers[i].close();
136-
}
68+
public void writeFileHeaders() {
69+
writers.get(PERSON).writeHeader(ImmutableList.of("id","firstName","lastName","gender",
70+
"birthday","creationDate","locationIP","browserUsed","language","email"));
71+
writers.get(PERSON_LOCATED_IN_PLACE).writeHeader(ImmutableList.of("Person.id","Place.id"));
72+
writers.get(PERSON_HAS_INTEREST_TAG).writeHeader(ImmutableList.of("Person.id","Tag.id"));
73+
writers.get(PERSON_WORK_AT).writeHeader(ImmutableList.of("Person.id","Organisation.id","workFrom"));
74+
writers.get(PERSON_STUDY_AT).writeHeader(ImmutableList.of("Person.id","Organisation.id","classYear"));
75+
writers.get(PERSON_KNOWS_PERSON).writeHeader(ImmutableList.of("Person.id","Person.id","creationDate"));
76+
13777
}
13878

13979
@Override
14080
protected void serialize(final Person p) {
141-
142-
ArrayList<String> arguments = new ArrayList<String>();
143-
144-
arguments.add(Long.toString(p.accountId()));
145-
arguments.add(p.firstName());
146-
arguments.add(p.lastName());
81+
String gender = "";
14782
if (p.gender() == 1) {
148-
arguments.add("male");
83+
gender ="male";
14984
} else {
150-
arguments.add("female");
85+
gender = "female";
15186
}
152-
153-
String dateString = Dictionaries.dates.formatDate(p.birthday());
154-
arguments.add(dateString);
155-
156-
dateString = Dictionaries.dates.formatDateTime(p.creationDate());
157-
arguments.add(dateString);
158-
arguments.add(p.ipAddress().toString());
159-
arguments.add(Dictionaries.browsers.getName(p.browserId()));
160-
16187
ArrayList<Integer> languages = p.languages();
16288
StringBuilder languagesBuilder = new StringBuilder();
16389
for (int i = 0; i < languages.size()-1; i++) {
@@ -166,8 +92,6 @@ protected void serialize(final Person p) {
16692
if(languages.size() > 0) {
16793
languagesBuilder.append(Dictionaries.languages.getLanguageName(languages.get(languages.size()-1)));
16894
}
169-
arguments.add(languagesBuilder.toString());
170-
17195
StringBuilder emailsBuilder = new StringBuilder();
17296
Iterator<String> itString = p.emails().iterator();
17397
for (int i = 0; i < p.emails().size()-1; i++) {
@@ -176,53 +100,35 @@ protected void serialize(final Person p) {
176100
if(itString.hasNext()) {
177101
emailsBuilder.append(itString.next());
178102
}
179-
arguments.add(emailsBuilder.toString());
180103

181-
writers[FileNames.PERSON.ordinal()].writeEntry(arguments);
104+
writers.get(PERSON).writeEntry(ImmutableList.of(Long.toString(p.accountId()),p.firstName(),p.lastName(),
105+
gender,Dictionaries.dates.formatDate(p.birthday()),Dictionaries.dates.formatDateTime(p.creationDate()),
106+
p.ipAddress().toString(),Dictionaries.browsers.getName(p.browserId()),languagesBuilder.toString(),
107+
emailsBuilder.toString()));
108+
182109

183-
arguments.clear();
184-
arguments.add(Long.toString(p.accountId()));
185-
arguments.add(Integer.toString(p.cityId()));
186-
writers[FileNames.PERSON_LOCATED_IN_PLACE.ordinal()].writeEntry(arguments);
110+
writers.get(PERSON_LOCATED_IN_PLACE).writeEntry(ImmutableList.of(Long.toString(p.accountId()),Integer.toString(p.cityId())));
187111

188112
Iterator<Integer> itInteger = p.interests().iterator();
189113
while (itInteger.hasNext()) {
190-
arguments.clear();
191114
Integer interestIdx = itInteger.next();
192-
arguments.add(Long.toString(p.accountId()));
193-
arguments.add(Integer.toString(interestIdx));
194-
writers[FileNames.PERSON_HAS_INTEREST_TAG.ordinal()].writeEntry(arguments);
115+
writers.get(PERSON_HAS_INTEREST_TAG).writeEntry(ImmutableList.of(Long.toString(p.accountId()),Integer.toString(interestIdx)));
195116
}
196117
}
197118

198119
@Override
199120
protected void serialize(final StudyAt studyAt) {
200-
ArrayList<String> arguments = new ArrayList<String>();
201-
String dateString = Dictionaries.dates.formatYear(studyAt.year);
202-
arguments.add(Long.toString(studyAt.user));
203-
arguments.add(Long.toString(studyAt.university));
204-
arguments.add(dateString);
205-
writers[FileNames.PERSON_STUDY_AT.ordinal()].writeEntry(arguments);
121+
writers.get(PERSON_STUDY_AT).writeEntry(ImmutableList.of(Long.toString(studyAt.user),Long.toString(studyAt.university),Dictionaries.dates.formatYear(studyAt.year)));
206122
}
207123

208124
@Override
209125
protected void serialize(final WorkAt workAt) {
210-
ArrayList<String> arguments = new ArrayList<String>();
211-
String dateString = Dictionaries.dates.formatYear(workAt.year);
212-
arguments.add(Long.toString(workAt.user));
213-
arguments.add(Long.toString(workAt.company));
214-
arguments.add(dateString);
215-
writers[FileNames.PERSON_WORK_AT.ordinal()].writeEntry(arguments);
126+
writers.get(PERSON_WORK_AT).writeEntry(ImmutableList.of(Long.toString(workAt.user),Long.toString(workAt.company),Dictionaries.dates.formatYear(workAt.year)));
216127
}
217128

218129
@Override
219130
protected void serialize(final Person p, Knows knows) {
220-
ArrayList<String> arguments = new ArrayList<String>();
221-
String dateString = Dictionaries.dates.formatDateTime(knows.creationDate());
222-
arguments.add(Long.toString(p.accountId()));
223-
arguments.add(Long.toString(knows.to().accountId()));
224-
arguments.add(dateString);
225-
writers[FileNames.PERSON_KNOWS_PERSON.ordinal()].writeEntry(arguments);
131+
writers.get(PERSON_KNOWS_PERSON).writeEntry(ImmutableList.of(Long.toString(p.accountId()),Long.toString(knows.to().accountId()), Dictionaries.dates.formatDateTime(knows.creationDate())));
226132
}
227133

228134
@Override

0 commit comments

Comments
 (0)