Skip to content

Commit 8bc2a0d

Browse files
miratepuffinmiratepuffin
committed
Refactored CSVDynamicPersonSerializer
1 parent ea682cc commit 8bc2a0d

File tree

1 file changed

+37
-160
lines changed

1 file changed

+37
-160
lines changed

‎src/main/java/ldbc/snb/datagen/serializer/snb/csv/dynamicserializer/person/CSVDynamicPersonSerializer.java

Lines changed: 37 additions & 160 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ Linked Data Benchmark Council (http://www.ldbcouncil.org)
3838

3939
package ldbc.snb.datagen.serializer.snb.csv.dynamicserializer.person;
4040

41+
import com.google.common.collect.ImmutableList;
4142
import ldbc.snb.datagen.dictionary.Dictionaries;
4243
import ldbc.snb.datagen.entities.dynamic.relations.Knows;
4344
import ldbc.snb.datagen.entities.dynamic.person.Person;
@@ -50,198 +51,74 @@ Linked Data Benchmark Council (http://www.ldbcouncil.org)
5051

5152
import java.io.IOException;
5253
import java.util.ArrayList;
54+
import java.util.Arrays;
5355
import java.util.Iterator;
5456
import java.util.List;
5557

56-
public class CSVDynamicPersonSerializer extends DynamicPersonSerializer {
57-
58-
private HDFSCSVWriter[] writers;
59-
60-
private enum FileNames {
61-
PERSON("person"),
62-
PERSON_SPEAKS_LANGUAGE("person_speaks_language"),
63-
PERSON_HAS_EMAIL("person_email_emailaddress"),
64-
PERSON_LOCATED_IN_PLACE("person_isLocatedIn_place"),
65-
PERSON_HAS_INTEREST_TAG("person_hasInterest_tag"),
66-
PERSON_WORK_AT("person_workAt_organisation"),
67-
PERSON_STUDY_AT("person_studyAt_organisation"),
68-
PERSON_KNOWS_PERSON("person_knows_person");
69-
70-
private final String name;
58+
import ldbc.snb.datagen.serializer.snb.csv.FileName;
7159

72-
private FileNames(String name) {
73-
this.name = name;
74-
}
60+
import static ldbc.snb.datagen.serializer.snb.csv.FileName.*;
7561

76-
public String toString() {
77-
return name;
78-
}
79-
}
62+
public class CSVDynamicPersonSerializer extends DynamicPersonSerializer {
8063

8164
@Override
8265
public List<FileName> getFileNames() {
83-
return null;
66+
return Arrays.asList(PERSON,PERSON_SPEAKS_LANGUAGE,PERSON_HAS_EMAIL,PERSON_LOCATED_IN_PLACE,PERSON_HAS_INTEREST_TAG,PERSON_WORK_AT,PERSON_STUDY_AT,PERSON_KNOWS_PERSON);
8467
}
8568

8669
@Override
8770
public void writeFileHeaders() {
88-
89-
}
90-
91-
@Override
92-
public void initialize(Configuration conf, int reducerId) throws IOException {
93-
int numFiles = FileNames.values().length;
94-
writers = new HDFSCSVWriter[numFiles];
95-
for (int i = 0; i < numFiles; ++i) {
96-
writers[i] = new HDFSCSVWriter(conf.get("ldbc.snb.datagen.serializer.socialNetworkDir")+"/dynamic/", FileNames
97-
.values()[i].toString() + "_" + reducerId, conf
98-
.getInt("ldbc.snb.datagen.serializer.numPartitions", 1), conf
99-
.getBoolean("ldbc.snb.datagen.serializer.compressed", false), "|", conf
100-
.getBoolean("ldbc.snb.datagen.serializer.endlineSeparator", false));
101-
}
102-
103-
ArrayList<String> arguments = new ArrayList<String>();
104-
arguments.add("id");
105-
arguments.add("firstName");
106-
arguments.add("lastName");
107-
arguments.add("gender");
108-
arguments.add("birthday");
109-
arguments.add("creationDate");
110-
arguments.add("locationIP");
111-
arguments.add("browserUsed");
112-
writers[FileNames.PERSON.ordinal()].writeHeader(arguments);
113-
114-
arguments.clear();
115-
arguments.add("Person.id");
116-
arguments.add("language");
117-
writers[FileNames.PERSON_SPEAKS_LANGUAGE.ordinal()].writeHeader(arguments);
118-
119-
arguments.clear();
120-
arguments.add("Person.id");
121-
arguments.add("email");
122-
writers[FileNames.PERSON_HAS_EMAIL.ordinal()].writeHeader(arguments);
123-
124-
arguments.clear();
125-
arguments.add("Person.id");
126-
arguments.add("Place.id");
127-
writers[FileNames.PERSON_LOCATED_IN_PLACE.ordinal()].writeHeader(arguments);
128-
129-
arguments.clear();
130-
arguments.add("Person.id");
131-
arguments.add("Tag.id");
132-
writers[FileNames.PERSON_HAS_INTEREST_TAG.ordinal()].writeHeader(arguments);
133-
134-
arguments.clear();
135-
arguments.add("Person.id");
136-
arguments.add("Organisation.id");
137-
arguments.add("workFrom");
138-
writers[FileNames.PERSON_WORK_AT.ordinal()].writeHeader(arguments);
139-
140-
arguments.clear();
141-
arguments.add("Person.id");
142-
arguments.add("Organisation.id");
143-
arguments.add("classYear");
144-
writers[FileNames.PERSON_STUDY_AT.ordinal()].writeHeader(arguments);
145-
146-
arguments.clear();
147-
arguments.add("Person.id");
148-
arguments.add("Person.id");
149-
arguments.add("creationDate");
150-
writers[FileNames.PERSON_KNOWS_PERSON.ordinal()].writeHeader(arguments);
151-
152-
}
153-
154-
@Override
155-
public void close() {
156-
int numFiles = FileNames.values().length;
157-
for (int i = 0; i < numFiles; ++i) {
158-
writers[i].close();
159-
}
71+
writers.get(PERSON).writeHeader(ImmutableList.of("id","firstName","lastName","gender","birthday","creationDate","locationIP","browserUsed"));
72+
writers.get(PERSON_SPEAKS_LANGUAGE).writeHeader(ImmutableList.of("Person.id","language"));
73+
writers.get(PERSON_HAS_EMAIL).writeHeader(ImmutableList.of("Person.id","email"));
74+
writers.get(PERSON_LOCATED_IN_PLACE).writeHeader(ImmutableList.of("Person.id","Place.id"));
75+
writers.get(PERSON_HAS_INTEREST_TAG).writeHeader(ImmutableList.of("Person.id","Tag.id"));
76+
writers.get(PERSON_WORK_AT).writeHeader(ImmutableList.of("Person.id","Organisation.id","workFrom"));
77+
writers.get(PERSON_STUDY_AT).writeHeader(ImmutableList.of("Person.id","Organisation.id","classYear"));
78+
writers.get(PERSON_KNOWS_PERSON).writeHeader(ImmutableList.of("Person.id","Person.id","creationDate"));
16079
}
16180

16281
@Override
16382
protected void serialize(final Person p) {
164-
165-
ArrayList<String> arguments = new ArrayList<String>();
166-
167-
arguments.add(Long.toString(p.accountId()));
168-
arguments.add(p.firstName());
169-
arguments.add(p.lastName());
170-
if (p.gender() == 1) {
171-
arguments.add("male");
172-
} else {
173-
arguments.add("female");
174-
}
175-
176-
String dateString = Dictionaries.dates.formatDate(p.birthday());
177-
arguments.add(dateString);
178-
179-
dateString = Dictionaries.dates.formatDateTime(p.creationDate());
180-
arguments.add(dateString);
181-
arguments.add(p.ipAddress().toString());
182-
arguments.add(Dictionaries.browsers.getName(p.browserId()));
183-
writers[FileNames.PERSON.ordinal()].writeEntry(arguments);
184-
185-
ArrayList<Integer> languages = p.languages();
186-
for (int i = 0; i < languages.size(); i++) {
187-
arguments.clear();
188-
arguments.add(Long.toString(p.accountId()));
189-
arguments.add(Dictionaries.languages.getLanguageName(languages.get(i)));
190-
writers[FileNames.PERSON_SPEAKS_LANGUAGE.ordinal()].writeEntry(arguments);
191-
}
83+
writers.get(PERSON).writeEntry(ImmutableList.of(
84+
Long.toString(p.accountId()),
85+
p.firstName(),
86+
p.lastName(),
87+
getGender(p.gender()),
88+
Dictionaries.dates.formatDate(p.birthday()),
89+
Dictionaries.dates.formatDateTime(p.creationDate()),
90+
p.ipAddress().toString(),
91+
Dictionaries.browsers.getName(p.browserId())
92+
));
93+
94+
for (Integer i:p.languages())
95+
writers.get(PERSON_SPEAKS_LANGUAGE).writeEntry(ImmutableList.of(Long.toString(p.accountId()),Dictionaries.languages.getLanguageName(i)));
19296

19397
Iterator<String> itString = p.emails().iterator();
194-
while (itString.hasNext()) {
195-
arguments.clear();
196-
String email = itString.next();
197-
arguments.add(Long.toString(p.accountId()));
198-
arguments.add(email);
199-
writers[FileNames.PERSON_HAS_EMAIL.ordinal()].writeEntry(arguments);
200-
}
201-
202-
arguments.clear();
203-
arguments.add(Long.toString(p.accountId()));
204-
arguments.add(Integer.toString(p.cityId()));
205-
writers[FileNames.PERSON_LOCATED_IN_PLACE.ordinal()].writeEntry(arguments);
98+
while (itString.hasNext())
99+
writers.get(PERSON_HAS_EMAIL).writeHeader(ImmutableList.of(Long.toString(p.accountId()),itString.next()));
100+
101+
102+
writers.get(PERSON_LOCATED_IN_PLACE).writeHeader(ImmutableList.of(Long.toString(p.accountId()),Integer.toString(p.cityId())));
206103

207104
Iterator<Integer> itInteger = p.interests().iterator();
208-
while (itInteger.hasNext()) {
209-
arguments.clear();
210-
Integer interestIdx = itInteger.next();
211-
arguments.add(Long.toString(p.accountId()));
212-
arguments.add(Integer.toString(interestIdx));
213-
writers[FileNames.PERSON_HAS_INTEREST_TAG.ordinal()].writeEntry(arguments);
214-
}
105+
while (itInteger.hasNext())
106+
writers.get(PERSON_HAS_INTEREST_TAG).writeHeader(ImmutableList.of(Long.toString(p.accountId()),Integer.toString(itInteger.next())));
215107
}
216108

217109
@Override
218110
protected void serialize(final StudyAt studyAt) {
219-
ArrayList<String> arguments = new ArrayList<String>();
220-
String dateString = Dictionaries.dates.formatYear(studyAt.year);
221-
arguments.add(Long.toString(studyAt.user));
222-
arguments.add(Long.toString(studyAt.university));
223-
arguments.add(dateString);
224-
writers[FileNames.PERSON_STUDY_AT.ordinal()].writeEntry(arguments);
111+
writers.get(PERSON_STUDY_AT).writeHeader(ImmutableList.of(Long.toString(studyAt.user),Long.toString(studyAt.university),Dictionaries.dates.formatYear(studyAt.year)));
225112
}
226113

227114
@Override
228115
protected void serialize(final WorkAt workAt) {
229-
ArrayList<String> arguments = new ArrayList<String>();
230-
String dateString = Dictionaries.dates.formatYear(workAt.year);
231-
arguments.add(Long.toString(workAt.user));
232-
arguments.add(Long.toString(workAt.company));
233-
arguments.add(dateString);
234-
writers[FileNames.PERSON_WORK_AT.ordinal()].writeEntry(arguments);
116+
writers.get(PERSON_WORK_AT).writeHeader(ImmutableList.of(Long.toString(workAt.user),Long.toString(workAt.company), Dictionaries.dates.formatYear(workAt.year)));
235117
}
236118

237119
@Override
238120
protected void serialize(final Person p, Knows knows) {
239-
ArrayList<String> arguments = new ArrayList<String>();
240-
String dateString = Dictionaries.dates.formatDateTime(knows.creationDate());
241-
arguments.add(Long.toString(p.accountId()));
242-
arguments.add(Long.toString(knows.to().accountId()));
243-
arguments.add(dateString);
244-
writers[FileNames.PERSON_KNOWS_PERSON.ordinal()].writeEntry(arguments);
121+
writers.get(PERSON_KNOWS_PERSON).writeHeader(ImmutableList.of(Long.toString(p.accountId()),Long.toString(knows.to().accountId()),Dictionaries.dates.formatDateTime(knows.creationDate())));
245122
}
246123

247124
}

0 commit comments

Comments
 (0)