Skip to content

Commit d67cf5f

Browse files
committed
Now the endline separator can be configured by means of the ldbc.snb.datagen.serialized.endlineSeparator option
Updated all the serializers to accept the option
1 parent 382142d commit d67cf5f

12 files changed

+12
-11
lines changed

src/main/java/ldbc/snb/datagen/serializer/graphalytics/CSVPersonSerializer.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@ public void initialize(Configuration conf, int reducerId) {
7272
int numFiles = FileNames.values().length;
7373
writers = new HDFSCSVWriter[numFiles];
7474
for( int i = 0; i < numFiles; ++i) {
75-
writers[i] = new HDFSCSVWriter(conf.get("ldbc.snb.datagen.serializer.socialNetworkDir"), FileNames.values()[i].toString()+"_"+reducerId,conf.getInt("ldbc.snb.datagen.numPartitions",1),conf.getBoolean("ldbc.snb.datagen.serializer.compressed",false),"|", false);
75+
writers[i] = new HDFSCSVWriter(conf.get("ldbc.snb.datagen.serializer.socialNetworkDir"), FileNames.values()[i].toString()+"_"+reducerId,conf.getInt("ldbc.snb.datagen.numPartitions",1),conf.getBoolean("ldbc.snb.datagen.serializer.compressed",false),"|", conf.getBoolean("ldbc.snb.datagen.serializer.endlineSeparator",false));
7676
}
7777

7878
ArrayList<String> arguments = new ArrayList<String>();

src/main/java/ldbc/snb/datagen/serializer/graphalytics/CSVPersonSerializerWithWeights.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ public void initialize(Configuration conf, int reducerId) {
7474
int numFiles = FileNames.values().length;
7575
writers = new HDFSCSVWriter[numFiles];
7676
for( int i = 0; i < numFiles; ++i) {
77-
writers[i] = new HDFSCSVWriter(conf.get("ldbc.snb.datagen.serializer.socialNetworkDir"), FileNames.values()[i].toString()+"_"+reducerId,conf.getInt("ldbc.snb.datagen.numPartitions",1),conf.getBoolean("ldbc.snb.datagen.serializer.compressed",false),"|", false);
77+
writers[i] = new HDFSCSVWriter(conf.get("ldbc.snb.datagen.serializer.socialNetworkDir"), FileNames.values()[i].toString()+"_"+reducerId,conf.getInt("ldbc.snb.datagen.numPartitions",1),conf.getBoolean("ldbc.snb.datagen.serializer.compressed",false),"|", conf.getBoolean("ldbc.snb.datagen.serializer.endlineSeparator",false));
7878
}
7979

8080
ArrayList<String> arguments = new ArrayList<String>();

src/main/java/ldbc/snb/datagen/serializer/small/CSVInvariantSerializer.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ public void initialize(Configuration conf, int reducerId) {
3737
int numFiles = FileNames.values().length;
3838
writers = new HDFSCSVWriter[numFiles];
3939
for( int i = 0; i < numFiles; ++i) {
40-
writers[i] = new HDFSCSVWriter(conf.get("ldbc.snb.datagen.serializer.socialNetworkDir"),FileNames.values()[i].toString()+"_"+reducerId,conf.getInt("ldbc.snb.datagen.numPartitions",1),conf.getBoolean("ldbc.snb.datagen.serializer.compressed",false),"|", false);
40+
writers[i] = new HDFSCSVWriter(conf.get("ldbc.snb.datagen.serializer.socialNetworkDir"),FileNames.values()[i].toString()+"_"+reducerId,conf.getInt("ldbc.snb.datagen.numPartitions",1),conf.getBoolean("ldbc.snb.datagen.serializer.compressed",false),"|", conf.getBoolean("ldbc.snb.datagen.serializer.endlineSeparator",false));
4141
}
4242

4343
ArrayList<String> arguments = new ArrayList<String>();

src/main/java/ldbc/snb/datagen/serializer/small/CSVPersonActivitySerializer.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ public void initialize(Configuration conf, int reducerId) {
4242
int numFiles = FileNames.values().length;
4343
writers = new HDFSCSVWriter[numFiles];
4444
for( int i = 0; i < numFiles; ++i) {
45-
writers[i] = new HDFSCSVWriter(conf.get("ldbc.snb.datagen.serializer.socialNetworkDir"),FileNames.values()[i].toString()+"_"+reducerId,conf.getInt("ldbc.snb.datagen.numPartitions",1),conf.getBoolean("ldbc.snb.datagen.serializer.compressed",false),"|", false);
45+
writers[i] = new HDFSCSVWriter(conf.get("ldbc.snb.datagen.serializer.socialNetworkDir"),FileNames.values()[i].toString()+"_"+reducerId,conf.getInt("ldbc.snb.datagen.numPartitions",1),conf.getBoolean("ldbc.snb.datagen.serializer.compressed",false),"|", conf.getBoolean("ldbc.snb.datagen.serializer.endlineSeparator",false));
4646
}
4747
arguments = new ArrayList<String>();
4848

src/main/java/ldbc/snb/datagen/serializer/small/CSVPersonSerializer.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,7 @@ public void initialize(Configuration conf, int reducerId) {
7575
int numFiles = FileNames.values().length;
7676
writers = new HDFSCSVWriter[numFiles];
7777
for( int i = 0; i < numFiles; ++i) {
78-
writers[i] = new HDFSCSVWriter(conf.get("ldbc.snb.datagen.serializer.socialNetworkDir"),FileNames.values()[i].toString()+"_"+reducerId,conf.getInt("ldbc.snb.datagen.numPartitions",1),conf.getBoolean("ldbc.snb.datagen.serializer.compressed",false),"|", false);
78+
writers[i] = new HDFSCSVWriter(conf.get("ldbc.snb.datagen.serializer.socialNetworkDir"),FileNames.values()[i].toString()+"_"+reducerId,conf.getInt("ldbc.snb.datagen.numPartitions",1),conf.getBoolean("ldbc.snb.datagen.serializer.compressed",false),"|", conf.getBoolean("ldbc.snb.datagen.serializer.endlineSeparator",false));
7979
}
8080

8181
ArrayList<String> arguments = new ArrayList<String>();

src/main/java/ldbc/snb/datagen/serializer/snb/interactive/CSVInvariantSerializer.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ public void initialize(Configuration conf, int reducerId) {
4848
int numFiles = FileNames.values().length;
4949
writers = new HDFSCSVWriter[numFiles];
5050
for( int i = 0; i < numFiles; ++i) {
51-
writers[i] = new HDFSCSVWriter(conf.get("ldbc.snb.datagen.serializer.socialNetworkDir"),FileNames.values()[i].toString()+"_"+reducerId,conf.getInt("ldbc.snb.datagen.numPartitions",1),conf.getBoolean("ldbc.snb.datagen.serializer.compressed",false),"|",false);
51+
writers[i] = new HDFSCSVWriter(conf.get("ldbc.snb.datagen.serializer.socialNetworkDir"),FileNames.values()[i].toString()+"_"+reducerId,conf.getInt("ldbc.snb.datagen.numPartitions",1),conf.getBoolean("ldbc.snb.datagen.serializer.compressed",false),"|",conf.getBoolean("ldbc.snb.datagen.serializer.endlineSeparator",false));
5252
}
5353

5454
ArrayList<String> arguments = new ArrayList<String>();

src/main/java/ldbc/snb/datagen/serializer/snb/interactive/CSVMergeForeignInvariantSerializer.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ public void initialize(Configuration conf, int reducerId) {
4242
int numFiles = FileNames.values().length;
4343
writers = new HDFSCSVWriter[numFiles];
4444
for( int i = 0; i < numFiles; ++i) {
45-
writers[i] = new HDFSCSVWriter(conf.get("ldbc.snb.datagen.serializer.socialNetworkDir"),FileNames.values()[i].toString()+"_"+reducerId,conf.getInt("ldbc.snb.datagen.numPartitions",1),conf.getBoolean("ldbc.snb.datagen.serializer.compressed",false),"|",false);
45+
writers[i] = new HDFSCSVWriter(conf.get("ldbc.snb.datagen.serializer.socialNetworkDir"),FileNames.values()[i].toString()+"_"+reducerId,conf.getInt("ldbc.snb.datagen.numPartitions",1),conf.getBoolean("ldbc.snb.datagen.serializer.compressed",false),"|",conf.getBoolean("ldbc.snb.datagen.serializer.endlineSeparator",false));
4646
}
4747

4848
ArrayList<String> arguments = new ArrayList<String>();

src/main/java/ldbc/snb/datagen/serializer/snb/interactive/CSVMergeForeignPersonActivitySerializer.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ public void initialize(Configuration conf, int reducerId) {
4545
int numFiles = FileNames.values().length;
4646
writers = new HDFSCSVWriter[numFiles];
4747
for( int i = 0; i < numFiles; ++i) {
48-
writers[i] = new HDFSCSVWriter(conf.get("ldbc.snb.datagen.serializer.socialNetworkDir"),FileNames.values()[i].toString()+"_"+reducerId,conf.getInt("ldbc.snb.datagen.numPartitions",1),conf.getBoolean("ldbc.snb.datagen.serializer.compressed",false),"|",false);
48+
writers[i] = new HDFSCSVWriter(conf.get("ldbc.snb.datagen.serializer.socialNetworkDir"),FileNames.values()[i].toString()+"_"+reducerId,conf.getInt("ldbc.snb.datagen.numPartitions",1),conf.getBoolean("ldbc.snb.datagen.serializer.compressed",false),"|",conf.getBoolean("ldbc.snb.datagen.serializer.endlineSeparator",false));
4949
}
5050
arguments = new ArrayList<String>();
5151

src/main/java/ldbc/snb/datagen/serializer/snb/interactive/CSVMergeForeignPersonSerializer.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ public void initialize(Configuration conf, int reducerId) {
4545
int numFiles = FileNames.values().length;
4646
writers = new HDFSCSVWriter[numFiles];
4747
for( int i = 0; i < numFiles; ++i) {
48-
writers[i] = new HDFSCSVWriter(conf.get("ldbc.snb.datagen.serializer.socialNetworkDir"),FileNames.values()[i].toString()+"_"+reducerId,conf.getInt("ldbc.snb.datagen.numPartitions",1),conf.getBoolean("ldbc.snb.datagen.serializer.compressed",false),"|",false);
48+
writers[i] = new HDFSCSVWriter(conf.get("ldbc.snb.datagen.serializer.socialNetworkDir"),FileNames.values()[i].toString()+"_"+reducerId,conf.getInt("ldbc.snb.datagen.numPartitions",1),conf.getBoolean("ldbc.snb.datagen.serializer.compressed",false),"|", conf.getBoolean("ldbc.snb.datagen.serializer.endlineSeparator",false));
4949
}
5050

5151
ArrayList<String> arguments = new ArrayList<String>();

src/main/java/ldbc/snb/datagen/serializer/snb/interactive/CSVPersonActivitySerializer.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ public void initialize(Configuration conf, int reducerId) {
5959
int numFiles = FileNames.values().length;
6060
writers = new HDFSCSVWriter[numFiles];
6161
for( int i = 0; i < numFiles; ++i) {
62-
writers[i] = new HDFSCSVWriter(conf.get("ldbc.snb.datagen.serializer.socialNetworkDir"),FileNames.values()[i].toString()+"_"+reducerId,conf.getInt("ldbc.snb.datagen.numPartitions",1),conf.getBoolean("ldbc.snb.datagen.serializer.compressed",false),"|",false);
62+
writers[i] = new HDFSCSVWriter(conf.get("ldbc.snb.datagen.serializer.socialNetworkDir"),FileNames.values()[i].toString()+"_"+reducerId,conf.getInt("ldbc.snb.datagen.numPartitions",1),conf.getBoolean("ldbc.snb.datagen.serializer.compressed",false),"|",conf.getBoolean("ldbc.snb.datagen.serializer.endlineSeparator",false) );
6363
}
6464
arguments = new ArrayList<String>();
6565

0 commit comments

Comments
 (0)