Skip to content

Commit cce8a3d

Browse files
miratepuffinmiratepuffin
committed
Initial refactoring of Hadoop package
1 parent 92d1c7d commit cce8a3d

File tree

46 files changed

+107
-51
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

46 files changed

+107
-51
lines changed

‎src/main/java/ldbc/snb/datagen/LDBCDatagen.java

Lines changed: 16 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,14 @@ Linked Data Benchmark Council (http://www.ldbcouncil.org)
3636
package ldbc.snb.datagen;
3737

3838
import ldbc.snb.datagen.dictionary.Dictionaries;
39-
import ldbc.snb.datagen.hadoop.*;
39+
import ldbc.snb.datagen.hadoop.generator.HadoopKnowsGenerator;
40+
import ldbc.snb.datagen.hadoop.generator.HadoopPersonActivityGenerator;
41+
import ldbc.snb.datagen.hadoop.generator.HadoopPersonGenerator;
42+
import ldbc.snb.datagen.hadoop.serializer.HadoopDynamicPersonSerializer;
43+
import ldbc.snb.datagen.hadoop.serializer.HadoopPersonSortAndSerializer;
44+
import ldbc.snb.datagen.hadoop.serializer.HadoopStaticSerializer;
45+
import ldbc.snb.datagen.hadoop.serializer.HadoopUpdateStreamSorterAndSerializer;
46+
import ldbc.snb.datagen.hadoop.miscjob.HadoopMergeFriendshipFiles;
4047
import ldbc.snb.datagen.objects.Person;
4148
import ldbc.snb.datagen.util.ConfigParser;
4249
import ldbc.snb.datagen.vocabulary.SN;
@@ -90,14 +97,14 @@ public int runGenerateJob(Configuration conf) throws Exception {
9097
printProgress("Starting: Person generation");
9198
long startPerson = System.currentTimeMillis();
9299
HadoopPersonGenerator personGenerator = new HadoopPersonGenerator(conf);
93-
personGenerator.run(hadoopPrefix + "/persons", "ldbc.snb.datagen.hadoop.UniversityKeySetter");
100+
personGenerator.run(hadoopPrefix + "/persons", "ldbc.snb.datagen.hadoop.utiljobs.keychanger.UniversityKeySetter");
94101
long endPerson = System.currentTimeMillis();
95102

96103
printProgress("Creating university location correlated edges");
97104
long startUniversity = System.currentTimeMillis();
98105
HadoopKnowsGenerator knowsGenerator = new HadoopKnowsGenerator(conf,
99-
"ldbc.snb.datagen.hadoop.UniversityKeySetter",
100-
"ldbc.snb.datagen.hadoop.RandomKeySetter",
106+
"ldbc.snb.datagen.hadoop.utiljobs.keychanger.UniversityKeySetter",
107+
"ldbc.snb.datagen.hadoop.utiljobs.keychanger.RandomKeySetter",
101108
percentages,
102109
0,
103110
conf.get("ldbc.snb.datagen.generator.knowsGenerator"));
@@ -110,8 +117,8 @@ public int runGenerateJob(Configuration conf) throws Exception {
110117
long startInterest = System.currentTimeMillis();
111118

112119
knowsGenerator = new HadoopKnowsGenerator(conf,
113-
"ldbc.snb.datagen.hadoop.InterestKeySetter",
114-
"ldbc.snb.datagen.hadoop.RandomKeySetter",
120+
"ldbc.snb.datagen.hadoop.utiljobs.keychanger.InterestKeySetter",
121+
"ldbc.snb.datagen.hadoop.utiljobs.keychanger.RandomKeySetter",
115122
percentages,
116123
1,
117124
conf.get("ldbc.snb.datagen.generator.knowsGenerator"));
@@ -124,8 +131,8 @@ public int runGenerateJob(Configuration conf) throws Exception {
124131
long startRandom = System.currentTimeMillis();
125132

126133
knowsGenerator = new HadoopKnowsGenerator(conf,
127-
"ldbc.snb.datagen.hadoop.RandomKeySetter",
128-
"ldbc.snb.datagen.hadoop.RandomKeySetter",
134+
"ldbc.snb.datagen.hadoop.utiljobs.keychanger.RandomKeySetter",
135+
"ldbc.snb.datagen.hadoop.utiljobs.keychanger.RandomKeySetter",
129136
percentages,
130137
2,
131138
"ldbc.snb.datagen.generator.generators.knowsgenerators.RandomKnowsGenerator");
@@ -141,7 +148,7 @@ public int runGenerateJob(Configuration conf) throws Exception {
141148
edgeFileNames.add(hadoopPrefix + "/interestEdges");
142149
edgeFileNames.add(hadoopPrefix + "/randomEdges");
143150
long startMerge = System.currentTimeMillis();
144-
HadoopMergeFriendshipFiles merger = new HadoopMergeFriendshipFiles(conf, "ldbc.snb.datagen.hadoop.RandomKeySetter");
151+
HadoopMergeFriendshipFiles merger = new HadoopMergeFriendshipFiles(conf, "ldbc.snb.datagen.hadoop.utiljobs.keychanger.RandomKeySetter");
145152
merger.run(hadoopPrefix + "/mergedPersons", edgeFileNames);
146153
long endMerge = System.currentTimeMillis();
147154

‎src/main/java/ldbc/snb/datagen/hadoop/HadoopBlockMapper.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,8 @@ Linked Data Benchmark Council (http://www.ldbcouncil.org)
3535
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.*/
3636
package ldbc.snb.datagen.hadoop;
3737

38+
import ldbc.snb.datagen.hadoop.key.blockkey.BlockKey;
39+
import ldbc.snb.datagen.hadoop.key.TupleKey;
3840
import ldbc.snb.datagen.objects.Person;
3941
import org.apache.hadoop.conf.Configuration;
4042
import org.apache.hadoop.io.LongWritable;

‎src/main/java/ldbc/snb/datagen/hadoop/HadoopBlockPartitioner.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ Linked Data Benchmark Council (http://www.ldbcouncil.org)
3535
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.*/
3636
package ldbc.snb.datagen.hadoop;
3737

38+
import ldbc.snb.datagen.hadoop.key.blockkey.BlockKey;
3839
import ldbc.snb.datagen.objects.Person;
3940
import org.apache.hadoop.mapreduce.Partitioner;
4041

‎src/main/java/ldbc/snb/datagen/hadoop/HadoopTuplePartitioner.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ Linked Data Benchmark Council (http://www.ldbcouncil.org)
3535
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.*/
3636
package ldbc.snb.datagen.hadoop;
3737

38+
import ldbc.snb.datagen.hadoop.key.TupleKey;
3839
import ldbc.snb.datagen.objects.Person;
3940
import org.apache.hadoop.mapreduce.Partitioner;
4041

‎src/main/java/ldbc/snb/datagen/hadoop/HadoopKnowsGenerator.java renamed to ‎src/main/java/ldbc/snb/datagen/hadoop/generator/HadoopKnowsGenerator.java

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,10 +33,17 @@ Linked Data Benchmark Council (http://www.ldbcouncil.org)
3333
You should have received a copy of the GNU General Public License
3434
along with this program; if not, write to the Free Software
3535
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.*/
36-
package ldbc.snb.datagen.hadoop;
36+
package ldbc.snb.datagen.hadoop.generator;
3737

3838
import ldbc.snb.datagen.generator.generators.knowsgenerators.KnowsGenerator;
3939
import ldbc.snb.datagen.LDBCDatagen;
40+
import ldbc.snb.datagen.hadoop.*;
41+
import ldbc.snb.datagen.hadoop.key.blockkey.BlockKey;
42+
import ldbc.snb.datagen.hadoop.key.blockkey.BlockKeyComparator;
43+
import ldbc.snb.datagen.hadoop.key.blockkey.BlockKeyGroupComparator;
44+
import ldbc.snb.datagen.hadoop.key.TupleKey;
45+
import ldbc.snb.datagen.hadoop.miscjob.keychanger.HadoopFileKeyChanger;
46+
import ldbc.snb.datagen.hadoop.miscjob.HadoopFileRanker;
4047
import ldbc.snb.datagen.objects.Person;
4148
import org.apache.hadoop.conf.Configuration;
4249
import org.apache.hadoop.fs.FileSystem;

‎src/main/java/ldbc/snb/datagen/hadoop/HadoopPersonActivityGenerator.java renamed to ‎src/main/java/ldbc/snb/datagen/hadoop/generator/HadoopPersonActivityGenerator.java

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,12 +33,19 @@ Linked Data Benchmark Council (http://www.ldbcouncil.org)
3333
You should have received a copy of the GNU General Public License
3434
along with this program; if not, write to the Free Software
3535
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.*/
36-
package ldbc.snb.datagen.hadoop;
36+
package ldbc.snb.datagen.hadoop.generator;
3737

3838
import ldbc.snb.datagen.dictionary.Dictionaries;
3939
import ldbc.snb.datagen.DatagenParams;
4040
import ldbc.snb.datagen.LDBCDatagen;
4141
import ldbc.snb.datagen.generator.generators.PersonActivityGenerator;
42+
import ldbc.snb.datagen.hadoop.HadoopBlockMapper;
43+
import ldbc.snb.datagen.hadoop.HadoopBlockPartitioner;
44+
import ldbc.snb.datagen.hadoop.key.blockkey.BlockKey;
45+
import ldbc.snb.datagen.hadoop.key.blockkey.BlockKeyComparator;
46+
import ldbc.snb.datagen.hadoop.key.blockkey.BlockKeyGroupComparator;
47+
import ldbc.snb.datagen.hadoop.key.TupleKey;
48+
import ldbc.snb.datagen.hadoop.miscjob.HadoopFileRanker;
4249
import ldbc.snb.datagen.objects.Knows;
4350
import ldbc.snb.datagen.objects.Person;
4451
import ldbc.snb.datagen.serializer.DynamicActivitySerializer;

‎src/main/java/ldbc/snb/datagen/hadoop/HadoopPersonGenerator.java renamed to ‎src/main/java/ldbc/snb/datagen/hadoop/generator/HadoopPersonGenerator.java

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,11 +33,13 @@ Linked Data Benchmark Council (http://www.ldbcouncil.org)
3333
You should have received a copy of the GNU General Public License
3434
along with this program; if not, write to the Free Software
3535
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.*/
36-
package ldbc.snb.datagen.hadoop;
36+
package ldbc.snb.datagen.hadoop.generator;
3737

3838
import ldbc.snb.datagen.DatagenParams;
3939
import ldbc.snb.datagen.LDBCDatagen;
4040
import ldbc.snb.datagen.generator.generators.PersonGenerator;
41+
import ldbc.snb.datagen.hadoop.miscjob.keychanger.HadoopFileKeyChanger;
42+
import ldbc.snb.datagen.hadoop.key.TupleKey;
4143
import ldbc.snb.datagen.objects.Person;
4244
import org.apache.hadoop.conf.Configuration;
4345
import org.apache.hadoop.fs.FileSystem;

‎src/main/java/ldbc/snb/datagen/hadoop/HadoopUpdateEventKeyPartitioner.java renamed to ‎src/main/java/ldbc/snb/datagen/hadoop/generator/HadoopUpdateEventKeyPartitioner.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,9 @@ Linked Data Benchmark Council (http://www.ldbcouncil.org)
3333
You should have received a copy of the GNU General Public License
3434
along with this program; if not, write to the Free Software
3535
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.*/
36-
package ldbc.snb.datagen.hadoop;
36+
package ldbc.snb.datagen.hadoop.generator;
3737

38+
import ldbc.snb.datagen.hadoop.key.updatekey.UpdateEventKey;
3839
import org.apache.hadoop.io.Text;
3940
import org.apache.hadoop.mapreduce.Partitioner;
4041

‎src/main/java/ldbc/snb/datagen/hadoop/TupleKey.java renamed to ‎src/main/java/ldbc/snb/datagen/hadoop/key/TupleKey.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ Linked Data Benchmark Council (http://www.ldbcouncil.org)
3333
You should have received a copy of the GNU General Public License
3434
along with this program; if not, write to the Free Software
3535
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.*/
36-
package ldbc.snb.datagen.hadoop;
36+
package ldbc.snb.datagen.hadoop.key;
3737

3838
import org.apache.hadoop.io.WritableComparable;
3939

‎src/main/java/ldbc/snb/datagen/hadoop/BlockKey.java renamed to ‎src/main/java/ldbc/snb/datagen/hadoop/key/blockkey/BlockKey.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,9 @@ Linked Data Benchmark Council (http://www.ldbcouncil.org)
3333
You should have received a copy of the GNU General Public License
3434
along with this program; if not, write to the Free Software
3535
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.*/
36-
package ldbc.snb.datagen.hadoop;
36+
package ldbc.snb.datagen.hadoop.key.blockkey;
3737

38+
import ldbc.snb.datagen.hadoop.key.TupleKey;
3839
import org.apache.hadoop.io.WritableComparable;
3940

4041
import java.io.DataInput;

0 commit comments

Comments
 (0)