Skip to content

Commit 7cf0a8e

Browse files
ArnauPratArnauPrat
committed
Added tests to query parameters
1 parent 6f49308 commit 7cf0a8e

File tree

7 files changed

+130
-29
lines changed

7 files changed

+130
-29
lines changed

‎src/main/java/ldbc/snb/datagen/dictionary/TagTextDictionary.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -163,6 +163,10 @@ public String generateText(Random randomTextSize, TreeSet<Integer> tags, int tex
163163
if (returnString.length() < textSize - 1) {
164164
returnString.append(" ");
165165
}
166+
if (returnString.length() > textSize) {
167+
returnString.delete(textSize-1,returnString.length());
168+
returnString.trimToSize();
169+
}
166170
return returnString.toString().replace("|", " ");
167171
}
168172
}

‎src/main/java/ldbc/snb/datagen/generator/LdbcSnbTextGenerator.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,17 +29,21 @@ public String generateText(PersonSummary member, TreeSet<Integer> tags, Properti
2929
int textSize;
3030
if( member.isLargePoster() && this.random.nextDouble() > (1.0f-DatagenParams.ratioLargePost) ) {
3131
textSize = Dictionaries.tagText.getRandomLargeTextSize( this.random,DatagenParams.minLargePostSize, DatagenParams.maxLargePostSize );
32+
assert textSize <= DatagenParams.maxLargePostSize && textSize >= DatagenParams.minLargePostSize : "Person creation date is larger than membership";
3233
} else {
3334
textSize = Dictionaries.tagText.getRandomTextSize( this.random, this.random, DatagenParams.minTextSize, DatagenParams.maxTextSize);
35+
assert textSize <= DatagenParams.maxTextSize && textSize >= DatagenParams.minTextSize : "Person creation date is larger than membership";
3436
}
3537
content = Dictionaries.tagText.generateText(this.random,tags,textSize);
3638
}
3739
else {//si no es post fer
3840
int textSize;
3941
if( member.isLargePoster() && this.random.nextDouble() > (1.0f-DatagenParams.ratioLargeComment) ) {
4042
textSize = Dictionaries.tagText.getRandomLargeTextSize(this.random, DatagenParams.minLargeCommentSize, DatagenParams.maxLargeCommentSize);
43+
assert textSize <= DatagenParams.maxLargeCommentSize && textSize >= DatagenParams.minLargeCommentSize : "Person creation date is larger than membership";
4144
} else {
4245
textSize = Dictionaries.tagText.getRandomTextSize( this.random, this.random, DatagenParams.minCommentSize, DatagenParams.maxCommentSize);
46+
assert textSize <= DatagenParams.maxCommentSize && textSize >= DatagenParams.minCommentSize : "Person creation date is larger than membership";
4347
}
4448
content = Dictionaries.tagText.generateText(this.random,tags,textSize);
4549
}

‎src/main/resources/params_test.ini

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,8 @@ ldbc.snb.datagen.generator.friendRejectRatio:0.02
44
ldbc.snb.datagen.generator.friendReApproveRatio:0.5
55
ldbc.snb.datagen.generator.minNumTagsPerUser:1
66
ldbc.snb.datagen.generator.maxNumTagsPerUser:80
7-
ldbc.snb.datagen.generator.maxNumPostPerMonth:30
8-
ldbc.snb.datagen.generator.maxNumGroupPostPerMonth:10
7+
ldbc.snb.datagen.generator.maxNumPostPerMonth:0
8+
ldbc.snb.datagen.generator.maxNumGroupPostPerMonth:0
99
ldbc.snb.datagen.generator.maxNumComments:20
1010
ldbc.snb.datagen.generator.maxEmails:5
1111
ldbc.snb.datagen.generator.maxCompanies:3
@@ -43,12 +43,12 @@ ldbc.snb.datagen.generator.cellSize:1
4343
ldbc.snb.datagen.generator.numberOfCellPerWindow:1000
4444
ldbc.snb.datagen.generator.baseProbCorrelated:0.95
4545
ldbc.snb.datagen.generator.limitProCorrelated:0.2
46-
ldbc.snb.datagen.generator.flashmobTagsPerMonth:30
46+
ldbc.snb.datagen.generator.flashmobTagsPerMonth:50
4747
ldbc.snb.datagen.generator.probInterestFlashmobTag:0.8
4848
ldbc.snb.datagen.generator.probRandomPerLevel:0.005
4949
ldbc.snb.datagen.generator.maxNumLike:1000
50-
ldbc.snb.datagen.generator.maxNumFlashmobPostPerMonth:30
51-
ldbc.snb.datagen.generator.maxNumGroupFlashmobPostPerMonth:10
50+
ldbc.snb.datagen.generator.maxNumFlashmobPostPerMonth:50
51+
ldbc.snb.datagen.generator.maxNumGroupFlashmobPostPerMonth:50
5252
ldbc.snb.datagen.generator.maxNumTagPerFlashmobPost:80
5353
ldbc.snb.datagen.generator.flashmobTagMinLevel:1
5454
ldbc.snb.datagen.generator.flashmobTagMaxLevel:20

‎src/test/java/ldbc/snb/datagen/test/LDBCDatagenTest.java

Lines changed: 73 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,11 @@
11
package ldbc.snb.datagen.test;
22

3+
import ldbc.snb.datagen.dictionary.Dictionaries;
4+
import ldbc.snb.datagen.generator.DatagenParams;
5+
import ldbc.snb.datagen.generator.LDBCDatagen;
36
import ldbc.snb.datagen.test.csv.*;
7+
import ldbc.snb.datagen.util.ConfigParser;
8+
import org.apache.hadoop.conf.Configuration;
49
import org.junit.BeforeClass;
510
import org.junit.Test;
611

@@ -31,6 +36,11 @@ public static void generateData() {
3136
}catch(Exception e) {
3237
System.err.println(e.getMessage());
3338
}
39+
40+
Configuration conf = ConfigParser.initialize();
41+
ConfigParser.readConfig(conf, "./test_params.ini");
42+
ConfigParser.readConfig(conf, LDBCDatagen.class.getResourceAsStream("/params.ini"));
43+
LDBCDatagen.init(conf);
3444
}
3545

3646
@Test
@@ -41,6 +51,7 @@ public void personTest() {
4151
@Test
4252
public void postTest() {
4353
testIdUniqueness(dir+"/post_0_0.csv", 0);
54+
testLongBetween(dir+"/post_0_0.csv",7,0,2001);
4455
}
4556

4657
@Test
@@ -51,6 +62,7 @@ public void forumTest() {
5162
@Test
5263
public void commentTest() {
5364
testIdUniqueness(dir+"/comment_0_0.csv", 0);
65+
testLongBetween(dir+"/comment_0_0.csv",5,0,2001);
5466
}
5567

5668
@Test
@@ -178,6 +190,7 @@ public void personLikesCommentCheck() {
178190
testPairUniquenessPlusExistance(dir+"/person_likes_comment_0_0.csv",0,1,dir+"/person_0_0.csv",0,dir+"/comment_0_0.csv",0);
179191
}
180192

193+
181194
@Test
182195
public void personLikesPostCheck() {
183196
testPairUniquenessPlusExistance(dir+"/person_likes_post_0_0.csv",0,1,dir+"/person_0_0.csv",0,dir+"/post_0_0.csv",0);
@@ -191,6 +204,8 @@ public void updateStreamForumsConsistencyCheck() {
191204

192205
@Test
193206
public void queryParamsTest() {
207+
208+
//Creating person id check
194209
LongParser parser = new LongParser();
195210
ColumnSet<Long> persons = new ColumnSet<Long>(parser,new File(dir+"/person_0_0.csv"),0,1);
196211
persons.initialize();
@@ -200,6 +215,7 @@ public void queryParamsTest() {
200215
personIndex.add(0);
201216
ExistsCheck<Long> existsPersonCheck = new ExistsCheck<Long>(parser,personIndex, personsRef);
202217

218+
//Creating name check
203219
StringParser strParser = new StringParser();
204220
ColumnSet<String> names = new ColumnSet<String>(strParser,new File(dir+"/person_0_0.csv"),1,1);
205221
names.initialize();
@@ -209,29 +225,60 @@ public void queryParamsTest() {
209225
namesIndex.add(1);
210226
ExistsCheck<String> existsNameCheck = new ExistsCheck<String>(strParser,namesIndex, namesRef);
211227

228+
229+
212230
FileChecker fileChecker = new FileChecker(sdir+"/query_1_param.txt");
213231
fileChecker.addCheck(existsPersonCheck);
214232
fileChecker.addCheck(existsNameCheck);
215233
if(!fileChecker.run(1)) assertEquals("ERROR PASSING TEST QUERY 1 PERSON AND NAME EXISTS ",true, false);
216234

235+
//Crating date interval check
217236
fileChecker = new FileChecker(sdir+"/query_2_param.txt");
218237
fileChecker.addCheck(existsPersonCheck);
219238
if(!fileChecker.run(1)) assertEquals("ERROR PASSING TEST QUERY 2 PERSON EXISTS ",true, false);
239+
testLongBetween(sdir+"/query_2_param.txt",1, Dictionaries.dates.getStartDateTime(), Dictionaries.dates.getEndDateTime());
240+
241+
//Creating country check
242+
ColumnSet<String> places = new ColumnSet<String>(strParser,new File(dir+"/place_0_0.csv"),1,1);
243+
places.initialize();
244+
List<ColumnSet<String>> placesRef = new ArrayList<ColumnSet<String>>();
245+
placesRef.add(places);
246+
List<Integer> countriesIndex = new ArrayList<Integer>();
247+
countriesIndex.add(3);
248+
countriesIndex.add(4);
249+
ExistsCheck<String> countryExists = new ExistsCheck<String>(strParser,countriesIndex, placesRef);
250+
251+
//Date duration check
252+
DateDurationCheck dateDurationCheck = new DateDurationCheck("Date duration check",1,2,Dictionaries.dates.getStartDateTime(), Dictionaries.dates.getEndDateTime());
220253

221254
fileChecker = new FileChecker(sdir+"/query_3_param.txt");
222255
fileChecker.addCheck(existsPersonCheck);
256+
fileChecker.addCheck(countryExists);
257+
fileChecker.addCheck(dateDurationCheck);
223258
if(!fileChecker.run(1)) assertEquals("ERROR PASSING TEST QUERY 3 PERSON EXISTS ",true, false);
224259

225260
fileChecker = new FileChecker(sdir+"/query_4_param.txt");
226261
fileChecker.addCheck(existsPersonCheck);
262+
fileChecker.addCheck(dateDurationCheck);
227263
if(!fileChecker.run(1)) assertEquals("ERROR PASSING TEST QUERY 4 PERSON EXISTS ",true, false);
228264

229265
fileChecker = new FileChecker(sdir+"/query_5_param.txt");
230266
fileChecker.addCheck(existsPersonCheck);
231267
if(!fileChecker.run(1)) assertEquals("ERROR PASSING TEST QUERY 5 PERSON EXISTS ",true, false);
268+
testLongBetween(sdir+"/query_5_param.txt",1, Dictionaries.dates.getStartDateTime(), Dictionaries.dates.getEndDateTime());
269+
270+
//Creating tag check
271+
ColumnSet<String> tags = new ColumnSet<String>(strParser,new File(dir+"/tag_0_0.csv"),1,1);
272+
tags.initialize();
273+
List<ColumnSet<String>> tagsRef = new ArrayList<ColumnSet<String>>();
274+
tagsRef.add(tags);
275+
List<Integer> tagsIndex = new ArrayList<Integer>();
276+
tagsIndex.add(1);
277+
ExistsCheck<String> tagExists = new ExistsCheck<String>(strParser,tagsIndex, tagsRef);
232278

233279
fileChecker = new FileChecker(sdir+"/query_6_param.txt");
234280
fileChecker.addCheck(existsPersonCheck);
281+
fileChecker.addCheck(tagExists);
235282
if(!fileChecker.run(1)) assertEquals("ERROR PASSING TEST QUERY 6 PERSON EXISTS ",true, false);
236283

237284
fileChecker = new FileChecker(sdir+"/query_7_param.txt");
@@ -245,17 +292,35 @@ public void queryParamsTest() {
245292
fileChecker = new FileChecker(sdir+"/query_9_param.txt");
246293
fileChecker.addCheck(existsPersonCheck);
247294
if(!fileChecker.run(1)) assertEquals("ERROR PASSING TEST QUERY 9 PERSON EXISTS ",true, false);
295+
testLongBetween(sdir+"/query_9_param.txt",1, Dictionaries.dates.getStartDateTime(), Dictionaries.dates.getEndDateTime());
248296

249297
fileChecker = new FileChecker(sdir+"/query_10_param.txt");
250298
fileChecker.addCheck(existsPersonCheck);
251299
if(!fileChecker.run(1)) assertEquals("ERROR PASSING TEST QUERY 10 PERSON EXISTS ",true, false);
300+
testLongBetween(sdir+"/query_10_param.txt",1, 1, 13);
301+
302+
//Creating country check
303+
countriesIndex.clear();
304+
countriesIndex.add(1);
305+
countryExists = new ExistsCheck<String>(strParser,countriesIndex, placesRef);
252306

253307
fileChecker = new FileChecker(sdir+"/query_11_param.txt");
254308
fileChecker.addCheck(existsPersonCheck);
309+
fileChecker.addCheck(countryExists);
255310
if(!fileChecker.run(1)) assertEquals("ERROR PASSING TEST QUERY 11 PERSON EXISTS ",true, false);
256311

312+
//Creating tagClass check
313+
ColumnSet<String> tagClass = new ColumnSet<String>(strParser,new File(dir+"/tagclass_0_0.csv"),1,1);
314+
tagClass.initialize();
315+
List<ColumnSet<String>> tagClassRef = new ArrayList<ColumnSet<String>>();
316+
tagClassRef.add(tagClass);
317+
List<Integer> tagClassIndex = new ArrayList<Integer>();
318+
tagClassIndex.add(1);
319+
ExistsCheck<String> tagClassExists = new ExistsCheck<String>(strParser,tagClassIndex, tagClassRef);
320+
257321
fileChecker = new FileChecker(sdir+"/query_12_param.txt");
258322
fileChecker.addCheck(existsPersonCheck);
323+
fileChecker.addCheck(tagClassExists);
259324
if(!fileChecker.run(1)) assertEquals("ERROR PASSING TEST QUERY 12 PERSON EXISTS ",true, false);
260325

261326
personIndex.add(1);
@@ -269,23 +334,7 @@ public void queryParamsTest() {
269334
fileChecker.addCheck(exists2PersonCheck);
270335
if(!fileChecker.run(1)) assertEquals("ERROR PASSING TEST QUERY 14 PERSON EXISTS ",true, false);
271336

272-
273337
}
274-
// test query parameters correctness
275-
// query 1, check person id existance and name existance
276-
// query 2, check person id existance and date time within simulation interval
277-
// query 3, check person id existance, country X and Y existance, startData + duration within simulation interval
278-
// query 4, check person id existance and startDate + duration within simulation interval
279-
// query 5, check person id and date within simulation interval
280-
// query 6, check person id and tag name existance
281-
// query 7, check person id existance
282-
// query 8, check person id existance
283-
// query 9, check person id and date within simulation interval
284-
// query 10, check person id existance and month between 1 and 12
285-
// query 11, check person id existance, Country existance and year something reasonable
286-
// query 12, check person id existance and tagclass existance
287-
// query 13, check persons id existance
288-
// query 14, check persons id existance
289338

290339
public void testLongPair(String fileName, Integer columnA, Integer columnB, NumericPairCheck.NumericCheckType type, long offsetA, long offsetB) {
291340
FileChecker fileChecker = new FileChecker(fileName);
@@ -302,6 +351,14 @@ public void testIdUniqueness(String fileName, int column) {
302351
if(!fileChecker.run(1)) assertEquals("ERROR PASSING TEST ID UNIQUENESS FOR FILE "+fileName,true, false);
303352
}
304353

354+
public void testLongBetween(String fileName, int column, long a, long b) {
355+
FileChecker fileChecker = new FileChecker(fileName);
356+
LongParser parser = new LongParser();
357+
LongCheck longcheck = new LongCheck(parser, "Date Test",column, NumericCheck.NumericCheckType.BETWEEN, a,b);
358+
fileChecker.addCheck(longcheck);
359+
if(!fileChecker.run(1)) assertEquals("ERROR PASSING BETWEENS TEST FOR FILE "+fileName+" column "+column+" between "+a+" and "+b,true, false);
360+
}
361+
305362
public void testPairUniquenessPlusExistance(String relationFileName, int columnA, int columnB, String entityFileNameA, int entityColumnA, String entityFileNameB, int entityColumnB) {
306363
LongParser parser = new LongParser();
307364
ColumnSet<Long> entitiesA = new ColumnSet<Long>(parser,new File(entityFileNameA),entityColumnA,1);

‎src/test/java/ldbc/snb/datagen/test/csv/DateDurationCheck.java

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
package ldbc.snb.datagen.test.csv;
2+
3+
import ldbc.snb.datagen.generator.DateGenerator;
4+
5+
import java.util.ArrayList;
6+
import java.util.List;
7+
8+
/**
9+
* Created by aprat on 13/05/16.
10+
*/
11+
public class DateDurationCheck extends Check {
12+
13+
Long val1;
14+
Long val2;
15+
16+
public DateDurationCheck(String name, Integer baseColumn, Integer offsetColumn, Long val1, Long val2 ) {
17+
super(name, new ArrayList<Integer>());
18+
getColumns().add(baseColumn);
19+
getColumns().add(offsetColumn);
20+
this.val1 = val1;
21+
this.val2 = val2;
22+
}
23+
24+
@Override
25+
public boolean check(List<String> values) {
26+
Long date = Long.valueOf(values.get(0)) + Long.valueOf(values.get(1))* DateGenerator.ONE_DAY;
27+
return date <= val2 && date >= val1;
28+
}
29+
}

‎src/test/java/ldbc/snb/datagen/test/csv/LongCheck.java

Lines changed: 14 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -5,42 +5,49 @@
55
*/
66
public class LongCheck extends NumericCheck<Long> {
77

8+
long offset = 0;
9+
10+
public LongCheck(Parser<Long> parser, String name, Integer column, NumericCheckType type, Long val1, Long val2, Long offset ) {
11+
super(parser, name, column, type, val1, val2);
12+
this.offset = offset;
13+
}
14+
815
public LongCheck(Parser<Long> parser, String name, Integer column, NumericCheckType type, Long val1, Long val2) {
916
super(parser, name, column, type, val1, val2);
1017
}
1118

1219
@Override
1320
public boolean greater(Long val1, Long val2) {
14-
return val1 > val2;
21+
return val1+offset > val2;
1522
}
1623

1724
@Override
1825
public boolean greaterEqual(Long val1, Long val2) {
19-
return val1 >= val2;
26+
return val1+offset >= val2;
2027
}
2128

2229
@Override
2330
public boolean less(Long val1, Long val2) {
24-
return val1 < val2;
31+
return val1+offset < val2;
2532
}
2633

2734
@Override
2835
public boolean lessEqual(Long val1, Long val2) {
29-
return val1 <= val2;
36+
return val1+offset <= val2;
3037
}
3138

3239
@Override
3340
public boolean equals(Long val1, Long val2) {
34-
return val1 == val2;
41+
return val1+offset == val2;
3542
}
3643

3744
@Override
3845
public boolean nonEquals(Long val1, Long val2) {
39-
return val1 != val2;
46+
return val1+offset != val2;
4047
}
4148

4249
@Override
4350
public boolean between(Long val1, Long val2, Long val3) {
44-
return (val1 >= val2 && val1 < val3);
51+
return (val1+offset >= val2 && val1+offset < val3);
4552
}
4653
}

‎src/test/java/ldbc/snb/datagen/test/csv/NumericCheck.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
*/
99
public abstract class NumericCheck<T extends Number > extends Check {
1010

11-
enum NumericCheckType {
11+
public enum NumericCheckType {
1212
G,
1313
GE,
1414
L,

0 commit comments

Comments
 (0)