Added tests to check string lengths to be of size 256 characters at most

Author: ArnauPrat

src/main/java/ldbc/snb/datagen/generator/ForumGenerator.java

@@ -7,6 +7,7 @@
 import ldbc.snb.datagen.objects.Knows;
 import ldbc.snb.datagen.objects.Person;
 import ldbc.snb.datagen.util.RandomGeneratorFarm;
+import ldbc.snb.datagen.util.StringUtils;
 import ldbc.snb.datagen.vocabulary.SN;
 
 import java.util.ArrayList;
@@ -25,7 +26,7 @@ public Forum createWall(RandomGeneratorFarm randomFarm, long forumId, Person per
 		Forum forum = new Forum(SN.formId(SN.composeId(forumId,person.creationDate()+DatagenParams.deltaTime)), 
 				person.creationDate()+DatagenParams.deltaTime, 
 				new Person.PersonSummary(person),  
-				"Wall of " + person.firstName() + " " + person.lastName(),
+				StringUtils.clampString("Wall of " + person.firstName() + " " + person.lastName(), 256),
 				person.cityId(),
 				language
 				);
@@ -62,7 +63,7 @@ public Forum createGroup(RandomGeneratorFarm randomFarm, long forumId, Person pe
 		Forum forum = new Forum(SN.formId(SN.composeId(forumId,date)),
 				date, 
 				new Person.PersonSummary(person),  
-				"Group for " + Dictionaries.tags.getName(interestId).replace("\"","\\\"") + " in " + Dictionaries.places.getPlaceName(person.cityId()),
+				StringUtils.clampString("Group for " + Dictionaries.tags.getName(interestId).replace("\"","\\\"") + " in " + Dictionaries.places.getPlaceName(person.cityId()),256),
 				person.cityId(),
 				language
 				);
@@ -119,7 +120,7 @@ public Forum createAlbum(RandomGeneratorFarm randomFarm, long forumId, Person pe
 		Forum forum = new Forum(SN.formId(SN.composeId(forumId,date)),
 				date, 
 				new Person.PersonSummary(person),  
-				"Album " + numAlbum + " of " + person.firstName() + " " + person.lastName(),
+				StringUtils.clampString("Album " + numAlbum + " of " + person.firstName() + " " + person.lastName(),256),
 				person.cityId(),
 				language
 				);

src/main/java/ldbc/snb/datagen/hadoop/HadoopInvariantSerializer.java

@@ -8,6 +8,7 @@
 import ldbc.snb.datagen.objects.Tag;
 import ldbc.snb.datagen.objects.TagClass;
 import ldbc.snb.datagen.serializer.InvariantSerializer;
+import ldbc.snb.datagen.util.StringUtils;
 import org.apache.hadoop.conf.Configuration;
 
 import java.util.Iterator;
@@ -65,7 +66,7 @@ private void exportTagHierarchy(Tag tag) {
             exportedClasses_.add(classId);
             TagClass tagClass = new TagClass();
             tagClass.id = classId;
-            tagClass.name = Dictionaries.tags.getClassName(classId);
+            tagClass.name = StringUtils.clampString(Dictionaries.tags.getClassName(classId),256);
             tagClass.parent = Dictionaries.tags.getClassParent(tagClass.id);
             invariantSerializer_[nextFile()].export(tagClass);
             classId = tagClass.parent;
@@ -77,6 +78,7 @@ public void exportPlaces() {
         Iterator<Integer> it = locations.iterator();
         while(it.hasNext()) {
             Place place = Dictionaries.places.getLocation(it.next());
+            place.setName(StringUtils.clampString(place.getName(),256));
             invariantSerializer_[nextFile()].export(place);
         }
     }
@@ -88,7 +90,7 @@ public void exportOrganizations() {
             Organization company = new Organization();
             company.id = it.next();
             company.type = Organization.OrganisationType.company;
-            company.name = Dictionaries.companies.getCompanyName(company.id);
+            company.name = StringUtils.clampString(Dictionaries.companies.getCompanyName(company.id),256);
             company.location = Dictionaries.companies.getCountry(company.id);
             invariantSerializer_[nextFile()].export(company);
         }
@@ -99,7 +101,7 @@ public void exportOrganizations() {
             Organization university = new Organization();
             university.id = it.next();
             university.type = Organization.OrganisationType.university;
-            university.name = Dictionaries.universities.getUniversityName(university.id);
+            university.name = StringUtils.clampString(Dictionaries.universities.getUniversityName(university.id),256);
             university.location = Dictionaries.universities.getUniversityCity(university.id);
             invariantSerializer_[nextFile()].export(university);
         }
@@ -111,7 +113,7 @@ public void exportTags() {
         while(it.hasNext()) {
             Tag tag = new Tag();
             tag.id = it.next();
-            tag.name = Dictionaries.tags.getName(tag.id);
+            tag.name = StringUtils.clampString(Dictionaries.tags.getName(tag.id),256);
             tag.name.replace("\"", "\\\"");
             tag.tagClass = Dictionaries.tags.getTagClass(tag.id);
             invariantSerializer_[nextFile()].export(tag);

src/main/java/ldbc/snb/datagen/util/StringUtils.java

@@ -0,0 +1,12 @@
+package ldbc.snb.datagen.util;
+
+/**
+ * Created by aprat on 25/11/16.
+ */
+public class StringUtils {
+
+    public static String clampString( String str, int length) {
+        if(str.length() > length) return str.substring(0,length);
+        return str;
+    }
+}

src/test/java/ldbc/snb/datagen/test/LDBCDatagenTest.java

@@ -57,6 +57,7 @@ public void postTest() {
     @Test
     public void forumTest() {
         testIdUniqueness(dir+"/forum_0_0.csv", 0);
+        testStringLength(dir+"/forum_0_0.csv", 1, 256);
     }
 
     @Test
@@ -68,21 +69,25 @@ public void commentTest() {
     @Test
     public void organisationTest() {
         testIdUniqueness(dir+"/organisation_0_0.csv", 0);
+        testStringLength(dir+"/organisation_0_0.csv", 2, 256);
     }
 
     @Test
     public void placeTest() {
         testIdUniqueness(dir+"/place_0_0.csv", 0);
+        testStringLength(dir+"/place_0_0.csv", 1, 256);
     }
 
     @Test
     public void tagTest() {
         testIdUniqueness(dir+"/tag_0_0.csv", 0);
+        testStringLength(dir+"/tag_0_0.csv", 1, 256);
     }
 
     @Test
     public void tagclassTest() {
         testIdUniqueness(dir+"/tagclass_0_0.csv", 0);
+        testStringLength(dir+"/tagclass_0_0.csv", 1, 256);
     }
 
     @Test
@@ -199,6 +204,7 @@ public void personLikesPostCheck() {
     @Test
     public void personEmailAddressCheck() {
         testIdExistance(dir+"/person_0_0.csv",0,dir+"/person_email_emailaddress_0_0.csv",0);
+        testStringLength(dir+"/person_email_emailaddress_0_0.csv", 1, 256);
     }
 
     // test update stream  time consistency
@@ -420,4 +426,11 @@ public void testIdExistance(String fileToCheckExistanceOf, int columnToCheckExis
         assertEquals("ERROR PASSING "+fileToCheckExistanceOf+" ID EXISTANCE TEST",true, fileChecker.run(1));
     }
 
+    public void testStringLength(String fileToCheckExistanceOf, int columnToCheckExistanceOf, int length) {
+        FileChecker fileChecker = new FileChecker(fileToCheckExistanceOf);
+        StringLengthCheck lengthCheck = new StringLengthCheck(columnToCheckExistanceOf, length);
+        fileChecker.addCheck(lengthCheck);
+        assertEquals("ERROR PASSING "+fileToCheckExistanceOf+" ID EXISTANCE TEST",true, fileChecker.run(1));
+    }
+
 }

src/test/java/ldbc/snb/datagen/test/csv/StringLengthCheck.java

@@ -0,0 +1,29 @@
+package ldbc.snb.datagen.test.csv;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+
+/**
+ * Created by aprat on 18/12/15.
+ */
+public class StringLengthCheck extends Check {
+
+    private int length;
+
+
+
+    public StringLengthCheck(int column, int length) {
+        super( "String length check of "+length+" unicode characters", (new ArrayList<Integer>()));
+        this.length = length;
+        this.getColumns().add(column);
+    }
+
+    @Override
+    public boolean check(List<String> vals) {
+        for(String value : vals) {
+            if(value.length() > length) return false;
+        }
+        return true;
+    }
+}