Merge branch 'master' of https://github.com/ldbc/ldbc_snb_datagen

Author: ArnauPrat

paramgenerator/generateparams.py

@@ -32,9 +32,10 @@ def findNameParameters(names, amount = 100):
 	while  counts[mid] - counts[i] < 0.1 * counts[mid]:
 		res.extend([name for name in hist[counts[i]]])
 		i -= 1
-
 	return res
 
+
+
 class CSVSerializer:
 	def __init__(self):
 		self.handlers = []
@@ -49,7 +50,7 @@ def registerHandler(self, handler, inputParams, header):
 		self.inputs.append(inputParams)
 
 	def writeCSV(self):
-		output = codecs.open(self.outputFile, "w", encoding="utf-8")
+		output = codecs.open( self.outputFile, "w",encoding="utf-8")
 
 		if len(self.inputs) == 0:
 			return
@@ -65,8 +66,7 @@ def writeCSV(self):
 				handler = self.handlers[j]
 				data = self.inputs[j][i]
 				csvLine.append(handler(data))
-
-			output.write("|".join(csvLine))
+			output.write('|'.join([s for s in csvLine]))
 			output.write("\n")
 		output.close()
 
@@ -75,6 +75,7 @@ def handlePersonParam(person):
 	#return {"PersonID": person, "PersonURI":(PERSON_PREFIX+str("%020d"%person))}
 
 def handleTimeParam(timeParam):
+	#print timeParam.year
 	#print timeParam.year
 	res =  str(timegm(date(year=int(timeParam.year), 
 		month=int(timeParam.month), day=int(timeParam.day)).timetuple())*1000)
@@ -141,7 +142,7 @@ def main(argv=None):
 			friendsFiles.append(indir+file)
 
 	# read precomputed counts from files	
-	(personFactors, countryFactors, tagFactors, tagClassFactors, nameFactors, ts) = readfactors.load(factorFiles, friendsFiles)
+	(personFactors, countryFactors, tagFactors, tagClassFactors, nameFactors, givenNames,  ts) = readfactors.load(factorFiles, friendsFiles)
 
 	# find person parameters
 	print "find parameter bindings for Persons"
@@ -233,12 +234,16 @@ def main(argv=None):
 		HS.append((HS0, HS1))
 
 	# Query 1 takes first name as a parameter
-	nameParams =  findNameParameters(nameFactors)# discoverparams.generate(nameFactors)
-	# if there are fewer first names than person parameters, repeat some of the names
-	if len(nameParams) < len(selectedPersonParams[2]):
-		oldlen = len(nameParams)
-		newlen = len(selectedPersonParams[2])
-		nameParams.extend([nameParams[random.randint(0, oldlen-1)] for j in range(newlen-oldlen)])
+	#nameParams =  findNameParameters(nameFactors)# discoverparams.generate(nameFactors)
+	## if there are fewer first names than person parameters, repeat some of the names
+	#if len(nameParams) < len(selectedPersonParams[2]):
+	#	oldlen = len(nameParams)
+	#	newlen = len(selectedPersonParams[2])
+	#	nameParams.extend([nameParams[random.randint(0, oldlen-1)] for j in range(newlen-oldlen)])
+	nameParams = []
+	for person in selectedPersonParams[1]:
+		n = givenNames.getValue(person)
+		nameParams.append(n)
 
 	# serialize all the parameters as CSV
 	csvWriters = {}

paramgenerator/readfactors.py

@@ -41,12 +41,23 @@ def getValue(self, person, factor):
 	def addValue(self, person, factor, value):
 		self.values[person].addValue(factor, value)
 
+class NameParameter:
+	def __init__(self, persons=[]):
+		self.values={}
+		for p in persons:
+			self.values[p] = 0
+	
+	def setValue(self, person, value):
+		self.values[person] = value
+
+	def getValue(self, person):
+		return self.values[person]
 
 def load(factorFiles, friendFiles):
 	print "loading input for parameter generation"
 	results = Factors()
 	countries = Factors()
-
+	givenNames = NameParameter()
 
 	tagClasses = {}
 	tags = {}
@@ -62,13 +73,15 @@ def load(factorFiles, friendFiles):
 				person = int(line[0])
 				if not results.existParam(person):
 					results.addNewParam(person)
-				results.addValue(person, "f", int(line[1]))
-				results.addValue(person, "p", int(line[2]))
-				results.addValue(person, "pl", int(line[3]))
-				results.addValue(person, "pt", int(line[4]))
-				results.addValue(person, "g", int(line[5]))
-				results.addValue(person, "w", int(line[6]))
-				results.addValue(person, "pr", int(line[7]))
+				name = line[1]
+				givenNames.setValue(person, name)
+				results.addValue(person, "f", int(line[2]))
+				results.addValue(person, "p", int(line[3]))
+				results.addValue(person, "pl", int(line[4]))
+				results.addValue(person, "pt", int(line[5]))
+				results.addValue(person, "g", int(line[6]))
+				results.addValue(person, "w", int(line[7]))
+				results.addValue(person, "pr", int(line[8]))
 
 			countryCount = int(f.readline())
 			for i in range(countryCount):
@@ -91,11 +104,6 @@ def load(factorFiles, friendFiles):
 				line = f.readline()
 				count = line[1+line.rfind(","):]
 				name = line[:line.rfind(",")]
-				try:
-					name.decode('ascii')
-				except UnicodeEncodeError:
-					continue
-				
 				if not name in tags:
 					tags[name] = 0
 				tags[name] += int(count)
@@ -104,11 +112,6 @@ def load(factorFiles, friendFiles):
 			for i in range(nameCount):
 				line = f.readline().split(",")
 				name = line[0]
-				try:
-					name.decode('ascii')
-				except UnicodeEncodeError:
-					continue
-
 				if not name in names:
 					names[name] = 0
 				names[name] += int(line[1])
@@ -120,7 +123,7 @@ def load(factorFiles, friendFiles):
 
 	loadFriends(friendFiles, results)
 
-	return (results, countries, tags.items(), tagClasses.items(), names.items(), timestamp)
+	return (results, countries, tags.items(), tagClasses.items(), names.items(), givenNames,timestamp)
 
 def loadFriends(friendFiles, factors):
 

paramgenerator/timeparameters.py

@@ -1,7 +1,6 @@
 import sys
 import getopt
 import math
-import random
 from readfactors import FactorCount
 from operator import itemgetter
 import itertools
@@ -68,7 +67,7 @@ def getTimeParamsWithMedian(factors, (medianFirstMonth, medianLastMonth, median)
 	for values in factors:
 		input = sorted(values,key=lambda myc: (myc.year, myc.month))
 		currentMedian = getMedian(values,lambda myc: myc.count, True)
-		if int(median) == 0 or int(currentMedian.count) == 0:
+		if int(median) == 0 or int(currentMedian.count) == 0 or int(currentMedian.year) == 0:
 			res.append(TimeParameter(START_YEAR,1,1,0))
 			continue
 		if currentMedian.count > median:

src/main/java/ldbc/socialnet/dbgen/dictionary/NamesDictionary.java

@@ -219,5 +219,17 @@ public String getRandomGivenName(Random random, int locationId, boolean isMale,
 
 		return name;
 	}
+
+    /**
+     *  return a given name which is the median of topN for a given location/gender/year
+     *  we use it for parameter generation
+     */
+    public String getMedianGivenName(int locationId, boolean isMale, int birthYear){
+        int period = 0;
+        Vector<HashMap<Integer, Vector<String>>> target = (isMale) ? givenNamesByLocationsMale : givenNamesByLocationsFemale;
+        int size = target.get(period).get(locationId).size();
+        String name = target.get(period).get(locationId).get(size/2);
+        return name;
+    }
 }
 

src/main/java/ldbc/socialnet/dbgen/generator/ScalableGenerator.java

@@ -252,6 +252,7 @@ public enum OrganisationType {
     private HashMap<Integer, Integer> tagClassCount;
     private HashMap<String, Integer> firstNameCount;
     private HashMap<Integer, Integer> tagNameCount;
+    private HashMap<Long, String> medianFirstName;
     // For blocking
     private static final int  reducerShift[] = { 26, 8, 1 };
 
@@ -429,6 +430,7 @@ public ScalableGenerator(int threadId, Configuration conf ){
         this.postsPerCountry = new HashMap<Integer, Integer>();
         this.tagClassCount = new HashMap<Integer, Integer>();
         this.firstNameCount = new HashMap<String, Integer>();
+        this.medianFirstName = new HashMap<Long, String>();
         this.tagNameCount = new HashMap<Integer, Integer>();
         if (threadId != -1){
             outUserProfile = "mr" + threadId + "_" + outUserProfileName;
@@ -793,6 +795,9 @@ public void generateUserActivity( ReducedUserProfile userProfile, Reducer<MapRed
         dataExporter.export(userInfo);
         int nameCount = firstNameCount.containsKey(extraInfo.getFirstName())? firstNameCount.get(extraInfo.getFirstName()):0;
         firstNameCount.put(extraInfo.getFirstName(), nameCount+1);
+        String medianName = namesDictionary.getMedianGivenName(userProfile.getLocationId(),
+                userProfile.getGender()==1, dateTimeGenerator.getBirthYear(userProfile.getBirthDay()));
+        medianFirstName.put(userProfile.getAccountId(), medianName);
         long init = System.currentTimeMillis();
         if(conf.getBoolean("activity",true)) {
             Group wall = generateWall(userInfo);
@@ -1570,7 +1575,9 @@ private void writeFactorTable(){
             	// correct the group counts
             	//count.numberOfGroups += count.numberOfFriends;
             	StringBuffer strbuf = new StringBuffer();
-            	strbuf.append(c.getKey()); strbuf.append(",");
+		strbuf.append(c.getKey()); strbuf.append(",");
+		String name = medianFirstName.get(c.getKey());
+		strbuf.append(name); strbuf.append(",");
             	strbuf.append(count.numberOfFriends); 		strbuf.append(",");
             	strbuf.append(count.numberOfPosts); 		strbuf.append(",");
             	strbuf.append(count.numberOfLikes); 		strbuf.append(",");