Correlated Name parameters for Query 1

Author: agubichev

paramgenerator/generateparams.py

@@ -32,9 +32,10 @@ def findNameParameters(names, amount = 100):
 	while  counts[mid] - counts[i] < 0.1 * counts[mid]:
 		res.extend([name for name in hist[counts[i]]])
 		i -= 1
-
 	return res
 
+
+
 class CSVSerializer:
 	def __init__(self):
 		self.handlers = []
@@ -141,7 +142,7 @@ def main(argv=None):
 			friendsFiles.append(indir+file)
 
 	# read precomputed counts from files	
-	(personFactors, countryFactors, tagFactors, tagClassFactors, nameFactors, ts) = readfactors.load(factorFiles, friendsFiles)
+	(personFactors, countryFactors, tagFactors, tagClassFactors, nameFactors, givenNames,  ts) = readfactors.load(factorFiles, friendsFiles)
 
 	# find person parameters
 	print "find parameter bindings for Persons"
@@ -233,12 +234,17 @@ def main(argv=None):
 		HS.append((HS0, HS1))
 
 	# Query 1 takes first name as a parameter
-	nameParams =  findNameParameters(nameFactors)# discoverparams.generate(nameFactors)
-	# if there are fewer first names than person parameters, repeat some of the names
-	if len(nameParams) < len(selectedPersonParams[2]):
-		oldlen = len(nameParams)
-		newlen = len(selectedPersonParams[2])
-		nameParams.extend([nameParams[random.randint(0, oldlen-1)] for j in range(newlen-oldlen)])
+	#nameParams =  findNameParameters(nameFactors)# discoverparams.generate(nameFactors)
+	## if there are fewer first names than person parameters, repeat some of the names
+	#if len(nameParams) < len(selectedPersonParams[2]):
+	#	oldlen = len(nameParams)
+	#	newlen = len(selectedPersonParams[2])
+	#	nameParams.extend([nameParams[random.randint(0, oldlen-1)] for j in range(newlen-oldlen)])
+	nameParams = []
+	for person in selectedPersonParams[1]:
+		nameParams.append(givenNames.getValue(person))
+
+	print nameParams
 
 	# serialize all the parameters as CSV
 	csvWriters = {}

paramgenerator/readfactors.py

@@ -41,12 +41,23 @@ def getValue(self, person, factor):
 	def addValue(self, person, factor, value):
 		self.values[person].addValue(factor, value)
 
+class NameParameter:
+	def __init__(self, persons=[]):
+		self.values={}
+		for p in persons:
+			self.values[p] = ""
+	
+	def setValue(self, person, value):
+		self.values[person] = value
+
+	def getValue(self, person):
+		return self.values[person]
 
 def load(factorFiles, friendFiles):
 	print "loading input for parameter generation"
 	results = Factors()
 	countries = Factors()
-
+	givenNames = NameParameter()
 
 	tagClasses = {}
 	tags = {}
@@ -62,13 +73,14 @@ def load(factorFiles, friendFiles):
 				person = int(line[0])
 				if not results.existParam(person):
 					results.addNewParam(person)
-				results.addValue(person, "f", int(line[1]))
-				results.addValue(person, "p", int(line[2]))
-				results.addValue(person, "pl", int(line[3]))
-				results.addValue(person, "pt", int(line[4]))
-				results.addValue(person, "g", int(line[5]))
-				results.addValue(person, "w", int(line[6]))
-				results.addValue(person, "pr", int(line[7]))
+				givenNames.setValue(person, line[1])
+				results.addValue(person, "f", int(line[2]))
+				results.addValue(person, "p", int(line[3]))
+				results.addValue(person, "pl", int(line[4]))
+				results.addValue(person, "pt", int(line[5]))
+				results.addValue(person, "g", int(line[6]))
+				results.addValue(person, "w", int(line[7]))
+				results.addValue(person, "pr", int(line[8]))
 
 			countryCount = int(f.readline())
 			for i in range(countryCount):
@@ -120,7 +132,7 @@ def load(factorFiles, friendFiles):
 
 	loadFriends(friendFiles, results)
 
-	return (results, countries, tags.items(), tagClasses.items(), names.items(), timestamp)
+	return (results, countries, tags.items(), tagClasses.items(), names.items(), givenNames,timestamp)
 
 def loadFriends(friendFiles, factors):
 

src/main/java/ldbc/socialnet/dbgen/dictionary/NamesDictionary.java

@@ -219,5 +219,17 @@ public String getRandomGivenName(Random random, int locationId, boolean isMale,
 
 		return name;
 	}
+
+    /**
+     *  return a given name which is the median of topN for a given location/gender/year
+     *  we use it for parameter generation
+     */
+    public String getMedianGivenName(int locationId, boolean isMale, int birthYear){
+        int period = 0;
+        Vector<HashMap<Integer, Vector<String>>> target = (isMale) ? givenNamesByLocationsMale : givenNamesByLocationsFemale;
+        int size = target.get(period).get(locationId).size();
+        String name = target.get(period).get(locationId).get(size/2);
+        return name;
+    }
 }
 

src/main/java/ldbc/socialnet/dbgen/generator/ScalableGenerator.java

@@ -252,6 +252,7 @@ public enum OrganisationType {
     private HashMap<Integer, Integer> tagClassCount;
     private HashMap<String, Integer> firstNameCount;
     private HashMap<Integer, Integer> tagNameCount;
+    private HashMap<Long, String> medianFirstName;
     // For blocking
     private static final int  reducerShift[] = { 26, 8, 1 };
 
@@ -429,6 +430,7 @@ public ScalableGenerator(int threadId, Configuration conf ){
         this.postsPerCountry = new HashMap<Integer, Integer>();
         this.tagClassCount = new HashMap<Integer, Integer>();
         this.firstNameCount = new HashMap<String, Integer>();
+        this.medianFirstName = new HashMap<Long, String>();
         this.tagNameCount = new HashMap<Integer, Integer>();
         if (threadId != -1){
             outUserProfile = "mr" + threadId + "_" + outUserProfileName;
@@ -793,6 +795,9 @@ public void generateUserActivity( ReducedUserProfile userProfile, Reducer<MapRed
         dataExporter.export(userInfo);
         int nameCount = firstNameCount.containsKey(extraInfo.getFirstName())? firstNameCount.get(extraInfo.getFirstName()):0;
         firstNameCount.put(extraInfo.getFirstName(), nameCount+1);
+        String medianName = namesDictionary.getMedianGivenName(userProfile.getLocationId(),
+                userProfile.getGender()==1, dateTimeGenerator.getBirthYear(userProfile.getBirthDay()));
+        medianFirstName.put(userProfile.getAccountId(), medianName);
         long init = System.currentTimeMillis();
         if(conf.getBoolean("activity",true)) {
             Group wall = generateWall(userInfo);
@@ -1570,7 +1575,9 @@ private void writeFactorTable(){
             	// correct the group counts
             	//count.numberOfGroups += count.numberOfFriends;
             	StringBuffer strbuf = new StringBuffer();
-            	strbuf.append(c.getKey()); strbuf.append(",");
+		strbuf.append(c.getKey()); strbuf.append(",");
+		String name = medianFirstName.get(c.getKey());
+		strbuf.append(name); strbuf.append(",");
             	strbuf.append(count.numberOfFriends); 		strbuf.append(",");
             	strbuf.append(count.numberOfPosts); 		strbuf.append(",");
             	strbuf.append(count.numberOfLikes); 		strbuf.append(",");