Fixed param generation scripts to work with the new factors version

Fixed minor bug at writing factors table

Author: ArnauPrat

paramgenerator/discoverparams.pyc

@@ -34,8 +34,6 @@ def findNameParameters(names, amount = 100):
 		i -= 1
 	return res
 
-
-
 class CSVSerializer:
 	def __init__(self):
 		self.handlers = []
@@ -130,19 +128,23 @@ def main(argv=None):
 		return 1
 
 	indir = argv[1]+"/"
-	factorFiles=[]
+	activityFactorFiles=[]
+	personFactorFiles=[]
 	friendsFiles = []
 	outdir = argv[2]+"/"
 	random.seed(SEED)
 
+
 	for file in os.listdir(indir):
-		if file.endswith("factors.txt"):
-			factorFiles.append(indir+file)
+		if file.endswith("activityFactors.txt"):
+			activityFactorFiles.append(indir+file)
+		if file.endswith("personFactors.txt"):
+			personFactorFiles.append(indir+file)
 		if file.startswith("m0friendList"):
 			friendsFiles.append(indir+file)
 
 	# read precomputed counts from files	
-	(personFactors, countryFactors, tagFactors, tagClassFactors, nameFactors, givenNames,  ts, postHisto) = readfactors.load(factorFiles, friendsFiles)
+	(personFactors, countryFactors, tagFactors, tagClassFactors, nameFactors, givenNames,  ts, postHisto) = readfactors.load(personFactorFiles, activityFactorFiles, friendsFiles)
 
 	# find person parameters
 	print "find parameter bindings for Persons"
@@ -234,7 +236,7 @@ def main(argv=None):
 	}
 
 	print "find parameter bindings for Timestamps"
-	selectedTimeParams = findTimeParams(timeSelectionInput, factorFiles, friendsFiles, ts[1])
+	selectedTimeParams = findTimeParams(timeSelectionInput, personFactorFiles, activityFactorFiles, friendsFiles, ts[1])
 	# Query 11 takes WorksFrom timestamp
 	selectedTimeParams[11] = [random.randint(ts[2], ts[3]) for j in range(len(selectedPersonParams[11]))]
 

paramgenerator/generateparams.py

@@ -299,18 +299,21 @@ def main(argv=None):
       return 1
 
    indir = argv[1]+"/"
-   factorFiles=[]
+   activityFactorFiles=[]
+   personFactorFiles=[]
    friendsFiles = []
    outdir = argv[2]+"/"
 
    for file in os.listdir(indir):
-      if file.endswith("factors.txt"):
-         factorFiles.append(indir+file)
+      if file.endswith("activityFactors.txt"):
+         activityFactorFiles.append(indir+file)
+      if file.endswith("personFactors.txt"):
+         personFactorFiles.append(indir+file)
       if file.startswith("m0friendList"):
          friendsFiles.append(indir+file)
 
    # read precomputed counts from files   
-   (personFactors, countryFactors, tagFactors, tagClassFactors, nameFactors, givenNames,  ts, postsHisto) = readfactors.load(factorFiles, friendsFiles)
+   (personFactors, countryFactors, tagFactors, tagClassFactors, nameFactors, givenNames,  ts, postsHisto) = readfactors.load(personFactorFiles,activityFactorFiles, friendsFiles)
    week_posts = convert_posts_histo(postsHisto)
 
    persons = []

paramgenerator/generateparamsbi.py

@@ -53,7 +53,7 @@ def setValue(self, person, value):
 	def getValue(self, person):
 		return self.values[person]
 
-def load(factorFiles, friendFiles):
+def load(personFactorFiles,activityFactorFiles, friendFiles):
 	print "loading input for parameter generation"
 	results = Factors()
 	countries = Factors()
@@ -65,12 +65,10 @@ def load(factorFiles, friendFiles):
 	names = {}
 	timestamp = [0,0,0,0]
 
-	for inputFileName in factorFiles:
-		with codecs.open(inputFileName, "r", "utf-8") as f:
-			line = f.readline()
-			personCount = int(line)
-			for i in range(personCount):
-				line = f.readline().split(",")
+	for inputfileName in personFactorFiles:
+		with codecs.open(inputfileName, "r", "utf-8") as f:
+			for line in f.readlines():
+				line = line.split(",")
 				person = int(line[0])
 				if not results.existParam(person):
 					results.addNewParam(person)
@@ -88,6 +86,8 @@ def load(factorFiles, friendFiles):
 						postsHisto.addNewParam(i)
 					postsHisto.addValue(i, "p", int(line[9+i]))
 
+	for inputFileName in activityFactorFiles:
+		with codecs.open(inputFileName, "r", "utf-8") as f:
 			countryCount = int(f.readline())
 			for i in range(countryCount):
 				line = f.readline().split(",")

paramgenerator/readfactors.py

@@ -143,18 +143,17 @@ def computeTimeMedians(factors, lastmonthcount = LAST_MONTHS):
 
 	return (medianFirstMonth, medianLastMonth, median)	
 
-def readTimeParams(persons, factorFiles, friendFiles):
+def readTimeParams(persons, personFactorFiles, activityFactorFiles, friendFiles):
 
 	postCounts = {}
 	groupCounts = {}
 	offset = 8
 	monthcount = 12*3 + 1
 
-	for inputFactorFile in factorFiles:
+	for inputFactorFile in personFactorFiles:
 		with open(inputFactorFile, 'r') as f:
-			personCount = int(f.readline())
-			for i in range(personCount):
-				line = f.readline().split(",")
+			for line in f.readlines():
+				line = line.split(",")
 				person = int(line[0])
 				localPostCounts = map(int,line[offset:offset+monthcount])
 				localGroupCounts = map(int, line[offset+monthcount:])
@@ -209,15 +208,15 @@ def readTimeParams(persons, factorFiles, friendFiles):
 
 
 
-def findTimeParams(input, factorFiles, friendFiles, startYear):
+def findTimeParams(input, personFactorFiles, activityFactorFiles, friendFiles, startYear):
 	START_YEAR = startYear
 	fPostCount = {}
 	ffPostCount = {}
 	persons = []
 	for queryId in input:
 		persons += input[queryId][0]
 
-	(fPostCount, ffPostCount, ffGroupCount) = readTimeParams(set(persons),factorFiles, friendFiles)
+	(fPostCount, ffPostCount, ffGroupCount) = readTimeParams(set(persons),personFactorFiles, activityFactorFiles, friendFiles)
 
 	mapParam = {
 		"f" : fPostCount,

paramgenerator/readfactors.pyc

@@ -304,52 +304,6 @@ public void writePersonFactors(OutputStream writer ) {
 
     public void writeActivityFactors(OutputStream writer ) {
         try {
-            /*Iterator<Map.Entry<Long,PersonCounts>> iter = personCounts_.entrySet().iterator();
-            while(iter.hasNext()) {
-                Map.Entry<Long,PersonCounts> entry = iter.next();
-                if (medianFirstName_.get(entry.getKey()) == null) {
-                    iter.remove();
-                }
-            }*/
-            writer.write(Integer.toString(personCounts_.size()).getBytes("UTF8"));
-            writer.write("\n".getBytes("UTF8"));
-            for (Map.Entry<Long, PersonCounts> c: personCounts_.entrySet()){
-            	PersonCounts count = c.getValue();
-            	// correct the group counts
-            	//count.numberOfGroups += count.numberOfFriends;
-                String name = medianFirstName_.get(c.getKey());
-                if( name != null ) {
-                StringBuffer strbuf = new StringBuffer();
-                strbuf.append(c.getKey()); strbuf.append(",");
-                    strbuf.append(name);
-                    strbuf.append(",");
-                    strbuf.append(count.numFriends());
-                    strbuf.append(",");
-                    strbuf.append(count.numPosts());
-                    strbuf.append(",");
-                    strbuf.append(count.numLikes());
-                    strbuf.append(",");
-                    strbuf.append(count.numTagsOfMessages());
-                    strbuf.append(",");
-                    strbuf.append(count.numForums());
-                    strbuf.append(",");
-                    strbuf.append(count.numWorkPlaces());
-                    strbuf.append(",");
-                    strbuf.append(count.numComments());
-                    strbuf.append(",");
-
-                    for (Long bucket : count.numMessagesPerMonth()) {
-                        strbuf.append(bucket);
-                        strbuf.append(",");
-                    }
-                    for (Long bucket : count.numForumsPerMonth()) {
-                        strbuf.append(bucket);
-                        strbuf.append(",");
-                    }
-                    strbuf.setCharAt(strbuf.length() - 1, '\n');
-                    writer.write(strbuf.toString().getBytes("UTF8"));
-                }
-            }
             writer.write(Integer.toString(postsPerCountry_.size()).getBytes("UTF8"));
             writer.write("\n".getBytes("UTF8"));
             for (Map.Entry<Integer, Long> c: postsPerCountry_.entrySet()){