fix unicode problems

Author: agubichev

paramgenerator/generateparams.py

@@ -50,7 +50,7 @@ def registerHandler(self, handler, inputParams, header):
 		self.inputs.append(inputParams)
 
 	def writeCSV(self):
-		output = codecs.open(self.outputFile, "w", encoding="utf-8")
+		output = codecs.open( self.outputFile, "w",encoding="utf-8")
 
 		if len(self.inputs) == 0:
 			return
@@ -66,8 +66,7 @@ def writeCSV(self):
 				handler = self.handlers[j]
 				data = self.inputs[j][i]
 				csvLine.append(handler(data))
-
-			output.write("|".join(csvLine))
+			output.write('|'.join([s for s in csvLine]))
 			output.write("\n")
 		output.close()
 
@@ -76,6 +75,7 @@ def handlePersonParam(person):
 	#return {"PersonID": person, "PersonURI":(PERSON_PREFIX+str("%020d"%person))}
 
 def handleTimeParam(timeParam):
+	#print timeParam.year
 	#print timeParam.year
 	res =  str(timegm(date(year=int(timeParam.year), 
 		month=int(timeParam.month), day=int(timeParam.day)).timetuple())*1000)
@@ -242,9 +242,8 @@ def main(argv=None):
 	#	nameParams.extend([nameParams[random.randint(0, oldlen-1)] for j in range(newlen-oldlen)])
 	nameParams = []
 	for person in selectedPersonParams[1]:
-		nameParams.append(givenNames.getValue(person))
-
-	print nameParams
+		n = givenNames.getValue(person)
+		nameParams.append(n)
 
 	# serialize all the parameters as CSV
 	csvWriters = {}

paramgenerator/readfactors.py

@@ -45,7 +45,7 @@ class NameParameter:
 	def __init__(self, persons=[]):
 		self.values={}
 		for p in persons:
-			self.values[p] = ""
+			self.values[p] = 0
 
 	def setValue(self, person, value):
 		self.values[person] = value
@@ -73,7 +73,8 @@ def load(factorFiles, friendFiles):
 				person = int(line[0])
 				if not results.existParam(person):
 					results.addNewParam(person)
-				givenNames.setValue(person, line[1])
+				name = line[1]
+				givenNames.setValue(person, name)
 				results.addValue(person, "f", int(line[2]))
 				results.addValue(person, "p", int(line[3]))
 				results.addValue(person, "pl", int(line[4]))
@@ -103,11 +104,6 @@ def load(factorFiles, friendFiles):
 				line = f.readline()
 				count = line[1+line.rfind(","):]
 				name = line[:line.rfind(",")]
-				try:
-					name.decode('ascii')
-				except UnicodeEncodeError:
-					continue
-				
 				if not name in tags:
 					tags[name] = 0
 				tags[name] += int(count)
@@ -116,11 +112,6 @@ def load(factorFiles, friendFiles):
 			for i in range(nameCount):
 				line = f.readline().split(",")
 				name = line[0]
-				try:
-					name.decode('ascii')
-				except UnicodeEncodeError:
-					continue
-
 				if not name in names:
 					names[name] = 0
 				names[name] += int(line[1])

paramgenerator/timeparameters.py

@@ -1,7 +1,6 @@
 import sys
 import getopt
 import math
-import random
 from readfactors import FactorCount
 from operator import itemgetter
 import itertools
@@ -68,7 +67,7 @@ def getTimeParamsWithMedian(factors, (medianFirstMonth, medianLastMonth, median)
 	for values in factors:
 		input = sorted(values,key=lambda myc: (myc.year, myc.month))
 		currentMedian = getMedian(values,lambda myc: myc.count, True)
-		if int(median) == 0 or int(currentMedian.count) == 0:
+		if int(median) == 0 or int(currentMedian.count) == 0 or int(currentMedian.year) == 0:
 			res.append(TimeParameter(START_YEAR,1,1,0))
 			continue
 		if currentMedian.count > median: