Extended BI parameter generation

Author: mkaufmann

paramgenerator/generateparams.py

@@ -142,7 +142,7 @@ def main(argv=None):
 			friendsFiles.append(indir+file)
 
 	# read precomputed counts from files	
-	(personFactors, countryFactors, tagFactors, tagClassFactors, nameFactors, givenNames,  ts) = readfactors.load(factorFiles, friendsFiles)
+	(personFactors, countryFactors, tagFactors, tagClassFactors, nameFactors, givenNames,  ts, postHisto) = readfactors.load(factorFiles, friendsFiles)
 
 	# find person parameters
 	print "find parameter bindings for Persons"

paramgenerator/generateparamsbi.py

@@ -9,7 +9,6 @@
 from timeparameters import *
 from calendar import timegm
 
-
 # class ParamsWriter:
 #    def __init__(self, name, num_params):
 #       self.files = []
@@ -93,18 +92,18 @@ def post_month_params(sample, lower_bound, upper_bound):
          results.append([[start_day, end_day], count_sum])
    return results
 
-def post_three_month_params(sample, lower_bound, upper_bound):
-   results = []
-   for ix in range(0, len(sample)/12):
-      start_ix = ix*12
-      count_sum = 0
-      for offset, count in sample[start_ix:start_ix+12]:
-         count_sum += count
-      if count_sum > lower_bound and count_sum < upper_bound:
-         start_day = sample[start_ix][0]
-         end_day = sample[start_ix+12][0]
-         results.append([[start_day, end_day], count_sum])
-   return results
+# def post_three_month_params(sample, lower_bound, upper_bound):
+#    results = []
+#    for ix in range(0, len(sample)/12):
+#       start_ix = ix*12
+#       count_sum = 0
+#       for offset, count in sample[start_ix:start_ix+12]:
+#          count_sum += count
+#       if count_sum > lower_bound and count_sum < upper_bound:
+#          start_day = sample[start_ix][0]
+#          end_day = sample[start_ix+12][0]
+#          results.append([[start_day, end_day], count_sum])
+#    return results
 
 
 def key_params(sample, lower_bound, upper_bound):
@@ -115,38 +114,24 @@ def key_params(sample, lower_bound, upper_bound):
    return results
 
 def serialize_q1(post_weeks):
-   f1 = open('params/q1.1.params', 'w+')
-   fcounts = open('params/q1.counts.params', 'w+')
+   writer = ParamsWriter("q1", 1)
    for week, count in post_weeks:
-      f1.write(str(week)+"\n")
-      fcounts.write(str(count)+"\n")
+      writer.append([str(week)], [count])
 
 def serialize_q2(country_sets, post_day_ranges):
-   # Generate Q2 params
-   f1 = open('params/q2.1.params', 'w+')
-   f2 = open('params/q2.2.params', 'w+')
-   f3 = open('params/q2.3.params', 'w+')
-   fcounts = open('params/q2.counts.params', 'w+')
+   writer = ParamsWriter("q2", 3)
    random.seed(1988+2)
    for country_set, count_country in country_sets:
       for day_range, count_post in post_day_ranges:
          if random.randint(0,len(country_sets) + len(post_day_ranges)) == 0:
-            f1.write(str(day_range[0])+"\n")
-            f2.write(str(day_range[1])+"\n")
-            f3.write("ctry_name = '"+"' or ctry_name = '".join(country_set)+"'\n")
-            fcounts.write(str(count_post)+"|"+str(count_country)+"\n")
+            writer.append([str(day_range[0]), str(day_range[1]), ",".join(country_set)], [count_post,count_post,count_country])
 
 def serialize_q3(post_months):
-   # Generate Q2 params
-   f1 = open('params/q3.1.params', 'w+')
-   f2 = open('params/q3.2.params', 'w+')
-   fcounts = open('params/q3.counts.params', 'w+')
+   writer = ParamsWriter("q3", 2)
    for ix in range(0,len(post_months)):
       week_range_a, count_a = post_months[ix]
       for week_range_b, count_b in post_months[ix+1:]:
-         f1.write(str(week_range_a[0])+"\n")
-         f2.write(str(week_range_b[0])+"\n")
-         fcounts.write(str(count_a)+"|"+str(count_b)+"\n")
+         writer.append([str(week_range_a),str(week_range_b)], [count_a,count_b])
 
 def serialize_q4(tagclasses, countries):
    writer = ParamsWriter("q4", 2)
@@ -188,23 +173,19 @@ def serialize_q10(tags):
       writer.append([tag], [count])
 
 def serialize_q12(post_weeks):
-   f1 = open('params/q12.1.params', 'w+')
-   fcounts = open('params/q12.counts.params', 'w+')
+   writer = ParamsWriter("q12", 1)
    for week, count in post_weeks:
-      f1.write(str(week)+"\n")
-      fcounts.write(str(count)+"\n")
+      writer.append([str(week)], [count])
 
 def serialize_q13(countries):
    writer = ParamsWriter("q13", 1)
    for country, count in countries:
       writer.append([country], [count])
 
 def serialize_q14(creationdates):
-   f1 = open('params/q14.1.params', 'w+')
-   fcounts = open('params/q14.counts.params', 'w+')
+   writer = ParamsWriter("q14", 1)
    for creation, count in creationdates:
-      f1.write(str(creation[0])+"\n")
-      fcounts.write(str(count)+"\n")
+      writer.append([str(creation)], [count])
 
 def serialize_q15(countries):
    writer = ParamsWriter("q15", 1)
@@ -223,10 +204,9 @@ def serialize_q17(countries):
       writer.append([country], [count])
 
 def serialize_q18(post_weeks):
-   f1 = open('params/q18.1.params', 'w+')
-   fcounts = open('params/q18.counts.params', 'w+')
+   writer = ParamsWriter("q18", 1)
    for week, count in post_weeks:
-      f1.write(str(week)+"\n")
+      writer.append([str(week)], [count])
 
 def serialize_q19(tagclasses):
    writer = ParamsWriter("q19", 2)
@@ -257,6 +237,18 @@ def serialize_q24(tagclasses):
    for tagclass, count in tagclasses:
       writer.append([tagclass], [count])
 
+def convert_posts_histo(histogram):
+   week_posts = []
+   month = 0
+   while (histogram.existParam(month)):
+      monthTotal = histogram.getValue(month, "p")
+      week_posts.append([month*30, monthTotal/4])
+      week_posts.append([month*30+7, monthTotal/4])
+      week_posts.append([month*30+14, monthTotal/4])
+      week_posts.append([month*30+21, monthTotal/4])
+      month = month + 1
+   return week_posts
+
 def main(argv=None):
    if argv is None:
       argv = sys.argv
@@ -277,7 +269,8 @@ def main(argv=None):
          friendsFiles.append(indir+file)
 
    # read precomputed counts from files   
-   (personFactors, countryFactors, tagFactors, tagClassFactors, nameFactors, givenNames,  ts) = readfactors.load(factorFiles, friendsFiles)
+   (personFactors, countryFactors, tagFactors, tagClassFactors, nameFactors, givenNames,  ts, postsHisto) = readfactors.load(factorFiles, friendsFiles)
+   week_posts = convert_posts_histo(postsHisto)
 
    country_sample = []
    for key, value in countryFactors.values.iteritems():
@@ -293,21 +286,43 @@ def main(argv=None):
    total_posts = 0
    for day, count in tag_posts:
       total_posts += count
+
+   person_sum = 0
+   for country, count in country_sample:
+      person_sum += count
+
+   country_lower_threshold = 0.1*total_posts*0.9
+   country_upper_threshold = 0.1*total_posts*1.1
+   country_sets = country_sets_params(country_sample, country_lower_threshold, country_upper_threshold, 4)
+
+   post_lower_threshold = 0.1*total_posts*0.9
+   post_upper_threshold = 0.1*total_posts*1.1
+   post_day_ranges = post_date_range_params(week_posts, post_lower_threshold, post_upper_threshold)
 
+   post_lower_threshold = (total_posts/(week_posts[len(week_posts)-1][0]/7/4))*0.8
+   post_upper_threshold = (total_posts/(week_posts[len(week_posts)-1][0]/7/4))*1.2
+   post_months = post_month_params(week_posts, post_lower_threshold, post_upper_threshold)
+
+   serialize_q2(country_sets, post_day_ranges)
+   serialize_q3(post_months)
+   serialize_q14(post_month_params(week_posts, post_lower_threshold*2, post_upper_threshold*2))
+
+   serialize_q1(post_date_right_open_range_params(week_posts, 0.3*total_posts, 0.6*total_posts))
+   serialize_q12(post_date_right_open_range_params(week_posts, 0.3*total_posts, 0.6*total_posts))
+   serialize_q18(post_date_right_open_range_params(week_posts, 0.3*total_posts, 0.6*total_posts))
+
    serialize_q4(key_params(tagclass_posts, total_posts/20, total_posts/10), key_params(country_sample, total_posts/120, total_posts/70))
    serialize_q5(key_params(country_sample, total_posts/200, total_posts/100))
    serialize_q6(key_params(tag_posts, total_posts/1300, total_posts/900))
    serialize_q7(key_params(tag_posts, total_posts/900, total_posts/600))
    serialize_q8(key_params(tag_posts, total_posts/600, total_posts/300))
    serialize_q9(key_params(tagclass_posts, 6000, 25000))
    serialize_q10(key_params(tag_posts, total_posts/900, total_posts/600))
-   # serialize_q12(post_date_right_open_range_params(week_posts, 0.3*total_posts, 0.6*total_posts))
    serialize_q13(key_params(country_sample, total_posts/200, total_posts/100))
    # serialize_q14(post_month_params(week_posts, post_lower_threshold*2, post_upper_threshold*2))
    serialize_q15(key_params(country_sample, total_posts/200, total_posts/100))
    serialize_q16(key_params(tagclass_posts, total_posts/30, total_posts/10), key_params(country_sample, total_posts/110, total_posts/70))
    serialize_q17(key_params(country_sample, total_posts/200, total_posts/100))
-   # serialize_q18(post_date_right_open_range_params(week_posts, 0.3*total_posts, 0.6*total_posts))
    serialize_q19(key_params(tagclass_posts, total_posts/60, total_posts/10))
    serialize_q21(key_params(country_sample, total_posts/200, total_posts/100))
    serialize_q22(key_params(country_sample, total_posts/120, total_posts/40))

paramgenerator/readfactors.py

@@ -57,6 +57,7 @@ def load(factorFiles, friendFiles):
 	print "loading input for parameter generation"
 	results = Factors()
 	countries = Factors()
+	postsHisto = Factors()
 	givenNames = NameParameter()
 
 	tagClasses = {}
@@ -82,6 +83,10 @@ def load(factorFiles, friendFiles):
 				results.addValue(person, "g", int(line[6]))
 				results.addValue(person, "w", int(line[7]))
 				results.addValue(person, "pr", int(line[8]))
+				for i in range((len(line)-9)/2):
+					if not postsHisto.existParam(i):
+						postsHisto.addNewParam(i)
+					postsHisto.addValue(i, "p", int(line[9+i]))
 
 			countryCount = int(f.readline())
 			for i in range(countryCount):
@@ -123,7 +128,7 @@ def load(factorFiles, friendFiles):
 
 	loadFriends(friendFiles, results)
 
-	return (results, countries, tags.items(), tagClasses.items(), names.items(), givenNames,timestamp)
+	return (results, countries, tags.items(), tagClasses.items(), names.items(), givenNames,timestamp, postsHisto)
 
 def loadFriends(friendFiles, factors):
 

run.sh

@@ -26,7 +26,7 @@ if [ $PARAM_GENERATION -eq 1 ]
 then
 	mkdir -p substitution_parameters
 	python paramgenerator/generateparams.py $LDBC_SNB_DATAGEN_HOME substitution_parameters/
-   python paramgenerator/generateparamsbi.py $LDBC_SNB_DATAGEN_HOME substitution_parameters/
+	python paramgenerator/generateparamsbi.py $LDBC_SNB_DATAGEN_HOME substitution_parameters/
   	rm -f m*factors*
   	rm -f .m*factors*
 	rm -f m0friendList*