remove bad rules

Author: xchrdw

propertysuggester/analyzer/RuleGenerator.py

@@ -4,7 +4,7 @@
 from propertysuggester.utils.datamodel import Entity
 
 
-def compute_rules(entities):
+def compute_rules(entities, min_probability=0.01):
     """
     @type entities: collections.Iterable[Entity]
     @return: list[Rule]
@@ -18,7 +18,7 @@ def compute_rules(entities):
         for analyzer in analyzers:
             analyzer.process(entity)
 
-    rules = itertools.chain(*(a.get_rules() for a in analyzers))
+    rules = filter(lambda rule: rule.probability > min_probability, itertools.chain(*(a.get_rules() for a in analyzers)))
     return rules
 
 
@@ -47,7 +47,7 @@ def get_rules(self):
             for pid2, value in row.iteritems():
                 if value > 0:
                     probability = value/float(pid1count)
-                    rules.append(Rule(pid1, None, pid2, pid1count, probability, self.context))
+                    rules.append(Rule(pid1, None, pid2, value, probability, self.context))
         return rules
 
 

propertysuggester/test/analyzer/test_rule_generator.py

@@ -27,7 +27,7 @@ def setUp(self):
 
     def test_table_generator(self):
         rules = list(RuleGenerator.compute_rules(test_data1))
-        self.assertThat(rules, ContainsAll([Rule(31, None, 373, 2, 0.5, "item"), Rule(373, None, 31, 1, 1.0, "item")]))
+        self.assertThat(rules, ContainsAll([Rule(31, None, 373, 1, 0.5, "item"), Rule(373, None, 31, 1, 1.0, "item")]))
 
 
     def test_table_with_multiple_occurance(self):