Merge pull request #12 from Wikidata-lib/use_idf_to_rank_common_properties_lower

try to reduce errors from id-properties

Author: xchrdw

propertysuggester/analyzer/impl/MainAnalyzer.py

@@ -1,4 +1,5 @@
 from collections import defaultdict
+import math
 from propertysuggester.analyzer.impl.Analyzer import Analyzer
 from propertysuggester.analyzer.rule import Rule
 
@@ -22,10 +23,13 @@ def _count_occurances(self, distinct_ids):
 
     def get_rules(self):
         rules = []
+        totalpropertycount = len(self.property_occurances)
         for pid1, row in self.pair_occurances.iteritems():
+            sharedpids = len(row)
+            idf = math.log(totalpropertycount/float(sharedpids))
             pid1count = self.property_occurances[pid1]
             for pid2, paircount in row.iteritems():
                 if paircount > 0:
-                    probability = (paircount/float(pid1count))
+                    probability = (paircount/float(pid1count)) * idf
                     rules.append(Rule(pid1, None, pid2, paircount, probability, "item"))
         return rules

propertysuggester/test/analyzer/test_rule_generator.py

@@ -1,4 +1,5 @@
 import unittest
+import math
 
 from testtools import TestCase
 from testtools.matchers import *
@@ -27,12 +28,13 @@ def setUp(self):
 
     def test_table_generator(self):
         rules = list(RuleGenerator.compute_rules(test_data1))
-        self.assertThat(rules, ContainsAll([Rule(31, None, 373, 1, 0.5, "item"), Rule(373, None, 31, 1, 1.0, "item")]))
-
+        self.assertThat(rules, ContainsAll([Rule(31, None, 373, 1, 0.5*math.log(2), "item"),
+                                            Rule(373, None, 31, 1, 1.0*math.log(2), "item")]))
 
     def test_table_with_multiple_occurance(self):
         rules = list(RuleGenerator.compute_rules(test_data2))
-        self.assertThat(rules, ContainsAll([Rule(31, None, 373, 1, 1.0, "item"), Rule(373, None, 31, 1, 1.0, "item")]))
+        self.assertThat(rules, ContainsAll([Rule(31, None, 373, 1, 1.0*math.log(2), "item"),
+                                            Rule(373, None, 31, 1, 1.0*math.log(2), "item")]))
 
     def test_table_with_qualifier_and_references(self):
         rules = list(RuleGenerator.compute_rules(test_data3))