Merge pull request #11 from Wikidata-lib/ConsiderClassifyingProperties

Consider classifying properties

Author: tobijat

propertysuggester/analyzer/RuleGenerator.py

@@ -1,18 +1,24 @@
+import ConfigParser
+import os
 from collections import defaultdict
 import itertools
 from propertysuggester.analyzer.impl.MainAnalyzer import ItemAnalyzer
 from propertysuggester.analyzer.impl.QualifierReferenceAnalyzer import QualifierAnalyzer, ReferenceAnalyzer
 from propertysuggester.analyzer.rule import Rule
 from propertysuggester.utils.datamodel import Entity
 
+config = ConfigParser.ConfigParser()
+config.read(os.path.join(os.path.abspath(os.path.dirname(__file__)), 'analyzer.ini'))
+classifying_pids = config.get("mainAnalyzer","classifying_properties").split(",")
+classifying_pids = map(int, classifying_pids)
 
 def compute_rules(entities, min_probability=0.01):
     """
     @type entities: collections.Iterable[Entity]
     @return: list[Rule]
     """
 
-    analyzers = [ItemAnalyzer(), QualifierAnalyzer(), ReferenceAnalyzer()]
+    analyzers = [ItemAnalyzer(classifying_pids), QualifierAnalyzer(), ReferenceAnalyzer()]
 
     for i, entity in enumerate(entities):
         if i % 100000 == 0 and i > 0:

propertysuggester/analyzer/analyzer.ini

@@ -0,0 +1,3 @@
+[mainAnalyzer]
+#List of classifying properties - at the moment limited to 'instance of' (31) and 'subclass of' (279)
+classifying_properties = 31,279

propertysuggester/analyzer/impl/MainAnalyzer.py

@@ -1,35 +1,42 @@
 from collections import defaultdict
-import math
 from propertysuggester.analyzer.impl.Analyzer import Analyzer
 from propertysuggester.analyzer.rule import Rule
 
-
 class ItemAnalyzer(Analyzer):
-    def __init__(self):
+    def __init__(self, classiying_property_ids = [31,279]):
         Analyzer.__init__(self)
-        self.property_occurances = defaultdict(int)
-        self.pair_occurances = defaultdict(lambda: defaultdict(int))
+        self.classiying_pids = classiying_property_ids
+        self.tuple_occurrences = defaultdict(int)
+        self.pair_occurrences = defaultdict(lambda: defaultdict(int))
 
-    def process(self, entity):
-        distinct_ids = set(claim.mainsnak.property_id for claim in entity.claims)
-        self._count_occurances(distinct_ids)
+    def process(self, item):
+        distinct_ids = set(claim.mainsnak.property_id for claim in item.claims)
+        property_value_pairs = [(claim.mainsnak.property_id, claim.mainsnak.value) for claim in item.claims]
+        self._count_occurrences(distinct_ids, property_value_pairs)
 
-    def _count_occurances(self, distinct_ids):
+    def _count_occurrences(self, distinct_ids, property_value_pairs):
         for pid1 in distinct_ids:
-            self.property_occurances[pid1] += 1
+            if pid1 in self.classiying_pids:
+                continue
+            currentTuple = (pid1, None)
+            self.tuple_occurrences[currentTuple] += 1
             for pid2 in distinct_ids:
                 if pid1 != pid2:
-                    self.pair_occurances[pid1][pid2] += 1
+                    self.pair_occurrences[currentTuple][pid2] += 1
+
+        for pid1, value in property_value_pairs:
+            if pid1 in self.classiying_pids and value[1:].isdigit():
+                self.tuple_occurrences[pid1, int(value[1:])] += 1
+                for pid2 in distinct_ids:
+                    if pid1 != pid2:
+                        self.pair_occurrences[pid1, int(value[1:])][pid2] += 1
 
     def get_rules(self):
         rules = []
-        totalpropertycount = len(self.property_occurances)
-        for pid1, row in self.pair_occurances.iteritems():
-            sharedpids = len(row)
-            idf = math.log(totalpropertycount/float(sharedpids))
-            pid1count = self.property_occurances[pid1]
+        for (pid1, value), row in self.pair_occurrences.iteritems():
+            pid1count = self.tuple_occurrences[pid1, value]
             for pid2, paircount in row.iteritems():
                 if paircount > 0:
-                    probability = (paircount/float(pid1count)) * idf
-                    rules.append(Rule(pid1, None, pid2, paircount, probability, "item"))
+                    probability = (paircount/float(pid1count))
+                    rules.append(Rule(pid1, value, pid2, paircount, probability, "item"))
         return rules

propertysuggester/test/analyzer/test_rule_generator.py

@@ -1,5 +1,4 @@
 import unittest
-import math
 
 from testtools import TestCase
 from testtools.matchers import *
@@ -11,7 +10,8 @@
 
 test_data1 = [Entity('Q15', [Claim(Snak(31, 'wikibase-entityid', 'Q5107')),
                              Claim(Snak(373, 'string', 'Africa'))]),
-              Entity('Q16', [Claim(Snak(31, 'wikibase-entityid', 'Q384'))])]
+              Entity('Q16', [Claim(Snak(31, 'wikibase-entityid', 'Q5107'))]),
+              Entity('Q17', [Claim(Snak(31, 'wikibase-entityid', 'Q1337'))])]
 
 test_data2 = [Entity('Q15', [Claim(Snak(31, 'wikibase-entityid', 'Q5107')),
                              Claim(Snak(373, 'string', 'Africa')),
@@ -28,13 +28,12 @@ def setUp(self):
 
     def test_table_generator(self):
         rules = list(RuleGenerator.compute_rules(test_data1))
-        self.assertThat(rules, ContainsAll([Rule(31, None, 373, 1, 0.5*math.log(2), "item"),
-                                            Rule(373, None, 31, 1, 1.0*math.log(2), "item")]))
+        self.assertThat(rules, ContainsAll([Rule(31, 5107, 373, 1, 0.5, "item"), Rule(373, None, 31, 1, 1.0, "item")]))
+
 
     def test_table_with_multiple_occurance(self):
         rules = list(RuleGenerator.compute_rules(test_data2))
-        self.assertThat(rules, ContainsAll([Rule(31, None, 373, 1, 1.0*math.log(2), "item"),
-                                            Rule(373, None, 31, 1, 1.0*math.log(2), "item")]))
+        self.assertThat(rules, ContainsAll([Rule(31, 5107, 373, 1, 1.0, "item"), Rule(373, None, 31, 1, 1.0, "item")]))
 
     def test_table_with_qualifier_and_references(self):
         rules = list(RuleGenerator.compute_rules(test_data3))

propertysuggester/test/parser/Wikidata-Q1.json.gz

@@ -30,6 +30,10 @@ nosetests
 
 ## Release Notes
 
+### 1.2
+* Consider classifying Properties
+* use Json dumps for analysis
+
 ### 1.1
 * Generate associationrules for qualifier and references
 * Improve ranking to avoid suggestions of human properties