Merge pull request #10 from Wikidata-lib/suggestQualifiersAndReferences

Suggest qualifiers and references

Author: xchrdw

analyzer.py

@@ -2,7 +2,7 @@
 import sys
 import time
 
-from propertysuggester.analyzer import CsvGenerator, TableGenerator
+from propertysuggester.analyzer import CsvGenerator, RuleGenerator
 from propertysuggester.parser import CsvReader
 from propertysuggester.utils.CompressedFileType import CompressedFileType
 
@@ -16,8 +16,8 @@
 
     start = time.time()
     print "computing table"
-    t = TableGenerator.compute_table(CsvReader.read_csv(args.input))
+    rules = RuleGenerator.compute_rules(CsvReader.read_csv(args.input))
     print "writing csv"
-    CsvGenerator.create_pair_csv(t, args.output)
+    CsvGenerator.create_pair_csv(rules, args.output)
     print "done - {0:.2f}s".format(time.time()-start)
     print "now import this csv file with PropertySuggester/maintenance/UpdateTable.php"

propertysuggester/analyzer/CsvGenerator.py

@@ -1,22 +1,19 @@
 import csv
+from propertysuggester.analyzer.rule import Rule
 
 
-def create_pair_csv(table, out, delimiter=","):
+def create_pair_csv(rules, out, delimiter=","):
     """
-    @type table: dict[int, dict]
+    @type rules: list[Rule]
     @type out: file or StringIO.StringIO
     @type delimiter: string
     """
     csv_writer = csv.writer(out, delimiter=delimiter, quoting=csv.QUOTE_MINIMAL, lineterminator='\n')
-    print "properties: {0}".format(len(table))
 
     csv_writer.writerow(("pid1", "qid1", "pid2", "count", "probability", "context"))
     rowcount = 0
-    for pid1, row in table.iteritems():
-        for pid2, value in row.iteritems():
-            if pid1 != pid2 and isinstance(pid2, int) and value > 0:  # "appearances" is in the same table, ignore them
-                probability = value/float(row["appearances"])
-                csv_writer.writerow((pid1, '', pid2, value, probability, 'item'))
-                rowcount += 1
-                if not rowcount % 1000:
-                    print "rows {0}".format(rowcount)
+    for rule in rules:
+        csv_writer.writerow((rule.pid1, rule.qid1 or '', rule.pid2, rule.count, rule.probability, rule.context))
+        rowcount += 1
+        if rowcount % 1000 == 0:
+            print "rows {0}".format(rowcount)

propertysuggester/analyzer/RuleGenerator.py

@@ -0,0 +1,25 @@
+from collections import defaultdict
+import itertools
+from propertysuggester.analyzer.impl.MainAnalyzer import ItemAnalyzer
+from propertysuggester.analyzer.impl.QualifierReferenceAnalyzer import QualifierAnalyzer, ReferenceAnalyzer
+from propertysuggester.analyzer.rule import Rule
+from propertysuggester.utils.datamodel import Entity
+
+
+def compute_rules(entities, min_probability=0.01):
+    """
+    @type entities: collections.Iterable[Entity]
+    @return: list[Rule]
+    """
+
+    analyzers = [ItemAnalyzer(), QualifierAnalyzer(), ReferenceAnalyzer()]
+
+    for i, entity in enumerate(entities):
+        if i % 100000 == 0 and i > 0:
+            print "entities {0}".format(i)
+        for analyzer in analyzers:
+            analyzer.process(entity)
+
+    rules = filter(lambda rule: rule.probability > min_probability, itertools.chain(*(a.get_rules() for a in analyzers)))
+    return rules
+

propertysuggester/analyzer/TableGenerator.py

@@ -0,0 +1,19 @@
+from collections import defaultdict
+from propertysuggester.analyzer.rule import Rule
+from propertysuggester.utils.datamodel import Entity
+
+class Analyzer:
+    def __init__(self):
+        pass
+
+    def process(self, entity):
+        """
+        @type entity: Entity
+        """
+        raise NotImplementedError("Please implement this method")
+
+    def get_rules(self):
+        """
+        @return: list[Rule]
+        """
+        raise NotImplementedError("Please implement this method")

propertysuggester/analyzer/impl/Analyzer.py

@@ -0,0 +1,31 @@
+from collections import defaultdict
+from propertysuggester.analyzer.impl.Analyzer import Analyzer
+from propertysuggester.analyzer.rule import Rule
+
+
+class ItemAnalyzer(Analyzer):
+    def __init__(self):
+        Analyzer.__init__(self)
+        self.property_occurances = defaultdict(int)
+        self.pair_occurances = defaultdict(lambda: defaultdict(int))
+    
+    def process(self, entity):
+        distinct_ids = set(claim.mainsnak.property_id for claim in entity.claims)
+        self._count_occurances(distinct_ids)
+
+    def _count_occurances(self, distinct_ids):
+        for pid1 in distinct_ids:
+            self.property_occurances[pid1] += 1
+            for pid2 in distinct_ids:
+                if pid1 != pid2:
+                    self.pair_occurances[pid1][pid2] += 1
+
+    def get_rules(self):
+        rules = []
+        for pid1, row in self.pair_occurances.iteritems():
+            pid1count = self.property_occurances[pid1]
+            for pid2, paircount in row.iteritems():
+                if paircount > 0:
+                    probability = (paircount/float(pid1count))
+                    rules.append(Rule(pid1, None, pid2, paircount, probability, "item"))
+        return rules

propertysuggester/analyzer/impl/MainAnalyzer.py

@@ -0,0 +1,45 @@
+from collections import defaultdict
+from propertysuggester.analyzer.impl.Analyzer import Analyzer
+from propertysuggester.analyzer.rule import Rule
+
+
+class QualifierAnalyzer(Analyzer):
+    def __init__(self):
+        Analyzer.__init__(self)
+        self.main_occurances = defaultdict(int)
+        self.qualifier_occurances = defaultdict(lambda: defaultdict(int))
+        self.context = "qualifier"
+
+    def process(self, entity):
+        for claim in entity.claims:
+            distinct_pids = set(q.property_id for q in self.get_special(claim))
+            if len(distinct_pids) > 0:
+                main_pid = claim.mainsnak.property_id
+                self.main_occurances[main_pid] += 1
+                self._count_special_appearances(main_pid, distinct_pids)
+
+    def _count_special_appearances(self, mainsnak_id, distinct_ids):
+        for pid in distinct_ids:
+            self.qualifier_occurances[mainsnak_id][pid] += 1
+
+    def get_special(self, claim):
+        return claim.qualifiers
+
+    def get_rules(self):
+        rules = []
+        for main_pid, row in self.qualifier_occurances.iteritems():
+            maincount = self.main_occurances[main_pid]
+            for qualifier_pid, paircount in row.iteritems():
+                if paircount > 0:
+                    probability = paircount/float(maincount)
+                    rules.append(Rule(main_pid, None, qualifier_pid, paircount, probability, self.context))
+        return rules
+
+
+class ReferenceAnalyzer(QualifierAnalyzer):
+    def __init__(self):
+        QualifierAnalyzer.__init__(self)
+        self.context = "reference"
+
+    def get_special(self, claim):
+        return claim.references

propertysuggester/analyzer/impl/QualifierReferenceAnalyzer.py

@@ -0,0 +1 @@
+__author__ = 'Christian'

propertysuggester/analyzer/impl/__init__.py

@@ -0,0 +1,24 @@
+
+class Rule:
+
+    def __init__(self, pid1, qid1, pid2, count, probability, context):
+        """
+        @type pid1: int
+        @type qid1: int|None
+        @type pid2: int
+        @type count: int
+        @type probability: float
+        @type context: string
+        """
+        self.pid1 = pid1
+        self.qid1 = qid1
+        self.pid2 = pid2
+        self.count = count
+        self.probability = probability
+        self.context = context
+
+    def __eq__(self, other):
+        return isinstance(other, Rule) and self.__dict__ == other.__dict__
+
+    def __str__(self):
+        return str(self.__dict__)

propertysuggester/analyzer/rule.py

@@ -9,7 +9,7 @@
 """
 import csv
 
-from propertysuggester.utils.datamodel import Claim, Entity
+from propertysuggester.utils.datamodel import Claim, Entity, Snak
 
 
 def read_csv(input_file, delimiter=","):
@@ -19,21 +19,30 @@ def read_csv(input_file, delimiter=","):
     @type delimiter: str
     """
     current_title = None
+    current_claim = None
     claims = []
     csv_reader = csv.reader(input_file, delimiter=delimiter, quoting=csv.QUOTE_MINIMAL)
 
-    for row_count, row in enumerate(csv_reader):
-        if len(row) != 4:
-            raise ValueError("Error in line {0}: {1}".format(row_count, row))
-        title, prop, datatype, value = row
+    for row in csv_reader:
+        if len(row) != 5:
+            print "error: {0}".format(row)
+        title, typ, property_id, datatype, value = row
         if current_title != title:
             if current_title is not None:
                 yield Entity(current_title, claims)
             current_title = title
             claims = []
-        claims.append(Claim(int(prop), datatype, value))
+        snak = Snak(int(property_id), datatype, value)
+        if typ == 'claim':
+            current_claim = Claim(snak)
+            claims.append(current_claim)
+        elif typ == 'reference':
+            current_claim.references.append(snak)
+        elif typ == 'qualifier':
+            current_claim.qualifiers.append(snak)
+        else:
+            print "unknown type: {0}".format(typ)
 
     if not current_title is None:
         yield Entity(current_title, claims)
 
-    return