refactor analyzer

Author: xchrdw

analyzer.py

@@ -2,7 +2,7 @@
 import sys
 import time
 
-from propertysuggester.analyzer import CsvGenerator, TableGenerator
+from propertysuggester.analyzer import CsvGenerator, RuleGenerator
 from propertysuggester.parser import CsvReader
 from propertysuggester.utils.CompressedFileType import CompressedFileType
 
@@ -16,8 +16,8 @@
 
     start = time.time()
     print "computing table"
-    t, q, r = TableGenerator.compute_table(CsvReader.read_csv(args.input))
+    rules = RuleGenerator.compute_rules(CsvReader.read_csv(args.input))
     print "writing csv"
-    CsvGenerator.create_pair_csv(t, q, r, args.output)
+    CsvGenerator.create_pair_csv(rules, args.output)
     print "done - {0:.2f}s".format(time.time()-start)
     print "now import this csv file with PropertySuggester/maintenance/UpdateTable.php"

propertysuggester/analyzer/CsvGenerator.py

@@ -1,30 +1,19 @@
 import csv
+from propertysuggester.analyzer.rule import Rule
 
 
-def create_pair_csv(table, table_qualifier, table_references, out, delimiter=","):
+def create_pair_csv(rules, out, delimiter=","):
     """
-    @type table: dict[int, dict]
+    @type rules: list[Rule]
     @type out: file or StringIO.StringIO
     @type delimiter: string
     """
     csv_writer = csv.writer(out, delimiter=delimiter, quoting=csv.QUOTE_MINIMAL, lineterminator='\n')
-    print "properties: {0}".format(len(table))
 
     csv_writer.writerow(("pid1", "qid1", "pid2", "count", "probability", "context"))
-
-    _write_entries(table, csv_writer, "item")
-    _write_entries(table_qualifier, csv_writer, "qualifier")
-    _write_entries(table_references, csv_writer, "reference")
-
-
-def _write_entries(table, csv_writer, context):
-    print "Writing entries with context " + context
     rowcount = 0
-    for pid1, row in table.iteritems():
-        for pid2, value in row.iteritems():
-            if pid1 != pid2 and isinstance(pid2, int) and value > 0:  # "appearances" is in the same table, ignore them
-                probability = value/float(row["appearances"])
-                csv_writer.writerow((pid1, '', pid2, value, probability, context))
-                rowcount += 1
-                if rowcount % 1000 == 0:
-                    print "rows {0}".format(rowcount)
+    for rule in rules:
+        csv_writer.writerow((rule.pid1, rule.qid1 or '', rule.pid2, rule.count, rule.probability, rule.context))
+        rowcount += 1
+        if rowcount % 1000 == 0:
+            print "rows {0}".format(rowcount)

propertysuggester/analyzer/RuleGenerator.py

@@ -0,0 +1,94 @@
+from collections import defaultdict
+import itertools
+from propertysuggester.analyzer.rule import Rule
+from propertysuggester.utils.datamodel import Entity
+
+
+def compute_rules(entities):
+    """
+    @type entities: collections.Iterable[Entity]
+    @return: list[Rule]
+    """
+
+    analyzers = [ItemAnalyzer(), QualifierAnalyzer(), ReferenceAnalyzer()]
+
+    for i, entity in enumerate(entities):
+        if i % 100000 == 0 and i > 0:
+            print "entities {0}".format(i)
+        for analyzer in analyzers:
+            analyzer.process(entity)
+
+    rules = itertools.chain(*(a.get_rules() for a in analyzers))
+    return rules
+
+
+class Analyzer:
+    def __init__(self, context):
+        """
+        @type context: string
+        """
+        self.propertyOccurances = defaultdict(int)
+        self.coOccurances = defaultdict(lambda: defaultdict(int))
+        self.context = context
+
+    def process(self, entity):
+        """
+        @type entity: Entity
+        """
+        raise NotImplemented("Please implement this method")
+
+    def get_rules(self):
+        """
+        @return: list[Rule]
+        """
+        rules = []
+        for pid1, row in self.coOccurances.iteritems():
+            pid1count = self.propertyOccurances[pid1]
+            for pid2, value in row.iteritems():
+                if value > 0:
+                    probability = value/float(pid1count)
+                    rules.append(Rule(pid1, None, pid2, pid1count, probability, self.context))
+        return rules
+
+
+class ItemAnalyzer(Analyzer):
+    def __init__(self):
+        Analyzer.__init__(self, "item")
+
+    def process(self, entity):
+        distinct_ids = set(claim.mainsnak.property_id for claim in entity.claims)
+        self._count_occurances(distinct_ids)
+
+    def _count_occurances(self, distinct_ids):
+        for pid1 in distinct_ids:
+            self.propertyOccurances[pid1] += 1
+            for pid2 in distinct_ids:
+                if pid1 != pid2:
+                    self.coOccurances[pid1][pid2] += 1
+
+
+class QualifierAnalyzer(Analyzer):
+    def __init__(self):
+        Analyzer.__init__(self, "qualifier")
+
+    def process(self, entity):
+        for claim in entity.claims:
+            distinct_pids = set(q.property_id for q in self.get_special(claim))
+            if len(distinct_pids) > 0:
+                self.propertyOccurances[claim.mainsnak.property_id] += 1
+                self._count_special_appearances(claim.mainsnak.property_id, distinct_pids)
+
+    def _count_special_appearances(self, mainsnak_id, distinct_ids):
+        for pid in distinct_ids:
+            self.coOccurances[mainsnak_id][pid] += 1
+
+    def get_special(self, claim):
+        return claim.qualifiers
+
+
+class ReferenceAnalyzer(QualifierAnalyzer):
+    def __init__(self):
+        Analyzer.__init__(self, "reference")
+
+    def get_special(self, claim):
+        return claim.references

propertysuggester/analyzer/TableGenerator.py

@@ -0,0 +1,24 @@
+
+class Rule:
+
+    def __init__(self, pid1, qid1, pid2, count, probability, context):
+        """
+        @type pid1: int
+        @type qid1: int|None
+        @type pid2: int
+        @type count: int
+        @type probability: float
+        @type context: string
+        """
+        self.pid1 = pid1
+        self.qid1 = qid1
+        self.pid2 = pid2
+        self.count = count
+        self.probability = probability
+        self.context = context
+
+    def __eq__(self, other):
+        return isinstance(other, Rule) and self.__dict__ == other.__dict__
+
+    def __str__(self):
+        return str(self.__dict__)

propertysuggester/analyzer/rule.py

@@ -28,7 +28,7 @@ def read_csv(input_file, delimiter=","):
             print "error: {0}".format(row)
         title, typ, property_id, datatype, value = row
         if current_title != title:
-            if not current_title is None:
+            if current_title is not None:
                 yield Entity(current_title, claims)
             current_title = title
             claims = []
@@ -39,7 +39,7 @@ def read_csv(input_file, delimiter=","):
         elif typ == 'reference':
             current_claim.references.append(snak)
         elif typ == 'qualifier':
-            current_claim.qualifier.append(snak)
+            current_claim.qualifiers.append(snak)
         else:
             print "unknown type: {0}".format(typ)
 

propertysuggester/parser/CsvReader.py

@@ -15,7 +15,7 @@ def write_csv(entities, output_file, delimiter=","):
         for claim in entity.claims:
             title = entity.title.encode("utf-8")
             write_row(csv_writer, title, "claim", claim.mainsnak)
-            for q in claim.qualifier:
+            for q in claim.qualifiers:
                 write_row(csv_writer, title, "qualifier", q)
             for ref in claim.references:
                 write_row(csv_writer, title, "reference", ref)

propertysuggester/parser/CsvWriter.py

@@ -5,6 +5,7 @@
 from testtools.matchers import Equals
 
 from propertysuggester.analyzer import CsvGenerator
+from propertysuggester.analyzer.rule import Rule
 
 
 class CsvGeneratorTest(TestCase):
@@ -13,13 +14,12 @@ def setUp(self):
         self.file = StringIO()
 
     def test_create_table(self):
-        table = {1: {'appearances': 8, 'type': 'string', 2: 5, 3: 0}}
-        CsvGenerator.create_pair_csv(table, {}, {}, self.file)
+        rule = Rule(1, None, 2, 5, 0.3, "item")
+        CsvGenerator.create_pair_csv([rule], self.file)
 
         self.file.seek(0)
         self.assertThat(self.file.readline().strip(), Equals("pid1,qid1,pid2,count,probability,context"))
-        prob = 5.0 / 8.0
-        self.assertThat(self.file.readline().strip(), Equals("1,,2,5,{0},item".format(prob)))
+        self.assertThat(self.file.readline().strip(), Equals("1,,2,5,0.3,item"))
 
 
 if __name__ == '__main__':

propertysuggester/test/analyzer/test_csv_generator.py

@@ -0,0 +1,46 @@
+import unittest
+
+from testtools import TestCase
+from testtools.matchers import *
+
+from propertysuggester.analyzer import RuleGenerator
+from propertysuggester.analyzer.rule import Rule
+from propertysuggester.utils.datamodel import Entity, Claim, Snak
+
+
+test_data1 = [Entity('Q15', [Claim(Snak(31, 'wikibase-entityid', 'Q5107')),
+                             Claim(Snak(373, 'string', 'Africa'))]),
+              Entity('Q16', [Claim(Snak(31, 'wikibase-entityid', 'Q384'))])]
+
+test_data2 = [Entity('Q15', [Claim(Snak(31, 'wikibase-entityid', 'Q5107')),
+                             Claim(Snak(373, 'string', 'Africa')),
+                             Claim(Snak(373, 'string', 'Europe'))])]
+
+test_data3 = [Entity('Q15', [Claim(Snak(31, 'wikibase-entityid', 'Q5107'),
+                                   [Snak(12, 'wikibase-entityid', 'Q123'), Snak(13, 'string', 'qual')],
+                                   [Snak(22, 'wikibase-entityid', 'Q345'), Snak(23, 'string', 'rel')])])]
+
+
+class RuleGeneratorTest(TestCase):
+    def setUp(self):
+        TestCase.setUp(self)
+
+    def test_table_generator(self):
+        rules = list(RuleGenerator.compute_rules(test_data1))
+        self.assertThat(rules, ContainsAll([Rule(31, None, 373, 2, 0.5, "item"), Rule(373, None, 31, 1, 1.0, "item")]))
+
+
+    def test_table_with_multiple_occurance(self):
+        rules = list(RuleGenerator.compute_rules(test_data2))
+        self.assertThat(rules, ContainsAll([Rule(31, None, 373, 1, 1.0, "item"), Rule(373, None, 31, 1, 1.0, "item")]))
+
+    def test_table_with_qualifier_and_references(self):
+        rules = list(RuleGenerator.compute_rules(test_data3))
+        self.assertThat(rules, ContainsAll([Rule(31, None, 12, 1, 1.0, "qualifier"),
+                                            Rule(31, None, 13, 1, 1.0, "qualifier"),
+                                            Rule(31, None, 22, 1, 1.0, "reference"),
+                                            Rule(31, None, 23, 1, 1.0, "reference")]))
+
+
+if __name__ == '__main__':
+    unittest.main()