improve xmlreader test

Author: xchrdw

propertysuggester/analyzer/CsvGenerator.py

@@ -26,5 +26,5 @@ def _write_entries(table, csv_writer, context):
                 probability = value/float(row["appearances"])
                 csv_writer.writerow((pid1, '', pid2, value, probability, context))
                 rowcount += 1
-                if not rowcount % 1000:
+                if rowcount % 1000 == 0:
                     print "rows {0}".format(rowcount)

propertysuggester/analyzer/TableGenerator.py

@@ -1,4 +1,3 @@
-
 from collections import defaultdict
 from propertysuggester.utils.datamodel import Entity
 
@@ -49,6 +48,7 @@ def _count_occurances(distinct_ids, table):
             if pid1 != pid2:
                 table[pid1][pid2] += 1
 
+
 def _count_special_appearances(claim, get_special, special_table):
     """
     @type claim: Claim

propertysuggester/parser/XmlReader.py

@@ -89,7 +89,6 @@ def _process_json((title, json_string)):
 
     claims = []
     for statement in data["claims"]:
-        claim_json = statement["m"]
         references = []
         for i in statement["refs"]:
             for a in i:
@@ -98,11 +97,11 @@ def _process_json((title, json_string)):
                     references.append(ref)
         qualifiers = []
         for q in statement["q"]:
-                qualifier = _parse_json_snak(q)
-                if qualifier:
-                    qualifiers.append(qualifier)
+            qualifier = _parse_json_snak(q)
+            if qualifier:
+                qualifiers.append(qualifier)
 
-        claim = _parse_json_snak(claim_json)
+        claim = _parse_json_snak(statement["m"])
         if claim:
             claims.append(Claim(claim, qualifiers, references))
 
@@ -132,4 +131,4 @@ def _parse_json_snak(claim_json):
         datatype = "unknown"
         value = claim_json[0]
     property_id = claim_json[1]
-    return Snak(property_id, datatype, value)
+    return Snak(property_id, datatype, value)

propertysuggester/test/parser/test_abstract_reader.py

@@ -5,9 +5,6 @@
 
 
 class AbstractUniverseTest(TestCase):
-    def setUp(self):
-        TestCase.setUp(self)
-
     def assert_universe(self, result):
         self.assertThat(result, HasLength(1))
         q1 = result[0]

propertysuggester/test/parser/test_csv_reader.py

@@ -1,12 +1,18 @@
 from StringIO import StringIO
 
 from testtools import TestCase
+from testtools.matchers import *
 
 from propertysuggester.parser import CsvReader
 from propertysuggester.test.parser.test_abstract_reader import AbstractUniverseTest
+from propertysuggester.utils.datamodel import Claim, Entity, Snak
 
 
 class CsvReaderTest(AbstractUniverseTest):
+    def setUp(self):
+        AbstractUniverseTest.setUp(self)
+
+
     def test_universe(self):
         out = StringIO()
         out.writelines(["Q1,claim,373,string,Universe\n",
@@ -17,13 +23,31 @@ def test_universe(self):
                         "Q1,qualifier,31,wikibase-entityid,Q41719\n"])
         out.seek(0)
         result = list(CsvReader.read_csv(out))
-
         self.assert_universe(result)
 
-class CsvReaderTest2(TestCase):
+    def test_multiple_entities(self):
+        out = StringIO()
+        out.writelines(["Q1,claim,373,string,Universe\n",
+                        "Q2,claim,143,wikibase-entityid,Q328\n"])
+        out.seek(0)
+        result = list(CsvReader.read_csv(out))
+
+        self.assertThat(result, HasLength(2))
+        self.assertThat(result[0].title, Equals("Q1"))
+        self.assertThat(result[1].title, Equals("Q2"))
+
+    def test_unknown_type(self):
+        out = StringIO()
+        out.writelines(["Q1,unknown,373,string,Universe\n"])
+        out.seek(0)
+        result = list(CsvReader.read_csv(out))
+
     def test_invalid_row_throws_exception(self):
         f = StringIO()
         f.writelines(["a,b"])
         f.seek(0)
-
         self.assertRaises(ValueError, lambda: list(CsvReader.read_csv(f)))
+
+    def test_tostring(self):
+        e = Entity("Q1", [Claim(Snak(2,"string","a"))])
+        str(e)

propertysuggester/test/parser/test_xml_reader.py

@@ -7,7 +7,7 @@
 
 from propertysuggester.test.parser.test_abstract_reader import AbstractUniverseTest
 from propertysuggester.parser import XmlReader
-from propertysuggester.utils.datamodel import Claim, Snak
+from propertysuggester.utils.datamodel import Claim, Snak, Entity
 
 
 class XmlReaderTest(AbstractUniverseTest):
@@ -26,6 +26,12 @@ def test_updated_dump(self):
         self.assertThat(q9351.claims, Contains(Claim(Snak(156, "wikibase-entityid", "Q1647331"))))
         self.assertThat(q9351.claims, Contains(Claim(Snak(1112, "quantity", "+25"))))
 
+    def test_special_cases(self):
+        self.assertThat(XmlReader._process_json(("Q1", "{}")), Equals(Entity("Q1", [])))
+        self.assertThat(XmlReader._process_json(("Q1", '{"claims":[{"m":["value","","bad"], "refs":[],"q":[]}]}')),
+                        Equals(Entity("Q1", [])))
+        self.assertThat(XmlReader._process_json(("Q1", '{"claims":[{"m":["value","","unknown"], "refs":[],"q":[]}]}')),
+                        Equals(Entity("Q1", [])))
 
 class MultiprocessingBigTest(TestCase):
     def test_simple_multiprocessing(self):