Merge branch 'master' of https://github.com/Wikidata-lib/PropertySuggester-Python into ConsiderClassifyingProperties

Conflicts:
	propertysuggester/parser/JsonReader.py
	propertysuggester/test/parser/test_json_reader.py

Author: Dacry

propertysuggester/parser/JsonReader.py

@@ -15,52 +15,59 @@
     print "ujson not found"
     import json as json
 
+
 def read_json(input_file):
+    """
+    @rtype : collections.Iterable[Entity]
+    @type input_file:  file or GzipFile or StringIO.StringIO
+    """
     count = 0
     for jsonline in input_file:
-        count += 1 
+        count += 1
         if count % 3000 == 0:
             print "processed %.2fMB" % (input_file.tell() / 1024.0 ** 2)
-        jsonline = jsonline[:-2]
-        try:
+
+        if jsonline[0] == "{":
+            jsonline = jsonline.rstrip(",\r\n")
             data = json.loads(jsonline)
-        except ValueError:
-            continue
-        if data["type"] == "item":
-            yield _process_json(data)
+            if data["type"] == "item":
+                yield _process_json(data)
+
 
 def _process_json(data):
     title = data["id"]
     if not "claims" in data:
         return Entity(title, [])
     claims = []
-    for prop, statements in data["claims"].iteritems():
+    for property_id, statements in data["claims"].iteritems():
         for statement in statements:
             references = []
             if "references" in statement:
-                for prop, snaks in statement["references"][0]["snaks"].iteritems():
-                    for snak in snaks:
-                        ref = _parse_json_snak(snak)
-                        if ref:
-                            references.append(ref)
+                for reference in statement["references"]:  # TODO: group reference snaks correctly
+                    for ref_id, snaks in reference["snaks"].iteritems():
+                        for snak in snaks:
+                            ref = _parse_json_snak(snak)
+                            if ref:
+                                references.append(ref)
             qualifiers = []
-            if "qualifiers" in statement:                            
-                for prop, snaks in statement["qualifiers"].iteritems():
-                    for snak in snaks: 
+            if "qualifiers" in statement:
+                for qual_id, snaks in statement["qualifiers"].iteritems():
+                    for snak in snaks:
                         qualifier = _parse_json_snak(snak)
-                    if qualifier:
-                        qualifiers.append(qualifier)
+                        if qualifier:
+                            qualifiers.append(qualifier)
             claim = _parse_json_snak(statement["mainsnak"])
             if claim:
                 claims.append(Claim(claim, qualifiers, references))
 
     return Entity(title, claims)
 
+
 def _parse_json_snak(claim_json):
     if claim_json["snaktype"] == "value":
         datatype = claim_json["datatype"]
         datavalue = claim_json["datavalue"]["value"]
-        if datatype == "string":
+        if datatype in ("string", "commonsMedia", "url"):
             value = datavalue
         elif datatype == "wikibase-item":
             if datavalue["entity-type"] == "item":
@@ -72,13 +79,15 @@ def _parse_json_snak(claim_json):
         elif datatype == "quantity":
             value = datavalue["amount"]
         elif datatype == "globe-coordinate":
-            value = "N{0}, E{1}".format(datavalue["latitude"], datavalue["longitude"])
+            value = "N{0[latitude]}, E{0[longitude]}".format(datavalue)
+        elif datatype == "monolingualtext":
+            value = u"{0[text]} ({0[language]})".format(datavalue)
         elif datatype == "bad":
             # for example in Q2241
             return None
         else:
-            #print "WARNING unknown wikidata datatype: %s" % datatype
-            value = "irrelevant"
+            print "WARNING unknown wikidata datatype: %s" % datatype
+            return None
     else:  # novalue, somevalue, ...
         datatype = "unknown"
         value = claim_json["snaktype"]

propertysuggester/test/parser/test_csv_reader.py

@@ -28,7 +28,7 @@ def test_universe(self):
     def test_multiple_entities(self):
         out = StringIO()
         out.writelines(["Q1,claim,373,string,Universe\n",
-                        "Q2,claim,143,wikibase-entityid,Q328\n"])
+                        "Q2,claim,143,wikibase-item,Q328\n"])
         out.seek(0)
         result = list(CsvReader.read_csv(out))
 

propertysuggester/test/parser/test_json_reader.py

@@ -16,7 +16,7 @@ def test_updated_dump(self):
         with gzip.open(resource_filename(__name__, "Wikidata-Q15511.json.gz"), "r") as f:
             result = list(JsonReader.read_json(f))
 
-        self.assertThat(len(result), Equals(1))
+        self.assertThat(result, HasLength(1))
         q15511 = result[0]
         self.assertThat(q15511.title, Equals("Q15511"))
         self.assertThat(q15511.claims, Contains(Claim(Snak(1082, "quantity", "+25"), [Snak(585, "time", "+00000002001-01-01T00:00:00Z"), Snak(459, "wikibase-item", "Q745221")], [Snak(248, "wikibase-item", "Q17597573")])))