fix only first qualifier was parsed

fix only first reference group was parsed
add url support

Author: xchrdw

propertysuggester/parser/JsonReader.py

@@ -15,56 +15,59 @@
     print "ujson not found"
     import json as json
 
+
 def read_json(input_file):
     """
     @rtype : collections.Iterable[Entity]
     @type input_file:  file or GzipFile or StringIO.StringIO
     """
     count = 0
     for jsonline in input_file:
-        count += 1 
+        count += 1
         if count % 3000 == 0:
             print "processed %.2fMB" % (input_file.tell() / 1024.0 ** 2)
 
-        jsonline = jsonline.rstrip(",\r\n")
-
-        data = json.loads(jsonline)
+        if jsonline[0] == "{":
+            jsonline = jsonline.rstrip(",\r\n")
+            data = json.loads(jsonline)
+            if data["type"] == "item":
+                yield _process_json(data)
 
-        if data["type"] == "item":
-            yield _process_json(data)
 
 def _process_json(data):
     title = data["id"]
     if not "claims" in data:
         return Entity(title, [])
     claims = []
-    for prop, statements in data["claims"].iteritems():
+    for property_id, statements in data["claims"].iteritems():
         for statement in statements:
             references = []
             if "references" in statement:
-                for prop, snaks in statement["references"][0]["snaks"].iteritems():
-                    for snak in snaks:
-                        ref = _parse_json_snak(snak)
-                        if ref:
-                            references.append(ref)
+                for reference in statement["references"]:  # TODO: group reference snaks correctly
+                    for ref_id, snaks in reference["snaks"].iteritems():
+                        for snak in snaks:
+                            ref = _parse_json_snak(snak)
+                            if ref:
+                                references.append(ref)
             qualifiers = []
-            if "qualifiers" in statement:                            
-                for prop, snaks in statement["qualifiers"].iteritems():
-                    for snak in snaks: 
+            if "qualifiers" in statement:
+                for qual_id, snaks in statement["qualifiers"].iteritems():
+                    for snak in snaks:
                         qualifier = _parse_json_snak(snak)
-                    if qualifier:
-                        qualifiers.append(qualifier)
+                        if qualifier:
+                            qualifiers.append(qualifier)
             claim = _parse_json_snak(statement["mainsnak"])
             if claim:
                 claims.append(Claim(claim, qualifiers, references))
 
     return Entity(title, claims)
 
+
 def _parse_json_snak(claim_json):
     if claim_json["snaktype"] == "value":
         datatype = claim_json["datatype"]
         datavalue = claim_json["datavalue"]["value"]
-        if datatype == "string":
+        if datatype in ("string", "commonsMedia", "url"):
             value = datavalue
         elif datatype == "wikibase-item":
             if datavalue["entity-type"] == "item":
@@ -76,9 +79,9 @@ def _parse_json_snak(claim_json):
         elif datatype == "quantity":
             value = datavalue["amount"]
         elif datatype == "globe-coordinate":
-            value = "N{0}, E{1}".format(datavalue["latitude"], datavalue["longitude"])
-        elif datatype == "commonsMedia":
-            value = datavalue
+            value = "N{0[latitude]}, E{0[longitude]}".format(datavalue)
+        elif datatype == "monolingualtext":
+            value = u"{0[text]} ({0[language]})".format(datavalue)
         elif datatype == "bad":
             # for example in Q2241
             return None