adapt analysis to Json-Dumps

Author: Dacry

dumpconverter.py

@@ -2,17 +2,16 @@
 import sys
 import time
 
-from propertysuggester.parser import XmlReader, CsvWriter
+from propertysuggester.parser import JsonReader, CsvWriter
 from propertysuggester.utils.CompressedFileType import CompressedFileType
 
 if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description="this program converts wikidata XML dumps to CSV data.")
-    parser.add_argument("input", help="The XML input file (a wikidata dump)", type=CompressedFileType('r'))
+    parser = argparse.ArgumentParser(description="this program converts wikidata JSON dumps to CSV data.")
+    parser.add_argument("input", help="The JSON input file (a wikidata dump)", type=CompressedFileType('r'))
     parser.add_argument("output", help="The CSV output file (default=sys.stdout)", default=sys.stdout, nargs='?',
                         type=CompressedFileType('wb'))
-    parser.add_argument("-p", "--processes", help="Number of processors to use (default 4)", type=int, default=4)
+    #parser.add_argument("-p", "--processes", help="Number of processors to use (default 4)", type=int, default=4)
     args = parser.parse_args()
-
     start = time.time()
-    CsvWriter.write_csv(XmlReader.read_xml(args.input, args.processes), args.output)
+    CsvWriter.write_csv(newJsonReader.process_json(args.input), args.output)
     print "total time: %.2fs" % (time.time() - start)

propertysuggester/parser/JsonReader.py

@@ -0,0 +1,85 @@
+"""
+process_json returns a generator that yields Entities)
+
+usage:
+with open("file.csv", "r") as f:
+    for entity in process_json(f):
+        do_things()
+
+"""
+from propertysuggester.utils.datamodel import Claim, Entity, Snak
+
+try:
+    import ujson as json
+except ImportError:
+    print "ujson not found"
+    import json as json
+
+def process_json(input_file):
+    count = 0
+    for jsonline in input_file:
+        count += 1 
+        if count % 3000 == 0:
+            print "processed %.2fMB" % (input_file.tell() / 1024.0 ** 2)
+        jsonline = jsonline[:-2]
+        try:
+            data = json.loads(jsonline)
+        except ValueError:
+            continue
+        if data["type"] == "item":
+            title = data["id"]
+            if not "claims" in data:
+                yield Entity(title, [])
+                continue
+            claims = []
+            for prop, statements in data["claims"].iteritems():
+                for statement in statements:
+                    references = []
+                    if "references" in statement:
+                        for prop, snaks in statement["references"][0]["snaks"].iteritems():
+                            for snak in snaks:
+                                ref = _parse_json_snak(snak)
+                                if ref:
+                                    references.append(ref)
+                    qualifiers = []
+                    if "qualifiers" in statement:                            
+                        for prop, snaks in statement["qualifiers"].iteritems():
+                            for snak in snaks: 
+                                qualifier = _parse_json_snak(snak)
+                            if qualifier:
+                                qualifiers.append(qualifier)
+                    claim = _parse_json_snak(statement["mainsnak"])
+                    if claim:
+                        claims.append(Claim(claim, qualifiers, references))
+
+            yield Entity(title, claims)
+
+
+def _parse_json_snak(claim_json):
+    if claim_json["snaktype"] == "value":
+        datatype = claim_json["datatype"]
+        datavalue = claim_json["datavalue"]["value"]
+        if datatype == "string":
+            value = datavalue
+        elif datatype == "wikibase-item":
+            if datavalue["entity-type"] == "item":
+                value = "Q" + str(datavalue["numeric-id"])
+            else:
+                print "WARNING unknown entitytype: {0}".format(datavalue["entity-type"])
+        elif datatype == "time":
+            value = datavalue["time"]
+        elif datatype == "quantity":
+            value = datavalue["amount"]
+        elif datatype == "globe-coordinate":
+            value = "N{0}, E{1}".format(datavalue["latitude"], datavalue["longitude"])
+        elif datatype == "bad":
+            # for example in Q2241
+            return None
+        else:
+            #print "WARNING unknown wikidata datatype: %s" % datatype
+            value = "irrelevant"
+    else:  # novalue, somevalue, ...
+        datatype = "unknown"
+        value = claim_json["snaktype"]
+    property_id = claim_json["property"][1:]
+    return Snak(property_id, datatype, value)

readme.md

@@ -30,5 +30,14 @@ nosetests
 
 ## Release Notes
 
+### 1.2
+* Consider classifying Properties
+* use Json dumps for analysis
+
+### 1.1
+* Generate associationrules for qualifier and references
+* Improve ranking to avoid suggestions of human properties
+* remove very unlikely rules (<1%)
+
 ### 1.0
 * Converts a wikidata dump to a csv file with associationrules between properties