|
| 1 | +"""Wrapper for UDPipe online web service.""" |
| 2 | +import io |
| 3 | +import sys |
| 4 | +import email.mime.multipart |
| 5 | +import email.mime.nonmultipart |
| 6 | +import email.policy |
| 7 | +import json |
| 8 | +import os |
| 9 | +import sys |
| 10 | +import urllib.error |
| 11 | +import urllib.request |
| 12 | + |
| 13 | +from udapi.block.read.conllu import Conllu as ConlluReader |
| 14 | +from udapi.core.root import Root |
| 15 | + |
| 16 | +class UDPipeOnline: |
| 17 | + """Wrapper for UDPipe online web service.""" |
| 18 | + |
| 19 | + def __init__(self, model, server="https://lindat.mff.cuni.cz/services/udpipe/api"): |
| 20 | + """Create the UDPipeOnline tool object.""" |
| 21 | + self.model = model |
| 22 | + self.server = server |
| 23 | + |
| 24 | + def list_models(self): |
| 25 | + with urllib.request.urlopen(self.server + "/models") as request: |
| 26 | + response = json.loads(request.read()) |
| 27 | + return list(response["models"].keys()) |
| 28 | + |
| 29 | + def perform_request(self, params, method="process"): |
| 30 | + if not params: |
| 31 | + request_headers, request_data = {}, None |
| 32 | + else: |
| 33 | + message = email.mime.multipart.MIMEMultipart("form-data", policy=email.policy.HTTP) |
| 34 | + |
| 35 | + for name, value in params.items(): |
| 36 | + payload = email.mime.nonmultipart.MIMENonMultipart("text", "plain") |
| 37 | + payload.add_header("Content-Disposition", "form-data; name=\"{}\"".format(name)) |
| 38 | + payload.add_header("Content-Transfer-Encoding", "8bit") |
| 39 | + payload.set_payload(value, charset="utf-8") |
| 40 | + message.attach(payload) |
| 41 | + |
| 42 | + request_data = message.as_bytes().split(b"\r\n\r\n", maxsplit=1)[1] |
| 43 | + request_headers = {"Content-Type": message["Content-Type"]} |
| 44 | + |
| 45 | + try: |
| 46 | + with urllib.request.urlopen(urllib.request.Request( |
| 47 | + url=f"{self.server}/{method}", headers=request_headers, data=request_data |
| 48 | + )) as request: |
| 49 | + response = json.loads(request.read()) |
| 50 | + except urllib.error.HTTPError as e: |
| 51 | + print("An exception was raised during UDPipe 'process' REST request.\n" |
| 52 | + "The service returned the following error:\n" |
| 53 | + " {}".format(e.fp.read().decode("utf-8")), file=sys.stderr) |
| 54 | + raise |
| 55 | + except json.JSONDecodeError as e: |
| 56 | + print("Cannot parse the JSON response of UDPipe 'process' REST request.\n" |
| 57 | + " {}".format(e.msg), file=sys.stderr) |
| 58 | + raise |
| 59 | + |
| 60 | + if "model" not in response or "result" not in response: |
| 61 | + raise ValueError("Cannot parse the UDPipe 'process' REST request response.") |
| 62 | + |
| 63 | + return response["result"] |
| 64 | + |
| 65 | + def tag_parse_tree(self, root): |
| 66 | + """Tag (+lemmatize, fill FEATS) and parse a tree (already tokenized).""" |
| 67 | + descendants = root.descendants |
| 68 | + if not descendants: |
| 69 | + return |
| 70 | + in_data = " ".join([n.form for n in descendants]) |
| 71 | + out_data = self.perform_request(params={"data": in_data, "input":"horizontal", "tagger":"", "parser":""}) |
| 72 | + conllu_reader = ConlluReader() |
| 73 | + conllu_reader.files.filehandle = io.StringIO(out_data) |
| 74 | + parsed_root = conllu_reader.read_tree() |
| 75 | + root.flatten() |
| 76 | + for parsed_node in parsed_root.descendants: |
| 77 | + node = descendants[parsed_node.ord - 1] |
| 78 | + node.parent = descendants[parsed_node.parent.ord - 1] if parsed_node.parent.ord else root |
| 79 | + for attr in 'upos xpos lemma feats deprel'.split(): |
| 80 | + setattr(node, attr, getattr(parsed_node, attr)) |
| 81 | + |
| 82 | + def tokenize_tag_parse_tree(self, root, resegment=False, tag=True, parse=True): |
| 83 | + """Tokenize, tag (+lemmatize, fill FEATS) and parse the text stored in `root.text`. |
| 84 | +
|
| 85 | + If resegment=True, the returned list of Udapi trees may contain multiple trees. |
| 86 | + """ |
| 87 | + if parse and not tag: |
| 88 | + raise ValueError('Combination parse=True tag=False is not allowed.') |
| 89 | + if root.children: |
| 90 | + raise ValueError('Tree already contained nodes before tokenization') |
| 91 | + |
| 92 | + # Tokenize and possibly segment the input text |
| 93 | + params = {"model": self.model, "data": root.text, "tokenizer":"" if resegment else "presegmented"} |
| 94 | + if tag: |
| 95 | + params["tagger"] = "" |
| 96 | + if parse: |
| 97 | + params["parser"] = "" |
| 98 | + out_data = self.perform_request(params=params) |
| 99 | + conllu_reader = ConlluReader(empty_parent="ignore") |
| 100 | + conllu_reader.files.filehandle = io.StringIO(out_data) |
| 101 | + trees = conllu_reader.read_trees() |
| 102 | + |
| 103 | + # The input "root" object must be the first item in "trees". |
| 104 | + for attr in ('_children', '_descendants', '_mwts', 'text', 'comment'): |
| 105 | + setattr(root, attr, getattr(trees[0], attr)) |
| 106 | + for node in root._children: |
| 107 | + node._parent = root |
| 108 | + for node in root._descendants: |
| 109 | + node._root = root |
| 110 | + trees[0] = root |
| 111 | + return trees |
| 112 | + |
| 113 | + def segment_text(self, text): |
| 114 | + """Segment the provided text into sentences returned as a Python list.""" |
| 115 | + params = {"model": self.model, "data": text, "tokenizer":"", "output": "plaintext=normalized_spaces"} |
| 116 | + return self.perform_request(params=params).rstrip().split("\n") |
0 commit comments