Adding a hack to handle a ZPar bug

- ZPar has a bug where it produces non-deterministic output for sentences that contain a single word in all caps. This hack title-cases such words to make the output deterministic. This will be removed once the underlying bug in ZPar is fixed which is under progress.

Author: desilinguist

zpar/DepParser.py

@@ -5,7 +5,13 @@
 '''
 
 import ctypes as c
+import logging
 import os
+import re
+
+# set up the logging
+logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.INFO)
+
 
 class DepParser(object):
     """The ZPar English Dependency Parser"""
@@ -16,6 +22,9 @@ def __init__(self, modelpath, libptr, zpar_session_obj):
         # save the zpar session object
         self._zpar_session_obj = zpar_session_obj
 
+        # set up a logger
+        self.logger = logging.getLogger(__name__)
+
         # get the library method that loads the parser models
         self._load_depparser = libptr.load_depparser
         self._load_depparser.restype = c.c_int
@@ -46,11 +55,25 @@ def dep_parse_sentence(self, sentence, tokenize=True):
             # return empty string if the input is empty
             ans = ""
         else:
-            zpar_compatible_sentence = sentence.strip() + "\n "
+            zpar_compatible_sentence = sentence
+            all_caps_word = ''
+            # detect if we are processing a sentence with a single word in all caps
+            # because that is a known bug. This is a hack for now and will be removed
+            # once the underlying bug is fixed in ZPar.
+            m = re.match(r'^([A-Z]+)$', zpar_compatible_sentence.strip())
+            if m:
+                all_caps_word = m.group(1)
+                fixed_word = all_caps_word.title()
+                self.logger.warning('Encountered sentence with all caps single word '
+                                    'which triggers a known bug in ZPar. Title-casing '
+                                    'to avoid buggy behavior.')
+                zpar_compatible_sentence = sentence.title()
+            zpar_compatible_sentence = zpar_compatible_sentence.strip() + "\n "
             zpar_compatible_sentence = zpar_compatible_sentence.encode('utf-8')
             parsed_sent = self._dep_parse_sentence(self._zpar_session_obj, zpar_compatible_sentence, tokenize)
-            ans = parsed_sent.decode('utf-8')
-
+            # replace the title-cased word with the original all-caps word if we need to
+            parsed_sent = parsed_sent.decode('utf-8')
+            ans = parsed_sent if not all_caps_word else parsed_sent.replace(fixed_word, all_caps_word)
         return ans
 
     def dep_parse_file(self, inputfile, outputfile, tokenize=True):

zpar/Parser.py

@@ -5,7 +5,12 @@
 '''
 
 import ctypes as c
+import logging
 import os
+import re
+
+# set up the logging
+logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.INFO)
 
 class Parser(object):
     """The ZPar English Constituency Parser"""
@@ -16,6 +21,9 @@ def __init__(self, modelpath, libptr, zpar_session_obj):
         # save the zpar session object
         self._zpar_session_obj = zpar_session_obj
 
+        # set up a logger
+        self.logger = logging.getLogger(__name__)
+
         # get the library method that loads the parser models
         self._load_parser = libptr.load_parser
         self._load_parser.restype = c.c_int
@@ -46,10 +54,26 @@ def parse_sentence(self, sentence, tokenize=True):
             # return empty string if the input is empty
             ans = ""
         else:
-            zpar_compatible_sentence = sentence.strip() + "\n "
+            zpar_compatible_sentence = sentence
+            all_caps_word = ''
+            # detect if we are processing a sentence with a single word in all caps
+            # because that is a known bug. This is a hack for now and will be removed
+            # once the underlying bug is fixed in ZPar.
+            m = re.match(r'^([A-Z]+)$', zpar_compatible_sentence.strip())
+            if m:
+                all_caps_word = m.group(1)
+                fixed_word = all_caps_word.title()
+                self.logger.warning('Encountered sentence with all caps single word '
+                                    'which triggers a known bug in ZPar. Title-casing '
+                                    'to avoid buggy behavior.')
+                zpar_compatible_sentence = sentence.title()
+            zpar_compatible_sentence = zpar_compatible_sentence.strip() + "\n "
             zpar_compatible_sentence = zpar_compatible_sentence.encode('utf-8')
             parsed_sent = self._parse_sentence(self._zpar_session_obj, zpar_compatible_sentence, tokenize)
-            ans = parsed_sent.decode('utf-8')
+            # replace the title-cased word with the original all-caps word if we need to
+            parsed_sent = parsed_sent.decode('utf-8')
+            ans = parsed_sent if not all_caps_word else parsed_sent.replace(fixed_word, all_caps_word)
+
         return ans
 
     def parse_file(self, inputfile, outputfile, tokenize=True):

zpar/Tagger.py

@@ -4,7 +4,13 @@
 :organization: ETS
 '''
 import ctypes as c
+import logging
 import os
+import re
+
+# set up the logging
+logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.INFO)
+
 
 class Tagger(object):
     """The ZPar English POS Tagger"""
@@ -15,6 +21,9 @@ def __init__(self, modelpath, libptr, zpar_session_obj):
         # save the zpar session object
         self._zpar_session_obj = zpar_session_obj
 
+        # set up a logger
+        self.logger = logging.getLogger(__name__)
+
         # get the library method that loads the tagger models
         self._load_tagger = libptr.load_tagger
         self._load_tagger.restype = c.c_int
@@ -37,10 +46,25 @@ def tag_sentence(self, sentence, tokenize=True):
             # return empty string if the input is empty
             ans = ""
         else:
-            zpar_compatible_sentence = sentence.strip() + "\n "
+            zpar_compatible_sentence = sentence
+            all_caps_word = ''
+            # detect if we are processing a sentence with a single word in all caps
+            # because that is a known bug. This is a hack for now and will be removed
+            # once the underlying bug is fixed in ZPar.
+            m = re.match(r'^([A-Z]+)$', zpar_compatible_sentence.strip())
+            if m:
+                all_caps_word = m.group(1)
+                fixed_word = all_caps_word.title()
+                self.logger.warning('Encountered sentence with all caps single word '
+                                    'which triggers a known bug in ZPar. Title-casing '
+                                    'to avoid buggy behavior.')
+                zpar_compatible_sentence = sentence.title()
+            zpar_compatible_sentence = zpar_compatible_sentence.strip() + "\n "
             zpar_compatible_sentence = zpar_compatible_sentence.encode('utf-8')
             tagged_sent = self._tag_sentence(self._zpar_session_obj, zpar_compatible_sentence, tokenize)
-            ans = tagged_sent.decode('utf-8')
+            # replace the title-cased word with the original all-caps word if we need to
+            tagged_sent = tagged_sent.decode('utf-8')
+            ans = tagged_sent if not all_caps_word else tagged_sent.replace(fixed_word, all_caps_word)
 
         return ans