Merge pull request #110 from LuminosoInsight/latin1-inconsistent-mojibake

Make the partial mojibake fixer actually work on Latin-1

Author: Lance Nathan

CHANGELOG.md

@@ -1,3 +1,21 @@
+## Version 5.5 (September 6, 2018)
+
+- Recent versions have emphasized making a reasonable attempt to fix short,
+  common mojibake sequences, such as `û`. In this version, we've expanded the
+  heuristics to recognize these sequences in MacRoman as well as Windows-125x
+  encodings.
+
+- A related rule for fixing isolated Windows-1252/UTF-8 mixups, even when they
+  were inconsistent with the rest of the string, claimed to work on Latin-1/UTF-8
+  mixups as well, but in practice it didn't. We've made the rule more robust.
+
+- Fixed a failure when testing the CLI on Windows.
+
+- Removed the `pytest-runner` invocation from setup.py, as it created complex
+  dependencies that would stop setup.py from working in some environments.
+  The `pytest` command still works fine. `pytest-runner` is just too clever.
+
+
 ## Version 5.4.1 (June 14, 2018)
 
 - Fixed a bug in the `setup.py` metadata.
@@ -19,6 +37,8 @@
 
 - Provides better metadata for the new PyPI.
 
+- Switched from nosetests to pytest.
+
 
 ## Version 5.3 (January 25, 2018)
 

ftfy/__init__.py

@@ -10,7 +10,7 @@
 from ftfy import fixes
 from ftfy.formatting import display_ljust
 
-__version__ = '5.4.1'
+__version__ = '5.5.0'
 
 
 # See the docstring for ftfy.bad_codecs to see what we're doing here.

ftfy/chardata.py

@@ -60,12 +60,12 @@ def _build_utf8_punct_regex():
     These are recognizable by the distinctive 'â€' ('\xe2\x80') sequence they
     all begin with when decoded as Windows-1252.
     """
-    # We're making a regex that has all the literal bytes from 0x80 to 0xbf in
-    # a range. "Couldn't this have just said [\x80-\xbf]?", you might ask.
-    # However, when we decode the regex as Windows-1252, the resulting
-    # characters won't even be remotely contiguous.
-    obvious_utf8 = ('â€['
-                    + bytes(range(0x80, 0xc0)).decode('sloppy-windows-1252')
+    # We need to recognize the Latin-1 decodings of bytes 0x80 to 0xbf, which
+    # are a contiguous range, as well as the different Windows-1252 decodings
+    # of 0x80 to 0x9f, which are not contiguous at all. (Latin-1 and
+    # Windows-1252 agree on bytes 0xa0 and up.)
+    obvious_utf8 = ('â[€\x80][\x80-\xbf'
+                    + bytes(range(0x80, 0xa0)).decode('sloppy-windows-1252')
                     + ']')
     return re.compile(obvious_utf8)
 PARTIAL_UTF8_PUNCT_RE = _build_utf8_punct_regex()
@@ -120,6 +120,10 @@ def _build_utf8_punct_regex():
 SINGLE_QUOTE_RE = re.compile('[\u02bc\u2018-\u201b]')
 DOUBLE_QUOTE_RE = re.compile('[\u201c-\u201f]')
 
+# This regex matches C1 control characters, which occupy some of the positions
+# in the Latin-1 character map that Windows assigns to other characters instead.
+C1_CONTROL_RE = re.compile(r'[\x80-\x9f]')
+
 
 def possible_encoding(text, encoding):
     """

ftfy/fixes.py

@@ -9,7 +9,8 @@
 from ftfy.chardata import (possible_encoding, CHARMAP_ENCODINGS,
                            CONTROL_CHARS, LIGATURES, WIDTH_MAP,
                            PARTIAL_UTF8_PUNCT_RE, ALTERED_UTF8_RE,
-                           LOSSY_UTF8_RE, SINGLE_QUOTE_RE, DOUBLE_QUOTE_RE)
+                           LOSSY_UTF8_RE, SINGLE_QUOTE_RE, DOUBLE_QUOTE_RE,
+                           C1_CONTROL_RE)
 from ftfy.badness import text_cost
 from html import entities
 
@@ -633,7 +634,7 @@ def replace_lossy_sequences(byts):
     not be used, and this function will not be run, so your weird control
     character will be left alone but wacky fixes like this won't be possible.
 
-    This is used as a step within `fix_encoding`.
+    This is used as a transcoder within `fix_encoding`.
     """
     return LOSSY_UTF8_RE.sub('\ufffd'.encode('utf-8'), byts)
 
@@ -644,16 +645,23 @@ def fix_partial_utf8_punct_in_1252(text):
     UTF-8 and decoded in Latin-1 or Windows-1252, even when this fix can't be
     consistently applied.
 
-    For this function, we assume the text has been decoded in Windows-1252.
-    If it was decoded in Latin-1, we'll call this right after it goes through
-    the Latin-1-to-Windows-1252 fixer.
+    One form of inconsistency we need to deal with is that some character might
+    be from the Latin-1 C1 control character set, while others are from the
+    set of characters that take their place in Windows-1252. So we first replace
+    those characters, then apply a fix that only works on Windows-1252 characters.
 
-    This is used as a step within `fix_encoding`.
+    This is used as a transcoder within `fix_encoding`.
     """
-    def replacement(match):
+    def latin1_to_w1252(match):
+        "The function to apply when this regex matches."
+        return match.group(0).encode('latin-1').decode('sloppy-windows-1252')
+
+    def w1252_to_utf8(match):
         "The function to apply when this regex matches."
         return match.group(0).encode('sloppy-windows-1252').decode('utf-8')
-    return PARTIAL_UTF8_PUNCT_RE.sub(replacement, text)
+    
+    text = C1_CONTROL_RE.sub(latin1_to_w1252, text)
+    return PARTIAL_UTF8_PUNCT_RE.sub(w1252_to_utf8, text)
 
 
 TRANSCODERS = {

setup.py

@@ -26,7 +26,7 @@
 
 setup(
     name="ftfy",
-    version='5.4.1',
+    version='5.5.0',
     maintainer='Luminoso Technologies, Inc.',
     maintainer_email='[email protected]',
     license="MIT",

tests/test_cases.json

@@ -151,6 +151,20 @@
         "fixed": "Arsenal v Wolfsburg: pre-season friendly – live!",
         "expect": "pass"
     },
+    {
+        "label": "Inconsistent UTF-8 / Latin-1 mojibake",
+        "original": "Ecuadorâ\u0080\u0099s â\u0080\u0098purely political decision on Assangeâ\u0080\u0099 is likely result of â\u0080\u0098US pressureâ\u0080\u0099\u0085",
+        "fixed-encoding": "Ecuador’s ‘purely political decision on Assange’ is likely result of ‘US pressure’…",
+        "fixed": "Ecuador's 'purely political decision on Assange' is likely result of 'US pressure'…",
+        "expect": "pass"
+    },
+    {
+        "label": "Inconsistent UTF-8 / Latin-1 mojibake with an ellipsis from the Windows-1252 character set",
+        "original": "Ecuadorâ\u0080\u0099s â\u0080\u0098purely political decision on Assangeâ\u0080\u0099 is likely result of â\u0080\u0098US pressureâ\u0080\u0099…",
+        "fixed-encoding": "Ecuador’s ‘purely political decision on Assange’ is likely result of ‘US pressure’…",
+        "fixed": "Ecuador's 'purely political decision on Assange' is likely result of 'US pressure'…",
+        "expect": "pass"
+    },
     {
         "label": "Handle Afrikaans 'n character",
         "original": "ʼn Chloroplas is ʼn organel wat in fotosinterende plante voorkom.",
@@ -333,12 +347,19 @@
         "expect": "fail"
     },
     {
-        "label": "Windows-1252 / MacRoman mixup in Spanish",
+        "label": "Latin-1 / MacRoman mixup in Spanish",
         "comment": "Requires something like encoding detection",
         "original": "Deja dos heridos hundimiento de barco tur\u0092stico en Acapulco.",
         "fixed": "Deja dos heridos hundimiento de barco turístico en Acapulco.",
         "expect": "fail"
     },
+    {
+        "label": "Latin-1 / MacRoman mixup in Spanish, 2 characters",
+        "comment": "Requires something like encoding detection",
+        "original": "Habitantes de Coatl\u0087n conf\u0092an en proyecto de edil electo independiente",
+        "fixed": "Habitantes de Coatlán confían en proyecto de edil electo independiente",
+        "expect": "fail"
+    },
     {
         "label": "UTF-8 / Windows-1251 mixup in tweet spam",
         "original": "Blog Traffic Tip 2 – Broadcast Email Your Blog",

tests/test_cli.py

@@ -53,7 +53,8 @@ def test_same_file():
     with pytest.raises(subprocess.CalledProcessError) as exception:
         get_command_output(['ftfy', TEST_FILENAME, '-o', TEST_FILENAME])
     error = exception.value.output.decode('utf-8')
-    assert error.startswith("ftfy error:\nCan't read and write the same file.")
+    assert error.startswith("ftfy error:")
+    assert "Can't read and write the same file" in error
 
 
 def test_stdin():