Merge pull request #31 from davidread/py3

Python 3 compatibility

Author: Dean Malmgren

.travis.yml

@@ -1,6 +1,7 @@
 language: python
 python:
   - "2.7"
+  - "3.5"
 
 # install python dependencies including this package in the travis
 # virtualenv

requirements/python-dev

@@ -14,5 +14,7 @@ nose
 sphinx
 sphinx_rtd_theme
 
-# for convenience
-ipdb
+# ipdb for convenience.
+# old versions needed for python 2.7 compatibility, despite what the docs say
+ipython==5.4
+ipdb==0.10.2

scrubadub/detectors/skype.py

@@ -44,7 +44,7 @@ def iter_filth(self, text):
         for i in skype_indices:
             jmin = max(i-self.word_radius, 0)
             jmax = min(i+self.word_radius+1, len(tokens))
-            for j in range(jmin, i) + range(i+1, jmax):
+            for j in list(range(jmin, i)) + list(range(i+1, jmax)):
                 token = tokens[j]
                 if self.filth_cls.SKYPE_USERNAME.match(token):
 

scrubadub/filth/__init__.py

@@ -26,7 +26,7 @@ def iter_filths():
     """Iterate over all instances of filth"""
     for filth_cls in iter_filth_clss():
         if issubclass(filth_cls, RegexFilth):
-            m = re.finditer(r"\s+", "fake pattern string").next()
+            m = next(re.finditer(r"\s+", "fake pattern string"))
             yield filth_cls(m)
         else:
             yield filth_cls()

scrubadub/scrubbers.py

@@ -1,12 +1,9 @@
-import re
 import operator
-
-import textblob
-import nltk
+import sys
 
 from . import exceptions
 from . import detectors
-from .filth import Filth, MergedFilth
+from .filth import Filth
 
 
 class Scrubber(object):
@@ -50,8 +47,10 @@ def clean(self, text, **kwargs):
         through to the  ``Filth.replace_with`` method to fine-tune how the
         ``Filth`` is cleaned.
         """
-        if not isinstance(text, unicode):
-            raise exceptions.UnicodeRequired
+        if sys.version_info < (3, 0):
+            # Only in Python 2. In 3 every string is a Python 2 unicode
+            if not isinstance(text, unicode):
+                raise exceptions.UnicodeRequired
 
         clean_chunks = []
         filth = Filth()
@@ -73,14 +72,15 @@ def iter_filth(self, text):
         # over all detectors simultaneously. just trying to get something
         # working right now and we can worry about efficiency later
         all_filths = []
-        for detector in self._detectors.itervalues():
+        for detector in self._detectors.values():
             for filth in detector.iter_filth(text):
                 if not isinstance(filth, Filth):
                     raise TypeError('iter_filth must always yield Filth')
                 all_filths.append(filth)
 
-        # Sort per start position and substrings
-        all_filths = sorted(all_filths, cmp=_sort_filths)
+        # Sort by start position. If two filths start in the same place then
+        # return the longer one first
+        all_filths.sort(key=lambda f: (f.beg, -f.end))
 
         # this is where the Scrubber does its hard work and merges any
         # overlapping filths.
@@ -94,22 +94,3 @@ def iter_filth(self, text):
             else:
                 filth = filth.merge(next_filth)
         yield filth
-
-
-def _sort_filths(a_filth, b_filth):
-    """Sort list of filths per starting position and substrings"""
-
-    # if a_filth starts first return a
-    if a_filth.beg < b_filth.beg:
-        return -1
-
-    # if b_filth starts first return b
-    if a_filth.beg > b_filth.beg:
-        return 1
-
-    # if boths filths start in the same position
-    # return the inclusive filth
-    if a_filth.end > b_filth.end:
-        return -1
-
-    return 1

scrubadub/utils.py

@@ -1,11 +1,16 @@
+try:
+    unicode
+except NameError:
+    basestring = str  # Compatibility for Python 2 and 3
+
 
 class CanonicalStringSet(set):
     """Just like a set, except it makes sure that all elements are lower case
     strings.
     """
 
     def _cast_as_lower(self, x):
-        if not isinstance(x, (str, unicode)):
+        if not isinstance(x, basestring):
             raise TypeError('CanonicalStringSet only works with strings')
         return x.lower()
 

tests/base.py

@@ -3,6 +3,11 @@
 import scrubadub
 
 
+try:
+    unicode
+except NameError:
+    unicode = str  # Python 2 and 3 compatibility
+
 # this is a mixin class to make it easy to centralize a lot of the core
 # functionality of the test suite
 class BaseTestCase(object):

tests/test_unicode.py

@@ -1,9 +1,11 @@
+import sys
 import unittest
 
 import scrubadub
 
 from base import BaseTestCase
 
+@unittest.skipIf(sys.version_info >= (3,0), "Test only needed in Python 2")
 class UnicodeTestCase(unittest.TestCase, BaseTestCase):
 
     def test_empty(self):