NF - add doctest search replace and markup parser

Author: matthew-brett

nibabel/analyze.py

@@ -141,8 +141,8 @@
      array_to_file, array_from_file, can_cast, \
      floating_point_types
 
-from .spatialimages import HeaderDataError, HeaderTypeError, \
-    ImageDataError, SpatialImage
+from .spatialimages import (HeaderDataError, HeaderTypeError,
+                            ImageDataError, SpatialImage)
 
 from . import imageglobals as imageglobals
 from .fileholders import FileHolderError, copy_file_map
@@ -395,10 +395,10 @@ def from_fileobj(klass, fileobj, endianness=None, check=True):
 
         Examples
         --------
-        >>> import StringIO
+        >>> from StringIO import StringIO as BytesIO
         >>> hdr = AnalyzeHeader()
-        >>> fileobj = StringIO.StringIO(hdr.binaryblock)
-        >>> fileobj.seek(0)
+        >>> fileobj = BytesIO(hdr.binaryblock)
+        >>> _ = fileobj.seek(0) # returns 0 in python 3
         >>> hdr2 = AnalyzeHeader.from_fileobj(fileobj)
         >>> hdr2.binaryblock == hdr.binaryblock
         True
@@ -445,8 +445,8 @@ def write_to(self, fileobj):
         Examples
         --------
         >>> hdr = AnalyzeHeader()
-        >>> import StringIO
-        >>> str_io = StringIO.StringIO()
+        >>> from StringIO import StringIO as BytesIO
+        >>> str_io = BytesIO()
         >>> hdr.write_to(str_io)
         >>> hdr.binaryblock == str_io.getvalue()
         True
@@ -612,8 +612,8 @@ def data_to_fileobj(self, data, fileobj):
         >>> hdr = AnalyzeHeader()
         >>> hdr.set_data_shape((1, 2, 3))
         >>> hdr.set_data_dtype(np.float64)
-        >>> from StringIO import StringIO
-        >>> str_io = StringIO()
+        >>> from StringIO import StringIO as BytesIO
+        >>> str_io = BytesIO()
         >>> data = np.arange(6).reshape(1,2,3)
         >>> hdr.data_to_fileobj(data, str_io)
         >>> data.astype(np.float64).tostring('F') == str_io.getvalue()
@@ -655,7 +655,7 @@ def __setitem__(self, item, value):
         --------
         >>> hdr = AnalyzeHeader()
         >>> hdr['descrip'] = 'description'
-        >>> str(hdr['descrip'])
+        >>> np.asscalar(hdr['descrip']) #23: bytes
         'description'
         '''
         self._header_data[item] = value

nibabel/trackvis.py

@@ -237,26 +237,26 @@ def write(fileobj, streamlines,  hdr_mapping=None, endianness=None):
 
     Examples
     --------
-    >>> from StringIO import StringIO
-    >>> file_obj = StringIO()
+    >>> from StringIO import StringIO as BytesIO
+    >>> file_obj = BytesIO()
     >>> pts0 = np.random.uniform(size=(10,3))
     >>> pts1 = np.random.uniform(size=(10,3))
     >>> streamlines = ([(pts0, None, None), (pts1, None, None)])
     >>> write(file_obj, streamlines)
-    >>> file_obj.seek(0)
+    >>> _ = file_obj.seek(0) # returns 0 in python 3
     >>> streams, hdr = read(file_obj)
     >>> len(streams)
     2
 
     If there are too many streamlines to fit in memory, you can pass an iterable
     thing instead of a list
 
-    >>> file_obj = StringIO()
+    >>> file_obj = BytesIO()
     >>> def gen():
     ...     yield (pts0, None, None)
     ...     yield (pts0, None, None)
     >>> write(file_obj, gen())
-    >>> file_obj.seek(0)
+    >>> _ = file_obj.seek(0)
     >>> streams, hdr = read(file_obj)
     >>> len(streams)
     2
@@ -387,8 +387,8 @@ def empty_header(endianness=None, version=2):
     >>> hdr = empty_header()
     >>> print hdr['version']
     2
-    >>> print hdr['id_string']
-    TRACK
+    >>> np.asscalar(hdr['id_string']) #23: bytes
+    'TRACK'
     >>> endian_codes[hdr['version'].dtype.byteorder] == native_code
     True
     >>> hdr = empty_header(swapped_code)

nisext/py3builder.py

@@ -1,5 +1,8 @@
 """ distutils utilities for porting to python 3 within 2-compatible tree """
 
+from __future__ import with_statement
+import re
+
 try:
     from distutils.command.build_py import build_py_2to3
 except ImportError:
@@ -35,4 +38,96 @@ def log_debug(self, msg, *args):
             r.refactor(files, write=True)
             # Then doctests
             r.refactor(files, write=True, doctests_only=True)
+            # Then custom doctests markup
+            doctest_markup_files(files)
+
+
+def doctest_markup_files(fnames):
+    """ Process simple doctest comment markup on sequence of filenames
+
+    Parameters
+    ----------
+    fnames : seq
+        sequence of filenames
+
+    Returns
+    -------
+    None
+    """
+    for fname in fnames:
+        with open(fname, 'rt') as fobj:
+            res = list(fobj)
+        out = doctest_markup(res)
+        with open(fname, 'wt') as fobj:
+            fobj.write(''.join(out))
+
+
+REGGIES = (
+     (re.compile('from\s+io\s+import\s+StringIO\s+as\s+BytesIO\s*?(?=$)'),
+     'from io import BytesIO'),
+)
+
+MARK_COMMENT = re.compile('#23:\s*(.*?)\s*$')
+FIRST_NW = re.compile('(\s*)(\S.*)', re.DOTALL)
+
+def doctest_markup(lines):
+    """ Process doctest comment markup on sequence of strings
+
+    The markup is very crude.  The algorithm looks for lines starting with
+    ``>>>``.  All other lines are passed through unchanged.
+
+    Next it looks for known simple search replaces and does those.
+
+    * ``from StringIO import StringIO as BytesIO`` replaced by ``from io import
+      BytesIO``.
+    * ``from StringIO import StringIO`` replaced by ``from io import StringIO``.
+
+    Next it looks to see if the line ends with a comment starting with ``#23:``.
+
+    * ``bytes`` : prepend 'b' to next output line; for when you want to show
+      output of bytes type in python 3
+
+    Parameters
+    ----------
+    lines : sequence of str
+
+    Returns
+    -------
+    newlines : sequence of str
+    """
+    newlines = []
+    lines = iter(lines)
+    while(True):
+        try:
+            line = next(lines)
+        except StopIteration:
+            break
+        if not line.lstrip().startswith('>>>'):
+            newlines.append(line)
+            continue
+        # Check simple regexps (no markup)
+        for reg, substr in REGGIES:
+            if reg.search(line):
+                line = reg.sub(substr, line)
+                break
+        # Check for specific markup
+        mark_match = MARK_COMMENT.search(line)
+        if mark_match is None:
+            newlines.append(line)
+            continue
+        markup = mark_match.groups()[0]
+        # Actually we don't know what to do with markup yet
+        if markup == 'bytes':
+            newlines.append(line)
+            try:
+                line = next(lines)
+            except StopIteration:
+                break
+            match = FIRST_NW.match(line)
+            if match:
+                line = '%sb%s' % match.groups()
+            newlines.append(line)
+        else:
+            newlines.append(line)
+    return newlines
 

nisext/tests/test_doctest_markup.py

@@ -0,0 +1,28 @@
+""" Testing doctest markup tests
+"""
+
+from ..py3builder import doctest_markup
+
+from numpy.testing import (assert_array_almost_equal,
+                           assert_array_equal)
+
+from nose.tools import assert_true, assert_equal, assert_raises
+
+
+def test_search_replace():
+    # Test search and replace regexps
+    assert_equal(['any string'], doctest_markup(['any string']))
+    line = 'from io import StringIO as BytesIO'
+    assert_equal([line], doctest_markup([line]))
+    line = '>>> ' + line
+    assert_equal(['>>> from io import BytesIO'], doctest_markup([line]))
+    line = line + '  '
+    assert_equal(['>>> from io import BytesIO'], doctest_markup([line]))
+    line = line + '\n'
+    assert_equal(['>>> from io import BytesIO\n'], doctest_markup([line]))
+    # Bytes output
+    assert_equal(['any', '  some'], doctest_markup(['any', '  some']))
+    assert_equal(['any #23: bytes', '  some'],
+                 doctest_markup(['any #23: bytes', '  some']))
+    assert_equal([' >>> any #23: bytes \n', '  bsome \n'],
+                 doctest_markup([' >>> any #23: bytes \n', '  some \n']))