DOC - mini-language doctest markup in action

Author: matthew-brett

nibabel/analyze.py

@@ -655,7 +655,7 @@ def __setitem__(self, item, value):
         --------
         >>> hdr = AnalyzeHeader()
         >>> hdr['descrip'] = 'description'
-        >>> np.asscalar(hdr['descrip']) #23: bytes
+        >>> np.asscalar(hdr['descrip']) #2to3: next; replace("'d", "b'd")
         'description'
         '''
         self._header_data[item] = value
@@ -830,15 +830,15 @@ def set_data_dtype(self, datatype):
         >>> hdr.set_data_dtype(np.dtype(np.uint8))
         >>> hdr.get_data_dtype()
         dtype('uint8')
-        >>> hdr.set_data_dtype('implausible')
+        >>> hdr.set_data_dtype('implausible') #doctest: +IGNORE_EXCEPTION_DETAIL
         Traceback (most recent call last):
            ...
         HeaderDataError: data dtype "implausible" not recognized
-        >>> hdr.set_data_dtype('none')
+        >>> hdr.set_data_dtype('none') #doctest: +IGNORE_EXCEPTION_DETAIL
         Traceback (most recent call last):
            ...
         HeaderDataError: data dtype "none" known but not supported
-        >>> hdr.set_data_dtype(np.void)
+        >>> hdr.set_data_dtype(np.void) #doctest: +IGNORE_EXCEPTION_DETAIL
         Traceback (most recent call last):
            ...
         HeaderDataError: data dtype "<type 'numpy.void'>" known but not supported

nibabel/nicom/structreader.py

@@ -14,7 +14,7 @@ class Unpacker(object):
 
     Examples
     --------
-    >>> a = '1234567890'
+    >>> a = '1234567890' #2to3: here; replace("'1", "b'1")
     >>> upk = Unpacker(a)
     >>> upk.unpack('2s')
     ('12',)

nibabel/trackvis.py

@@ -387,7 +387,7 @@ def empty_header(endianness=None, version=2):
     >>> hdr = empty_header()
     >>> print hdr['version']
     2
-    >>> np.asscalar(hdr['id_string']) #23: bytes
+    >>> np.asscalar(hdr['id_string']) #2to3: next; line.replace("'T", "b'T")
     'TRACK'
     >>> endian_codes[hdr['version'].dtype.byteorder] == native_code
     True

nibabel/volumeutils.py

@@ -390,12 +390,12 @@ def array_from_file(shape, in_dtype, infile, offset=0, order='F'):
     >>> from StringIO import StringIO as BytesIO
     >>> bio = BytesIO()
     >>> arr = np.arange(6).reshape(1,2,3)
-    >>> _ = bio.write(arr.tostring('F')) # outputs no in python3
+    >>> _ = bio.write(arr.tostring('F')) # outputs int in python3
     >>> arr2 = array_from_file((1,2,3), arr.dtype, bio)
     >>> np.all(arr == arr2)
     True
     >>> bio = BytesIO()
-    >>> _ = bio.write(' ' * 10) #2to3: line.replace(' ', b' ')
+    >>> _ = bio.write(' ' * 10) #2to3: here; replace("' '", "b' '")
     >>> _ = bio.write(arr.tostring('F'))
     >>> arr2 = array_from_file((1,2,3), arr.dtype, bio, 10)
     >>> np.all(arr == arr2)
@@ -869,7 +869,7 @@ def rec2dict(rec):
     --------
     >>> r = np.zeros((), dtype = [('x', 'i4'), ('s', 'S10')])
     >>> d = rec2dict(r)
-    >>> d == {'x': 0, 's': ''}
+    >>> d == {'x': 0, 's': ''} #2to3: here; replace("''", "b''")
     True
     '''
     dct = {}

nisext/py3builder.py

@@ -67,10 +67,11 @@ def doctest_markup_files(fnames):
      'from io import BytesIO'),
 )
 
-MARK_COMMENT = re.compile('#23:\s*(.*?)\s*$')
-FIRST_NW = re.compile('(\s*)(\S.*)', re.DOTALL)
+MARK_COMMENT = re.compile('(.*?)(#2to3:\s*)(.*?\s*)$', re.DOTALL)
+PLACE_EXPR = re.compile('\s*([\w+\- ]+);\s*(.*)$')
 
-def doctest_markup(lines):
+
+def doctest_markup(in_lines):
     """ Process doctest comment markup on sequence of strings
 
     The markup is very crude.  The algorithm looks for lines starting with
@@ -82,52 +83,80 @@ def doctest_markup(lines):
       BytesIO``.
     * ``from StringIO import StringIO`` replaced by ``from io import StringIO``.
 
-    Next it looks to see if the line ends with a comment starting with ``#23:``.
+    Next it looks to see if the line ends with a comment starting with ``#2to3:``.
+
+    The stuff after the ``#2to3:`` marker is, of course, a little language, of
+    form <place>; <expr>
+
+    * <place> is an expression giving a line number.  In this expression, ``here`` is
+      a variable referring to the current line number, and ``next`` is just
+      ``here+1``.
+    * <expr> is a python3 expression returning a processed value, where
+      ``line`` contains the line number referred to by ``here``, and ``lines``
+      is a list of all lines, such that ``lines[here]`` gives the value of
+      ``line``.
 
-    * ``bytes`` : prepend 'b' to next output line; for when you want to show
-      output of bytes type in python 3
+    An <expr> beginning with "replace(" we take to be short for "line.replace(".
 
     Parameters
     ----------
-    lines : sequence of str
+    in_lines : sequence of str
 
     Returns
     -------
-    newlines : sequence of str
+    out_lines : sequence of str
     """
-    newlines = []
-    lines = iter(lines)
-    while(True):
-        try:
-            line = next(lines)
-        except StopIteration:
-            break
-        if not line.lstrip().startswith('>>>'):
-            newlines.append(line)
+    pos = 0
+    lines = list(in_lines)
+    while pos < len(lines):
+        this = lines[pos]
+        here = pos
+        pos += 1
+        if not this.lstrip().startswith('>>>'):
             continue
         # Check simple regexps (no markup)
         for reg, substr in REGGIES:
-            if reg.search(line):
-                line = reg.sub(substr, line)
+            if reg.search(this):
+                lines[here] = reg.sub(substr, this)
                 break
         # Check for specific markup
-        mark_match = MARK_COMMENT.search(line)
+        mark_match = MARK_COMMENT.search(this)
         if mark_match is None:
-            newlines.append(line)
             continue
-        markup = mark_match.groups()[0]
-        # Actually we don't know what to do with markup yet
-        if markup == 'bytes':
-            newlines.append(line)
+        start, marker, markup = mark_match.groups()
+        pe_match = PLACE_EXPR.match(markup)
+        if pe_match:
+            place, expr = pe_match.groups()
             try:
-                line = next(lines)
-            except StopIteration:
-                break
-            match = FIRST_NW.match(line)
-            if match:
-                line = '%sb%s' % match.groups()
-            newlines.append(line)
-        else:
-            newlines.append(line)
-    return newlines
+                place = eval(place, {'here': here, 'next': here+1})
+            except:
+                print('Error finding place with "%s", line "%s"; skipping' %
+                      (place, this))
+                continue
+            # Prevent processing operating on comment
+            if place == here:
+                line = start
+            else:
+                line = lines[place]
+            expr = expr.strip()
+            # Shorthand stuff
+            if expr == 'bytes':
+                # Any strings on the given line are byte strings
+                raise NotImplementedError()
+            # If expr starts with 'replace', implies "line.replace"
+            if expr.startswith('replace('):
+                expr = 'line.' + expr
+            try:
+                res = eval(expr, dict(line=line,
+                                      lines=lines))
+            except:
+                print('Error working on "%s" at line %d with "%s"; skipping' %
+                      (line, place, expr))
+                continue
+            # Put back comment if removed
+            if place == here:
+                res += marker + markup
+            if res != line:
+                lines[place] = res
+    return lines
 

nisext/tests/test_doctest_markup.py

@@ -21,8 +21,27 @@ def test_search_replace():
     line = line + '\n'
     assert_equal(['>>> from io import BytesIO\n'], doctest_markup([line]))
     # Bytes output
-    assert_equal(['any', '  some'], doctest_markup(['any', '  some']))
-    assert_equal(['any #23: bytes', '  some'],
-                 doctest_markup(['any #23: bytes', '  some']))
-    assert_equal([' >>> any #23: bytes \n', '  bsome \n'],
-                 doctest_markup([' >>> any #23: bytes \n', '  some \n']))
+    marked_lines = ['any', '  some']
+    assert_equal(marked_lines, doctest_markup(marked_lines))
+    marked_lines = ['any #2to3: here+1; line.replace("some", "boo") ',
+                    ' some ']
+    assert_equal(marked_lines, doctest_markup(marked_lines))
+    marked_lines = ['>>> any #2to3: here+1; line.replace("some", "boo") ',
+                    ' some ']
+    exp_out = ['>>> any #2to3: here+1; line.replace("some", "boo") ',
+               ' boo ']
+    assert_equal(exp_out, doctest_markup(marked_lines))
+    marked_lines = ['>>> any #2to3: next; line.replace("some", "boo") ',
+                    ' some ']
+    exp_out = ['>>> any #2to3: next; line.replace("some", "boo") ',
+               ' boo ']
+    assert_equal(exp_out, doctest_markup(marked_lines))
+    assert_equal(['>>> woo #2to3: here ; line.replace("wow", "woo") '],
+                 doctest_markup(
+                     ['>>> wow #2to3: here ; line.replace("wow", "woo") ']))
+    assert_equal(['>>> woo #2to3: here ; line.replace("wow", "woo") \n'],
+                 doctest_markup(
+                     ['>>> wow #2to3: here ; line.replace("wow", "woo") \n']))
+    assert_equal(['>>> woo #2to3: here ; replace("wow", "woo") '],
+                 doctest_markup(
+                     ['>>> wow #2to3: here ; replace("wow", "woo") ']))