python3 (only) support

rkern/grin#1
and
rkern/grin#4

Author: rsalmaso

examples/grin

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # NOTE: You may have to adjust the line above to specify the precise python
 # executable that has the grin module installed.
 import grin

examples/grind

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # NOTE: You may have to adjust the line above to specify the precise python
 # executable that has the grin module installed.
 import grin

examples/grinimports.py

@@ -1,11 +1,11 @@
-#!/usr/bin/env python
-# -*- coding: UTF-8 -*-
-""" Transform Python files into normalized import statements for grepping.
+#!/usr/bin/env python3
+"""
+Transform Python files into normalized import statements for grepping.
 """
 
 import compiler
 from compiler.visitor import ASTVisitor, walk
-from cStringIO import StringIO
+from io import StringIO
 import os
 import shlex
 import sys
@@ -70,7 +70,7 @@ def normalize_file(filename, *args):
     """
     try:
         ast = compiler.parseFile(filename)
-    except Exception, e:
+    except:
         return StringIO('')
     ip = ImportPuller()
     walk(ast, ip)

examples/grinpython.py

@@ -1,9 +1,9 @@
-#!/usr/bin/env python
-# -*- coding: UTF-8 -*-
-""" Transform Python code by omitting strings, comments, and/or code.
+#!/usr/bin/env python3
+"""
+Transform Python code by omitting strings, comments, and/or code.
 """
 
-from cStringIO import StringIO
+from io import BytesIO
 import os
 import shlex
 import string
@@ -55,7 +55,7 @@ def __call__(self, filename, mode='rb'):
         """ Open a file and convert it to a filelike object with transformed
         contents.
         """
-        g = StringIO()
+        g = BytesIO()
         f = open(filename, mode)
         try:
             gen = tokenize.generate_tokens(f.readline)

grin.py

@@ -1,5 +1,6 @@
-#!/usr/bin/env python
-""" grin searches text files.
+#!/usr/bin/env python3
+"""
+grin searches text files.
 """
 
 import argparse
@@ -12,6 +13,7 @@
 import shlex
 import stat
 import sys
+from io import UnsupportedOperation
 
 # Constants
 __version__ = '1.2.1'
@@ -22,17 +24,18 @@
 POST = 1
 
 # Use file(1)'s choices for what's text and what's not.
-TEXTCHARS = ''.join(map(chr, [7, 8, 9, 10, 12, 13, 27] + range(0x20, 0x100)))
-ALLBYTES = ''.join(map(chr, range(256)))
+TEXTCHARS = bytes([7,8,9,10,12,13,27] + list(range(0x20, 0x100)))
+ALLBYTES = bytes(range(256))
 
 COLOR_TABLE = ['black', 'red', 'green', 'yellow', 'blue', 'magenta', 'cyan', 'white', 'default']
 COLOR_STYLE = {
     'filename': dict(fg="green", bold=True),
     'searchterm': dict(fg="black", bg="yellow"),
 }
+to_str = lambda s : s.decode('latin1')
 
 # gzip magic header bytes.
-GZIP_MAGIC = '\037\213'
+GZIP_MAGIC = b'\037\213'
 
 # Target amount of data to read into memory at a time.
 READ_BLOCKSIZE = 16 * 1024 * 1024
@@ -235,6 +238,10 @@ def read_block_with_context(self, prev, fp, fp_size):
             block_main = fp.read(target_io_size)
             is_last_block = target_io_size == remaining
 
+        if not isinstance(block_main, str):
+            # python3: ensure string are returned
+            block_main = block_main.decode("utf-8")
+
         if prev is None:
             if is_last_block:
                 # FAST PATH: the entire file fits into a single block, so we
@@ -279,7 +286,7 @@ def read_block_with_context(self, prev, fp, fp_size):
             after_lines = ''.join(after_lines_list)
 
         result = DataBlock(
-            data=before_lines + curr_block + after_lines,
+            data=(before_lines + curr_block + after_lines).decode("utf-8"),
             start=len(before_lines),
             end=len(before_lines) + len(curr_block),
             before_count=before_count,
@@ -307,13 +314,15 @@ def do_grep(self, fp):
             fp_size = None  # gzipped data is usually longer than the file
         else:
             try:
-                status = os.fstat(fp.fileno())
+                file_no = fp.fileno()
+            except (AttributeError, UnsupportedOperation):  # doesn't support fileno()
+                fp_size = None
+            else:
+                status = os.fstat(file_no)
                 if stat.S_ISREG(status.st_mode):
                     fp_size = status.st_size
                 else:
                     fp_size = None
-            except AttributeError:  # doesn't support fileno()
-                fp_size = None
 
         block = self.read_block_with_context(None, fp, fp_size)
         while block.end > block.start:
@@ -494,8 +503,8 @@ def grep_a_file(self, filename, opener=open):
             f = sys.stdin
             filename = '<STDIN>'
         else:
-            # 'r' does the right thing for both open ('rt') and gzip.open ('rb')
-            f = opener(filename, 'r')
+            # Always open in binary mode
+            f = opener(filename, 'rb')
         try:
             unique_context = self.do_grep(f)
         finally:
@@ -568,10 +577,8 @@ def is_binary(self, filename):
         -------
         is_binary : bool
         """
-        f = open(filename, 'rb')
-        is_binary = self._is_binary_file(f)
-        f.close()
-        return is_binary
+        with open(filename, 'rb') as fp:
+            return self._is_binary_file(fp)
 
     def _is_binary_file(self, f):
         """ Determine if a given filelike object has binary data or not.
@@ -585,12 +592,12 @@ def _is_binary_file(self, f):
         is_binary : bool
         """
         try:
-            bytes = f.read(self.binary_bytes)
+            data = f.read(self.binary_bytes)
         except Exception as e:
             # When trying to read from something that looks like a gzipped file,
             # it may be corrupt. If we do get an error, assume that the file is binary.
             return True
-        return is_binary_string(bytes)
+        return is_binary_string(data)
 
     def is_gzipped_text(self, filename):
         """ Determine if a given file is a gzip-compressed text file or not.
@@ -607,20 +614,17 @@ def is_gzipped_text(self, filename):
         is_gzipped_text : bool
         """
         is_gzipped_text = False
-        f = open(filename, 'rb')
-        marker = f.read(2)
-        f.close()
+        with open(filename, 'rb') as fp:
+            marker = fp.read(2)
+
         if marker == GZIP_MAGIC:
-            fp = gzip.open(filename)
-            try:
+            with gzip.open(filename) as fp:
                 try:
                     is_gzipped_text = not self._is_binary_file(fp)
                 except IOError:
                     # We saw the GZIP_MAGIC marker, but it is not actually a gzip
                     # file.
                     is_gzipped_text = False
-            finally:
-                fp.close()
         return is_gzipped_text
 
     def recognize(self, filename):
@@ -1209,10 +1213,6 @@ def grin_main(argv=None):
         raise
 
 
-def print_line(filename):
-    print(filename)
-
-
 def print_null(filename):
     # Note that the final filename will have a trailing NUL, just like
     # "find -print0" does.
@@ -1233,7 +1233,7 @@ def grind_main(argv=None):
         if args.null_separated:
             output = print_null
         else:
-            output = print_line
+            output = print
 
         if args.sys_path:
             args.dirs.extend(sys.path)

setup.cfg

@@ -9,7 +9,7 @@ with-doctest=1
 doctest-tests=1
 
 [bdist_wheel]
-universal = 1
+universal = 0
 
 [metadata]
 description-file = README.rst

tests/test_file_recognizer.py

@@ -8,22 +8,22 @@
 import sys
 
 import nose
+from grin import FileRecognizer, GZIP_MAGIC
 
-from grin import FileRecognizer
 
 def empty_file(filename, open=open):
     f = open(filename, 'wb')
     f.close()
 
 def binary_file(filename, open=open):
     f = open(filename, 'wb')
-    f.write(''.join(map(chr, range(256))))
+    f.write(bytes(range(255)))
     f.close()
 
 def text_file(filename, open=open):
-    lines = ['foo\n', 'bar\n'] * 100
-    lines.append('baz\n')
-    lines.extend(['foo\n', 'bar\n'] * 100)
+    lines = [b'foo\n', b'bar\n'] * 100
+    lines.append(b'baz\n')
+    lines.extend([b'foo\n', b'bar\n'] * 100)
     f = open(filename, 'wb')
     f.writelines(lines)
     f.close()
@@ -32,18 +32,17 @@ def fake_gzip_file(filename, open=open):
     """ Write out a binary file that has the gzip magic header bytes, but is not
     a gzip file.
     """
-    GZIP_MAGIC = '\037\213'
     f = open(filename, 'wb')
     f.write(GZIP_MAGIC)
-    f.write(''.join(map(chr, range(256))))
+    f.write(bytes(range(255)))
     f.close()
 
 def binary_middle(filename, open=open):
     """ Write out a file that is text for the first 100 bytes, then 100 binary
     bytes, then 100 text bytes to test that the recognizer only reads some of
     the file.
     """
-    text = 'a'*100 + '\0'*100 + 'b'*100
+    text = b'a'*100 + b'\0'*100 + b'b'*100
     f = open(filename, 'wb')
     f.write(text)
     f.close()
@@ -56,25 +55,25 @@ def unreadable_file(filename):
     """ Write a file that does not have read permissions.
     """
     text_file(filename)
-    os.chmod(filename, 0200)
+    os.chmod(filename, 0o200)
 
 def unreadable_dir(filename):
     """ Make a directory that does not have read permissions.
     """
     os.mkdir(filename)
-    os.chmod(filename, 0300)
+    os.chmod(filename, 0o300)
 
 def unexecutable_dir(filename):
     """ Make a directory that does not have execute permissions.
     """
     os.mkdir(filename)
-    os.chmod(filename, 0600)
+    os.chmod(filename, 0o600)
 
 def totally_unusable_dir(filename):
     """ Make a directory that has neither read nor execute permissions.
     """
     os.mkdir(filename)
-    os.chmod(filename, 0100)
+    os.chmod(filename, 0o100)
 
 def setup():
     # Make files to test individual recognizers.
@@ -135,22 +134,22 @@ def setup():
     text_file('tree/.skip_hidden_file')
     os.mkdir('tree/unreadable_dir')
     text_file('tree/unreadable_dir/text')
-    os.chmod('tree/unreadable_dir', 0300)
+    os.chmod('tree/unreadable_dir', 0o300)
     os.mkdir('tree/unexecutable_dir')
     text_file('tree/unexecutable_dir/text')
-    os.chmod('tree/unexecutable_dir', 0600)
+    os.chmod('tree/unexecutable_dir', 0o600)
     os.mkdir('tree/totally_unusable_dir')
     text_file('tree/totally_unusable_dir/text')
-    os.chmod('tree/totally_unusable_dir', 0100)
+    os.chmod('tree/totally_unusable_dir', 0o100)
 
-def ensure_deletability(arg, dirname, fnames):
-    """ os.path.walk() callback function which will make sure every directory is
-    readable and executable so that it may be easily deleted.
-    """
-    for fn in fnames:
-        fn = os.path.join(dirname, fn)
-        if os.path.isdir(fn):
-            os.chmod(fn, 0700)
+#def ensure_deletability(arg, dirname, fnames):
+    #""" os.path.walk() callback function which will make sure every directory is
+    #readable and executable so that it may be easily deleted.
+    #"""
+    #for fn in fnames:
+        #fn = os.path.join(dirname, fn)
+        #if os.path.isdir(fn):
+            #os.chmod(fn, 0700)
 
 def teardown():
     files_to_delete = ['empty', 'binary', 'binary_middle', 'text', 'text~',
@@ -168,10 +167,14 @@ def teardown():
                 os.unlink(filename)
             else:
                 os.rmdir(filename)
-        except Exception, e:
-            print >>sys.stderr, 'Could not delete %s: %s' % (filename, e)
+        except Exception as e:
+            print('Could not delete %s: %s' % (filename, e), file=sys.stderr)
     os.unlink('socket_test')
-    os.path.walk('tree', ensure_deletability, None)
+#    os.path.walk('tree', ensure_deletability, None)
+    for dirpath, dirnames, filenames in os.walk('tree'):
+        # Make sure every directory can be deleted
+        for dirname in dirnames:
+            os.chmod(os.path.join(dirpath, dirname), 0o700)
     shutil.rmtree('tree')
 
 
@@ -409,5 +412,3 @@ def test_dot_dot():
     ]
     result = sorted(fr.walk('..'))
     assert result == truth
-
-