Merge pull request #27 from bartosh/master

Ported to Python3. Fixes #14

Author: kad

.travis.yml

@@ -2,14 +2,13 @@ language: python
 python:
   - "2.6"
   - "2.7"
-# bmap-tools don't support 3.x yet, thus disable.
-# pypy 2.x currently also disabled, until testing fixed.
-#  - "3.2"
-#  - "3.3"
-#  - "3.4"
-#  - "3.5"
-#  - "3.5-dev" # 3.5 development branch
-#  - "nightly" # currently points to 3.6-dev
+  - "3.3"
+  - "3.4"
+  - "3.5"
+  - "3.6"
+  - "3.7-dev" # 3.7 development branch
+  - "nightly" # currently points to 3.6-dev
+# pypy 2.x currently disabled, until testing fixed.
 #  - "pypy"
 #  - "pypy3"
 # command to install dependencies

bmaptools/BmapCopy.py

@@ -61,16 +61,21 @@
 import sys
 import hashlib
 import logging
-import Queue
-import thread
 import datetime
+if sys.version[0] == '2':
+    import Queue
+    import thread
+else:
+    import queue as  Queue
+    import _thread as thread
+
 from xml.etree import ElementTree
 from bmaptools.BmapHelpers import human_size
 
 _log = logging.getLogger(__name__)  # pylint: disable=C0103
 
 # The highest supported bmap format version
-SUPPORTED_BMAP_VERSION = 2
+SUPPORTED_BMAP_VERSION = "2.0"
 
 
 class Error(Exception):
@@ -208,7 +213,7 @@ def __init__(self, image, dest, bmap=None, image_size=None):
         if image_size:
             self._set_image_size(image_size)
 
-        self._batch_blocks = self._batch_bytes / self.block_size
+        self._batch_blocks = self._batch_bytes // self.block_size
 
     def set_progress_indicator(self, file_obj, format_string):
         """
@@ -241,8 +246,7 @@ def _set_image_size(self, image_size):
 
         self.image_size = image_size
         self.image_size_human = human_size(image_size)
-        self.blocks_cnt = self.image_size + self.block_size - 1
-        self.blocks_cnt /= self.block_size
+        self.blocks_cnt = (self.image_size + self.block_size - 1) // self.block_size
 
         if self.mapped_cnt is None:
             self.mapped_cnt = self.blocks_cnt
@@ -264,10 +268,10 @@ def _verify_bmap_checksum(self):
         mapped_bmap = mmap.mmap(self._f_bmap.fileno(), 0,
                                 access=mmap.ACCESS_COPY)
 
-        chksum_pos = mapped_bmap.find(correct_chksum)
+        chksum_pos = mapped_bmap.find(correct_chksum.encode())
         assert chksum_pos != -1
 
-        mapped_bmap[chksum_pos:chksum_pos + self._cs_len] = '0' * self._cs_len
+        mapped_bmap[chksum_pos:chksum_pos + self._cs_len] = b'0' * self._cs_len
 
         hash_obj = hashlib.new(self._cs_type)
         hash_obj.update(mapped_bmap)
@@ -305,7 +309,7 @@ def _parse_bmap(self):
         # Make sure we support this version
         self.bmap_version_major = int(self.bmap_version.split('.', 1)[0])
         self.bmap_version_minor = int(self.bmap_version.split('.', 1)[1])
-        if self.bmap_version_major > SUPPORTED_BMAP_VERSION:
+        if self.bmap_version_major > int(SUPPORTED_BMAP_VERSION.split('.', 1)[0]):
             raise Error("only bmap format version up to %d is supported, "
                         "version %d is not supported"
                         % (SUPPORTED_BMAP_VERSION, self.bmap_version_major))
@@ -320,7 +324,7 @@ def _parse_bmap(self):
         self.mapped_size_human = human_size(self.mapped_size)
         self.mapped_percent = (self.mapped_cnt * 100.0) / self.blocks_cnt
 
-        blocks_cnt = (self.image_size + self.block_size - 1) / self.block_size
+        blocks_cnt = (self.image_size + self.block_size - 1) // self.block_size
         if self.blocks_cnt != blocks_cnt:
             raise Error("Inconsistent bmap - image size does not match "
                         "blocks count (%d bytes != %d blocks * %d bytes)"
@@ -514,7 +518,7 @@ def _get_data(self, verify):
                     if verify and chksum:
                         hash_obj.update(buf)
 
-                    blocks = (len(buf) + self.block_size - 1) / self.block_size
+                    blocks = (len(buf) + self.block_size - 1) // self.block_size
                     _log.debug("queueing %d blocks, queue length is %d" %
                                (blocks, self._batch_queue.qsize()))
 
@@ -577,7 +581,7 @@ def copy(self, sync=True, verify=True):
                 # The reader thread encountered an error and passed us the
                 # exception.
                 exc_info = batch[1]
-                raise exc_info[0], exc_info[1], exc_info[2]
+                raise exc_info[1].with_traceback(exc_info[2])
 
             (start, end, buf) = batch[1:4]
 
@@ -664,7 +668,7 @@ def __init__(self, image, dest, bmap=None, image_size=None):
         # Call the base class constructor first
         BmapCopy.__init__(self, image, dest, bmap, image_size)
 
-        self._dest_fsync_watermark = (6 * 1024 * 1024) / self.block_size
+        self._dest_fsync_watermark = (6 * 1024 * 1024) // self.block_size
 
         self._sysfs_base = None
         self._sysfs_scheduler_path = None

bmaptools/BmapCreate.py

@@ -282,7 +282,7 @@ def _bmap_file_end(self):
 
         self._f_bmap.seek(0)
         hash_obj = hashlib.new(self._cs_type)
-        hash_obj.update(self._f_bmap.read())
+        hash_obj.update(self._f_bmap.read().encode())
         chksum = hash_obj.hexdigest()
         self._f_bmap.seek(self._chksum_pos)
         self._f_bmap.write("%s" % chksum)

bmaptools/CLI.py

@@ -451,7 +451,7 @@ def copy_command(args):
 
     # Print the progress indicator while copying
     if not args.quiet and not args.debug and \
-            os.isatty(sys.stderr.fileno()) and os.isatty(sys.stdout.fileno()):
+       sys.stderr.isatty() and sys.stdout.isatty():
         writer.set_progress_indicator(sys.stderr, "bmaptool: info: %d%% copied")
 
     start_time = time.time()
@@ -491,7 +491,7 @@ def copy_command(args):
         raise SystemExit(1)
 
     copying_time = time.time() - start_time
-    copying_speed = writer.mapped_size / copying_time
+    copying_speed = writer.mapped_size // copying_time
     log.info("copying time: %s, copying speed %s/sec"
              % (BmapHelpers.human_time(copying_time),
                 BmapHelpers.human_size(copying_speed)))
@@ -720,12 +720,12 @@ def main():
         log.info("The contents of /proc/meminfo:")
         with open('/proc/meminfo', 'rt') as file_obj:
             for line in file_obj:
-                print line,
+                print(line.strip())
 
         log.info("The contents of /proc/self/status:")
         with open('/proc/self/status', 'rt') as file_obj:
             for line in file_obj:
-                print line,
+                print(line.strip())
 
 if __name__ == "__main__":
     sys.exit(main())

bmaptools/Filemap.py

@@ -85,8 +85,7 @@ def __init__(self, image):
             raise Error("cannot get block size for '%s': %s"
                         % (self._image_path, err))
 
-        self.blocks_cnt = self.image_size + self.block_size - 1
-        self.blocks_cnt /= self.block_size
+        self.blocks_cnt = (self.image_size + self.block_size - 1) // self.block_size
 
         try:
             self._f_image.flush()
@@ -247,7 +246,7 @@ def block_is_mapped(self, block):
         if offs == -1:
             result = False
         else:
-            result = (offs / self.block_size == block)
+            result = (offs // self.block_size == block)
 
         _log.debug("FilemapSeek: block_is_mapped(%d) returns %s"
                    % (block, result))
@@ -279,8 +278,8 @@ def _get_ranges(self, start, count, whence1, whence2):
             if end > limit:
                 end = limit
 
-            start_blk = start / self.block_size
-            end_blk = end / self.block_size - 1
+            start_blk = start // self.block_size
+            end_blk = end // self.block_size - 1
             _log.debug("FilemapSeek: yielding range (%d, %d)"
                        % (start_blk, end_blk))
             yield (start_blk, end_blk)
@@ -345,7 +344,7 @@ def __init__(self, image):
 
         # Calculate how many 'struct fiemap_extent' elements fit the buffer
         self._buf_size -= _FIEMAP_SIZE
-        self._fiemap_extent_cnt = self._buf_size / _FIEMAP_EXTENT_SIZE
+        self._fiemap_extent_cnt = self._buf_size // _FIEMAP_EXTENT_SIZE
         assert self._fiemap_extent_cnt > 0
         self._buf_size = self._fiemap_extent_cnt * _FIEMAP_EXTENT_SIZE
         self._buf_size += _FIEMAP_SIZE
@@ -450,11 +449,11 @@ def _do_get_mapped_ranges(self, start, count):
                 # Start of the extent
                 extent_start = fiemap_extent[0]
                 # Starting block number of the extent
-                extent_block = extent_start / self.block_size
+                extent_block = extent_start // self.block_size
                 # Length of the extent
                 extent_len = fiemap_extent[2]
                 # Count of blocks in the extent
-                extent_count = extent_len / self.block_size
+                extent_count = extent_len // self.block_size
 
                 # Extent length and offset have to be block-aligned
                 assert extent_start % self.block_size == 0
@@ -476,7 +475,7 @@ def get_mapped_ranges(self, start, count):
         _log.debug("FilemapFiemap: get_mapped_ranges(%d,  %d(%d))"
                    % (start, count, start + count - 1))
         iterator = self._do_get_mapped_ranges(start, count)
-        first_prev, last_prev = iterator.next()
+        first_prev, last_prev = next(iterator)
 
         for first, last in iterator:
             if last_prev == first - 1:

bmaptools/TransRead.py

@@ -26,7 +26,11 @@
 
 import os
 import errno
-import urlparse
+import sys
+if sys.version[0] == '2':
+    import urlparse
+else:
+    import urllib.parse as urlparse
 import logging
 import threading
 import subprocess
@@ -376,7 +380,7 @@ def is_tar_lz4(name):
         else:
             args = decompressor + " " + args
 
-        if type(self._f_objs[-1]) == file:
+        if hasattr(self._f_objs[-1], 'fileno'):
             child_stdin = self._f_objs[-1].fileno()
         else:
             child_stdin = subprocess.PIPE
@@ -487,10 +491,15 @@ def _print_warning(timeout):
                          "proxy configured correctly? Keep trying ..." %
                          timeout)
 
-        import urllib2
-        import httplib
         import socket
 
+        if sys.version[0] == '2':
+            import httplib
+            import urllib2
+        else:
+            import http.client as httplib
+            import urllib.request as urllib2
+
         parsed_url = urlparse.urlparse(url)
 
         if parsed_url.scheme == "ssh":
@@ -535,7 +544,7 @@ def _print_warning(timeout):
             try:
                 f_obj = opener.open(url, timeout=timeout)
             # Handling the timeout case in Python 2.7
-            except socket.timeout, err:
+            except socket.timeout as err:
                 if timeout is not None:
                     _print_warning(timeout)
                 else:

setup.py

@@ -34,5 +34,9 @@ def get_version():
                      "can be copied or flashed a lot faster with bmaptool "
                      "than with traditional tools like \"dd\" or \"cp\". See "
                      "source.tizen.org/documentation/reference/bmaptool for "
-                     "more information."
+                     "more information.",
+    classifiers=[
+        "Programming Language :: Python :: 2.7",
+        "Programming Language :: Python :: 3"
+    ]
 )

tests/helpers.py

@@ -27,6 +27,7 @@
 import random
 import itertools
 import hashlib
+import struct
 import sys
 import os
 from bmaptools import BmapHelpers, BmapCopy, TransRead
@@ -43,7 +44,7 @@ def _create_random_sparse_file(file_obj, size):
 
     file_obj.truncate(size)
     block_size = BmapHelpers.get_block_size(file_obj)
-    blocks_cnt = (size + block_size - 1) / block_size
+    blocks_cnt = (size + block_size - 1) // block_size
 
     def process_block(block):
         """
@@ -60,17 +61,16 @@ def process_block(block):
             write = random.randint(1, block_size - seek)
             assert seek + write <= block_size
             file_obj.seek(block * block_size + seek)
-            file_obj.write(chr(random.getrandbits(8)) * write)
-
+            file_obj.write(struct.pack("=B", random.getrandbits(8)) * write)
         return map_the_block
 
     mapped = []
     unmapped = []
-    iterator = xrange(0, blocks_cnt)
+    iterator = range(0, blocks_cnt)
     for was_mapped, group in itertools.groupby(iterator, process_block):
         # Start of a mapped region or a hole. Find the last element in the
         # group.
-        first = group.next()
+        first = next(group)
         last = first
         for last in group:
             pass
@@ -98,7 +98,8 @@ def _create_random_file(file_obj, size):
         if written + chunk_size > size:
             chunk_size = size - written
 
-        file_obj.write(chr(random.getrandbits(8)) * chunk_size)
+        file_obj.write(struct.pack("=B", random.getrandbits(8)) * chunk_size)
+
         written += chunk_size
 
     file_obj.flush()
@@ -161,13 +162,13 @@ def generate_test_files(max_size=4 * 1024 * 1024, directory=None, delete=True):
     file_obj.close()
 
     # And 10 holes of random size
-    for i in xrange(10):
+    for i in range(10):
         size = random.randint(1, max_size)
         file_obj = tempfile.NamedTemporaryFile("wb+", suffix=".img",
                                                delete=delete, dir=directory,
                                                prefix="rand_hole_%d_" % i)
         file_obj.truncate(size)
-        blocks_cnt = (size + block_size - 1) / block_size
+        blocks_cnt = (size + block_size - 1) // block_size
         yield (file_obj, size, [], [(0, blocks_cnt - 1)])
         file_obj.close()
 
@@ -200,7 +201,7 @@ def generate_test_files(max_size=4 * 1024 * 1024, directory=None, delete=True):
     file_obj.close()
 
     # And 10 files of random size
-    for i in xrange(10):
+    for i in range(10):
         size = random.randint(1, max_size)
         file_obj = tempfile.NamedTemporaryFile("wb+", suffix=".img",
                                                delete=delete, dir=directory,
@@ -246,13 +247,13 @@ def generate_test_files(max_size=4 * 1024 * 1024, directory=None, delete=True):
     file_obj.close()
 
     # And 10 mapped files of random size
-    for i in xrange(10):
+    for i in range(10):
         size = random.randint(1, max_size)
         file_obj = tempfile.NamedTemporaryFile("wb+", suffix=".img",
                                                delete=delete, dir=directory,
                                                prefix="rand_mapped_%d_" % i)
         _create_random_file(file_obj, size)
-        blocks_cnt = (size + block_size - 1) / block_size
+        blocks_cnt = (size + block_size - 1) // block_size
         yield (file_obj, size, [(0, blocks_cnt - 1)], [])
         file_obj.close()
 
@@ -282,16 +283,15 @@ def copy_and_verify_image(image, dest, bmap, image_chksum, image_size):
     """
 
     f_image = TransRead.TransRead(image)
-    f_dest = open(dest, "w+")
+    f_dest = open(dest, "w+b")
     if (bmap):
         f_bmap = open(bmap, "r")
     else:
         f_bmap = None
 
     writer = BmapCopy.BmapCopy(f_image, f_dest, f_bmap, image_size)
     # Randomly decide whether we want the progress bar or not
-    if bool(random.getrandbits(1)) and hasattr(sys.stdout, 'fileno') and \
-            os.isatty(sys.stdout.fileno()):
+    if bool(random.getrandbits(1)) and sys.stdout.isatty():
         writer.set_progress_indicator(sys.stdout, None)
     writer.copy(bool(random.getrandbits(1)), bool(random.getrandbits(1)))