More py3 (#400)

* fix img_info tests in py3

* update transforms_t and webapp_t

* py3 fixes in parameters.py

* py3 fixes in webapp.py

* py3 fixes for logging_t and transforms_t

* transforms.py: remove unused import and comment

* py3: fix resolver tests

* py3: fix jp2_extractor_t

* py3: authorizer tests pass

* py3: fix failures on py3.4 and don't allow py3 test failures in travis anymore

* authorizer: require salt config parameter to be bytes; other unicode/bytes updates

* add a logger warning call if we get unicode Authorization header and encode it as utf8

* show value of 'salt' parameter if it's a ConfigError in RulesAuthorizer

* don't modify dict in place and return it, just modify it; fix logging statement

Author: bcail

.travis.yml

@@ -22,12 +22,6 @@ python:
   - 3.5
   - 3.6
 
-matrix:
-  allow_failures:
-    - python: 3.4
-    - python: 3.5
-    - python: 3.6
-
 install:
  - pip install -r requirements_test.txt
 

loris/authorizer.py

@@ -53,11 +53,9 @@ def __init__(self, config):
 
     def _strip_empty_fields(self, svc):
         # dicts are modified in place
-        for (k, v) in svc.items():
+        for k, v in list(svc.items()):
             if not v:
                 del svc[k]
-        # but return it just in case
-        return svc
 
     def is_protected(self, info):
         """
@@ -183,7 +181,7 @@ def __init__(self, config):
         self.token_secret = config['token_secret']
 
         self.use_jwt = config.get('use_jwt', True)
-        self.salt = config.get('salt', '')
+        self.salt = config.get('salt', b'')
         self.roles_key = config.get('roles_key', 'roles')
         self.id_key = config.get('id_key', 'sub')
 
@@ -203,6 +201,9 @@ def _validate_config(self, config):
                 config['use_jwt']
             )
 
+        if ('salt' in config) and (not isinstance(config['salt'], bytes)):
+            raise ConfigError('"salt" config parameter must be bytes; got %r (%s)' % (config['salt'], type(config['salt'])))
+
     def kdf(self):
         return PBKDF2HMAC(algorithm=hashes.SHA256(), length=32, salt=self.salt,
             iterations=100000, backend=default_backend())
@@ -251,22 +252,28 @@ def _roles_from_request(self, request):
         origin = request.headers.get('origin', '')
         if not origin:
             origin = request.headers.get('referer', '*')
-        origin = self.basic_origin(origin)
+        origin = self.basic_origin(origin).encode('utf8')
 
-        logger.debug("Got basic origin: %s" % origin)
+        logger.debug('Got basic origin: %s', origin)
 
-        if request.path.endswith("info.json"):
+        if request.path.endswith('info.json'):
             token = request.headers.get('Authorization', '')        
-            token = token.replace("Bearer", '')
+            if not isinstance(token, bytes):
+                logger.warning('encoding Authorization header as utf8')
+                token = token.encode('utf8')
+            token = token.replace(b'Bearer', b'')
             cval = token.strip()
             if not cval:
                 return []
-            secret = "%s-%s" % (self.token_secret, origin)
+            secret = b'-'.join([self.token_secret, origin])
         else:
             cval = request.cookies.get(self.cookie_name)
             if not cval:
                 return []
-            secret = "%s-%s" % (self.cookie_secret, origin)
+            secret = b'-'.join([self.cookie_secret, origin])
+
+        if not isinstance(cval, bytes):
+            cval = cval.encode('utf8')
 
         if self.use_jwt:
             try:
@@ -276,8 +283,7 @@ def _roles_from_request(self, request):
                 logger.debug(value)
                 raise AuthorizerException(message="invalidCredentials: expired")
         else:
-            cval = cval.encode('utf-8')
-            key = base64.urlsafe_b64encode(self.kdf().derive(secret.encode('utf-8')))
+            key = base64.urlsafe_b64encode(self.kdf().derive(secret))
             fern = Fernet(key)
             value = fern.decrypt(cval)
             if not value.startswith(origin):
@@ -286,6 +292,10 @@ def _roles_from_request(self, request):
             else:
                 value = value[len(origin)+1:]
 
+        try:
+            value = value.decode('utf8')
+        except AttributeError:
+            pass
         roles = self._roles_from_value(value)
         return roles
 

loris/img_info.py

@@ -230,7 +230,7 @@ def _from_jp2(self, fp):
                 self.extract_jp2(jp2)
             except JP2ExtractionError as err:
                 logger.warning(
-                    "Error extracting JP2 %s: %r", fp, err.message
+                    "Error extracting JP2 %s: %r", fp, str(err)
                 )
                 raise ImageInfoException("Invalid JP2 file")
 

loris/jp2_extractor.py

@@ -329,11 +329,11 @@ def extract_jp2(self, jp2):
 
         scaleFactors = []
 
-        window =  deque(jp2.read(2), 2)
+        window = deque(jp2.read(2), 2)
         # start of codestream
-        while map(ord, window) != [0xFF, 0x4F]: # (SOC - required, see pg 14)
+        while ((window[0] != b'\xFF') or (window[1] != b'\x4F')): # (SOC - required, see pg 14)
             window.append(jp2.read(1))
-        while map(ord, window) != [0xFF, 0x51]:  # (SIZ  - required, see pg 14)
+        while ((window[0] != b'\xFF') or (window[1] != b'\x51')):  # (SIZ  - required, see pg 14)
             window.append(jp2.read(1))
         jp2.read(20) # through Lsiz (16), Rsiz (16), Xsiz (32), Ysiz (32), XOsiz (32), YOsiz (32)
         tile_width = int(struct.unpack(">I", jp2.read(4))[0]) # XTsiz (32)
@@ -345,10 +345,8 @@ def extract_jp2(self, jp2):
             self.tiles[0]['height'] = tile_height
         jp2.read(10) # XTOsiz (32), YTOsiz (32), Csiz (16)
 
-        window =  deque(jp2.read(2), 2)
-        # while (ord(b) != 0xFF): b = jp2.read(1)
-        # b = jp2.read(1) # 0x52: The COD marker segment
-        while map(ord, window) != [0xFF, 0x52]:  # (COD - required, see pg 14)
+        window = deque(jp2.read(2), 2)
+        while ((window[0] != b'\xFF') or (window[1] != b'\x52')):  # (COD - required, see pg 14)
             window.append(jp2.read(1))
 
         jp2.read(7) # through Lcod (16), Scod (8), SGcod (32)

loris/parameters.py

@@ -98,7 +98,7 @@ def _canonicalize(self):
         # all adjustments have been made.
         if self.mode != FULL_MODE:
             px = (self.pixel_x, self.pixel_y, self.pixel_w, self.pixel_h)
-            self.canonical_uri_value = ','.join(map(str, px))
+            self.canonical_uri_value = ','.join([str(p) for p in px])
         else:
             self.canonical_uri_value = FULL_MODE
         logger.debug('canonical uri_value for region %s', self.canonical_uri_value)
@@ -150,7 +150,7 @@ def _populate_slots_from_pct(self):
             RequestException
         '''
         # we convert these to pixels and update uri_value
-        dimensions = map(float, self.uri_value.split(':')[1].split(','))
+        dimensions = [float(x) for x in self.uri_value.split(':')[1].split(',')]
 
         if len(dimensions) != 4:
             raise SyntaxException("Exactly (4) coordinates must be supplied.")
@@ -165,7 +165,7 @@ def _populate_slots_from_pct(self):
 
         # decimals
         self.decimal_x, self.decimal_y, self.decimal_w, \
-            self.decimal_h = map(RegionParameter._pct_to_decimal, dimensions)
+            self.decimal_h = [RegionParameter._pct_to_decimal(d) for d in dimensions]
 
         # pixels
         self.pixel_x = int(floor(self.decimal_x * self.img_info.width))
@@ -179,16 +179,17 @@ def _populate_slots_for_square(self):
             RequestException
             SyntaxException
         '''
+        #dimensions must be ints, for passing to _populate_slots_from_pixels
         if self.img_info.width > self.img_info.height:
-            offset = (self.img_info.width - self.img_info.height) / 2
+            offset = (self.img_info.width - self.img_info.height) // 2
             dimensions = (offset, 0, self.img_info.height, self.img_info.height)
         else:
-            offset = (self.img_info.height - self.img_info.width) / 2
+            offset = (self.img_info.height - self.img_info.width) // 2
             dimensions = (0, offset, self.img_info.width, self.img_info.width)
         return self._populate_slots_from_pixels(dimensions)
 
     def _pixel_dims_to_ints(self):
-        dimensions = map(int, self.uri_value.split(','))
+        dimensions = [int(d) for d in self.uri_value.split(',')]
         if any(n <= 0 for n in dimensions[2:]):
             raise RequestException("Width and height must be greater than 0.")
         if len(dimensions) != 4:
@@ -199,10 +200,10 @@ def _populate_slots_from_pixels(self, dimensions):
         # pixels
         self.pixel_x, self.pixel_y, self.pixel_w, self.pixel_h = dimensions
         # decimals
-        self.decimal_x = self.pixel_x / Decimal(str(self.img_info.width))
-        self.decimal_y = self.pixel_y / Decimal(str(self.img_info.height))
-        self.decimal_w = self.pixel_w / Decimal(str(self.img_info.width))
-        self.decimal_h = self.pixel_h / Decimal(str(self.img_info.height))
+        self.decimal_x = Decimal(self.pixel_x) / Decimal(str(self.img_info.width))
+        self.decimal_y = Decimal(self.pixel_y) / Decimal(str(self.img_info.height))
+        self.decimal_w = Decimal(self.pixel_w) / Decimal(str(self.img_info.width))
+        self.decimal_h = Decimal(self.pixel_h) / Decimal(str(self.img_info.height))
 
     @staticmethod
     def _mode_from_region_segment(region_segment, img_info):
@@ -243,6 +244,7 @@ def _mode_from_region_segment(region_segment, img_info):
     def _pct_to_decimal(n):
         return Decimal(str(n)) / Decimal('100.0')
 
+
 class SizeParameter(object):
     '''Internal representation of the size slice of an IIIF image URI.
 
@@ -348,13 +350,13 @@ def _populate_slots_from_pixels(self, region_parameter):
 
         if (not best_fit) and request_w and (not request_h):
             self.force_aspect = False
-            self.w = request_w
+            self.w = Decimal(request_w)
             reduce_by = Decimal(request_w) / region_parameter.pixel_w
             self.h = region_parameter.pixel_h * reduce_by
 
         elif (not best_fit) and (not request_w) and request_h:
             self.force_aspect = False
-            self.h = request_h
+            self.h = Decimal(request_h)
             reduce_by = Decimal(request_h) / region_parameter.pixel_h
             self.w = region_parameter.pixel_w * reduce_by
 
@@ -369,17 +371,17 @@ def _populate_slots_from_pixels(self, region_parameter):
 
         elif request_w and request_h:
             self.force_aspect = True
-            self.w = request_w
-            self.h = request_h
+            self.w = Decimal(request_w)
+            self.h = Decimal(request_h)
 
         else:  # pragma: no cover
             assert False, "Incomplete size data in URI: %r" % self.uri_value
 
-        if self.h < 1:
+        if 0 < self.h < DECIMAL_ONE:
             self.h = 1
         else:
             self.h = int(self.h)
-        if self.w < 1:
+        if 0 < self.w < DECIMAL_ONE:
             self.w = 1
         else:
             self.w = int(self.w)

loris/resolver.py

@@ -304,7 +304,7 @@ def _cache_subroot(ident):
     @staticmethod
     def _ident_file_structure(ident):
         file_structure = ''
-        ident_hash = hashlib.md5(quote_plus(ident)).hexdigest()
+        ident_hash = hashlib.md5(quote_plus(ident).encode('utf8')).hexdigest()
         # First level 2 digit directory then do three digits...
         file_structure_list = [ident_hash[0:2]] + [ident_hash[i:i+3] for i in range(2, len(ident_hash), 3)]
 

loris/webapp.py

@@ -353,7 +353,7 @@ def _load_transformers(self):
         transformers = {}
         for sf in source_formats:
             # merge [transforms] options and [transforms][source_format]] options
-            config = dict(self.app_configs['transforms'][sf].items() + global_tranform_options.items())
+            config = dict(list(self.app_configs['transforms'][sf].items()) + list(global_tranform_options.items()))
             transformers[sf] = self._load_transformer(config)
         return transformers
 
@@ -451,7 +451,7 @@ def get_index(self, request):
         '''
         Just so there's something at /.
         '''
-        f = file(path.join(self.www_dp, 'index.txt'))
+        f = open(path.join(self.www_dp, 'index.txt'), 'rb')
         r = Response(f, content_type='text/plain')
         if self.enable_caching:
             r.add_etag()
@@ -460,7 +460,7 @@ def get_index(self, request):
 
     def get_favicon(self, request):
         f = path.join(self.www_dp, 'icons', 'loris-icon.png')
-        r = Response(file(f), content_type='image/x-icon')
+        r = Response(open(f, 'rb'), content_type='image/x-icon')
         if self.enable_caching:
             r.add_etag()
             r.make_conditional(request)
@@ -470,9 +470,9 @@ def get_info(self, request, ident, base_uri):
         try:
             info, last_mod = self._get_info(ident,request,base_uri)
         except ResolverException as re:
-            return NotFoundResponse(re.message)
+            return NotFoundResponse(str(re))
         except ImageInfoException as ie:
-            return ServerSideErrorResponse(ie.message)
+            return ServerSideErrorResponse(str(ie))
         except IOError as e:
             msg = '%s \n(This is likely a permissions problem)' % e
             return ServerSideErrorResponse(msg)
@@ -586,7 +586,7 @@ def get_img(self, request, ident, region, size, rotation, quality, target_fmt, b
             # ... still cheaper than always resolving as likely to be cached
             info = self._get_info(ident, request, base_uri)[0]
         except ResolverException as re:
-            return NotFoundResponse(re.message)
+            return NotFoundResponse(str(re))
 
         if self.authorizer and self.authorizer.is_protected(info):
             authed = self.authorizer.is_authorized(info, request)
@@ -615,7 +615,7 @@ def get_img(self, request, ident, region, size, rotation, quality, target_fmt, b
                 r.status_code = 200
                 r.last_modified = img_last_mod
                 r.headers['Content-Length'] = path.getsize(fp)
-                r.response = file(fp)
+                r.response = open(fp, 'rb')
 
                 # hand the Image object its info
                 info = self._get_info(ident, request, base_uri)[0]
@@ -651,11 +651,11 @@ def get_img(self, request, ident, region, size, rotation, quality, target_fmt, b
                 fp = self._make_image(image_request, info.src_img_fp, info.src_format)
 
             except ResolverException as re:
-                return NotFoundResponse(re.message)
+                return NotFoundResponse(str(re))
             except TransformException as te:
                 return ServerSideErrorResponse(te)
             except (RequestException, SyntaxException) as e:
-                return BadRequestResponse(e.message)
+                return BadRequestResponse(str(e))
             except (ImageException,ImageInfoException) as ie:
                 # 500s!
                 # ImageException is only raised in when ImageRequest.info
@@ -679,7 +679,7 @@ def get_img(self, request, ident, region, size, rotation, quality, target_fmt, b
         r.last_modified = datetime.utcfromtimestamp(path.getctime(fp))
         r.headers['Content-Length'] = path.getsize(fp)
         self._set_canonical_link(request, image_request, r)
-        r.response = file(fp)
+        r.response = open(fp, 'rb')
 
         if not self.enable_caching:
             r.call_on_close(lambda: unlink(fp))