Merge branch 'develop' into jep/grouping

Author: springcomp

.gitignore

@@ -1,5 +1,10 @@
 *.py[cod]
 
+# pyvenv
+bin/
+share/
+pyvenv.cfg
+
 # C extensions
 *.so
 

bin/jp.py

@@ -42,6 +42,9 @@ def main():
     except exceptions.JMESPathTypeError as e:
         sys.stderr.write("invalid-type: %s\n" % e)
         return 1
+    except exceptions.JMESPathValueError as e:
+        sys.stderr.write("invalid-value: %s\n" % e)
+        return 1
     except exceptions.UnknownFunctionError as e:
         sys.stderr.write("unknown-function: %s\n" % e)
         return 1

jmespath/ast.py

@@ -2,6 +2,14 @@
 # {"type": <node type>", children: [], "value": ""}
 
 
+def arithmetic_unary(operator, expression):
+    return {'type': 'arithmetic_unary', 'children': [expression], 'value': operator}
+
+
+def arithmetic(operator, left, right):
+    return {'type': 'arithmetic', 'children': [left, right], 'value': operator}
+
+
 def comparator(name, first, second):
     return {'type': 'comparator', 'children': [first, second], 'value': name}
 

jmespath/compat.py

@@ -1,8 +1,8 @@
-import sys
 import inspect
-from itertools import zip_longest
 
+iteritems = dict.items
 
+map = map
 text_type = str
 string_type = str
 

jmespath/exceptions.py

@@ -112,6 +112,19 @@ def __str__(self):
                     self.expected_types, self.actual_type))
 
 
+@with_str_method
+class JMESPathValueError(JMESPathError):
+    def __init__(self, function_name, current_value, expected_types):
+        self.function_name = function_name
+        self.current_value = current_value
+        self.expected_types = expected_types
+
+    def __str__(self):
+        return ('In function %s(), invalid value: "%s", '
+                'expected: %s"%s"' % (
+                    self.function_name, self.current_value,
+                    self.expected_types))
+
 class EmptyExpressionError(JMESPathError):
     def __init__(self):
         super(EmptyExpressionError, self).__init__(

jmespath/functions.py

@@ -4,8 +4,10 @@
 from collections import OrderedDict
 
 from jmespath import exceptions
-from jmespath.compat import string_type as STRING_TYPE
 from jmespath.compat import get_methods
+from jmespath.compat import iteritems
+from jmespath.compat import map
+from jmespath.compat import string_type as STRING_TYPE
 
 
 # python types -> jmespath types
@@ -83,22 +85,28 @@ def call_function(self, function_name, resolved_args):
         return function(self, *resolved_args)
 
     def _validate_arguments(self, args, signature, function_name):
-        if signature and signature[-1].get('variadic'):
+        required_arguments_count = len([param for param in signature if not param.get('optional') or not param['optional']])
+        optional_arguments_count = len([param for param in signature if param.get('optional') and param['optional']])
+        has_variadic = signature[-1].get('variadic') if signature != None else False
+        if has_variadic:
             if len(args) < len(signature):
                 raise exceptions.VariadictArityError(
                     len(signature), len(args), function_name)
-        elif len(args) != len(signature):
+        elif optional_arguments_count > 0:
+            if len(args) < required_arguments_count or len(args) > (required_arguments_count + optional_arguments_count):
+                raise exceptions.ArityError(
+                    len(signature), len(args), function_name)
+        elif len(args) != required_arguments_count:
             raise exceptions.ArityError(
                 len(signature), len(args), function_name)
         return self._type_check(args, signature, function_name)
 
     def _type_check(self, actual, signature, function_name):
-        for i in range(len(signature)):
-            allowed_types = signature[i]['types']
+        for i in range(min(len(signature), len(actual))):
+            allowed_types = self._get_allowed_types_from_signature(signature[i])
             if allowed_types:
                 self._type_check_single(actual[i], allowed_types,
                                         function_name)
-
     def _type_check_single(self, current, types, function_name):
         # Type checking involves checking the top level type,
         # and in the case of arrays, potentially checking the types
@@ -122,6 +130,13 @@ def _type_check_single(self, current, types, function_name):
             self._subtype_check(current, allowed_subtypes,
                                 types, function_name)
 
+    ## signature supports monotype {'type': 'type-name'}
+    ## or multiple types {'types': ['type1-name', 'type2-name']}
+    def _get_allowed_types_from_signature(self, spec):
+        if spec.get('type'):
+            spec.update({'types': [spec.get('type')]})
+        return spec.get('types')
+
     def _get_allowed_pytypes(self, types):
         allowed_types = []
         allowed_subtypes = []
@@ -166,6 +181,14 @@ def _subtype_check(self, current, allowed_subtypes, types, function_name):
     @signature({'types': ['number']})
     def _func_abs(self, arg):
         return abs(arg)
+    
+    @signature({'types': ['string']})
+    def _func_lower(self, arg):
+        return arg.lower()
+    
+    @signature({'types': ['string']})
+    def _func_upper(self, arg):
+        return arg.upper()
 
     @signature({'types': ['array-number']})
     def _func_avg(self, arg):
@@ -283,12 +306,172 @@ def _func_sort(self, arg):
     def _func_sum(self, arg):
         return sum(arg)
 
+    @signature({'types': ['object']})
+    def _func_items(self, arg):
+        return list(map(list, iteritems(arg)))
+
+    @signature({'types': ['array']})
+    def _func_from_items(self, items):
+        return dict(items)
+
     @signature({"types": ['object']})
     def _func_keys(self, arg):
         # To be consistent with .values()
         # should we also return the indices of a list?
         return list(arg.keys())
 
+    @signature(
+        {'type': 'string'},
+        {'type': 'string'},
+        {'type': 'number', 'optional': True},
+        {'type': 'number', 'optional': True})
+    def _func_find_first(self, text, search, start = 0, end = None):
+        self._ensure_integer('find_first', 'start', start)
+        self._ensure_integer('find_first', 'end', end)
+        return self._find_impl(
+            text,
+            search,
+            lambda t, s: t.find(s),
+            start, 
+            end
+        )
+
+    @signature(
+        {'type': 'string'},
+        {'type': 'string'},
+        {'type': 'number', 'optional': True},
+        {'type': 'number', 'optional': True})
+    def _func_find_last(self, text, search, start = 0, end = None):
+        self._ensure_integer('find_last', 'start', start)
+        self._ensure_integer('find_last', 'end', end)
+        return self._find_impl(
+            text,
+            search,
+            lambda t, s: t.rfind(s),
+            start, 
+            end
+        )
+
+    def _find_impl(self, text, search, func, start, end):
+        if len(search) == 0:
+            return None
+        if end == None:
+            end = len(text)
+
+        pos = func(text[start:end], search)
+        if start < 0:
+            start = start + len(text)
+
+        # restrict resulting range to valid indices
+        start = min(max(start, 0), len(text))
+        return start + pos if pos != -1 else None
+
+    @signature(
+        {'type': 'string'},
+        {'type': 'number'},
+        {'type': 'string', 'optional': True})
+    def _func_pad_left(self, text, width, padding = ' '):
+        self._ensure_non_negative_integer('pad_left', 'width', width)
+        return self._pad_impl(lambda : text.rjust(width, padding), padding)
+
+    @signature(
+        {'type': 'string'},
+        {'type': 'number'},
+        {'type': 'string', 'optional': True})
+    def _func_pad_right(self, text, width, padding = ' '):
+        self._ensure_non_negative_integer('pad_right', 'width', width)
+        return self._pad_impl(lambda : text.ljust(width, padding), padding)
+
+    def _pad_impl(self, func, padding):
+        if len(padding) != 1:
+            raise exceptions.JMESPathError(
+                'syntax-error: pad_right() expects $padding to have a '
+                'single character, but received `{}` instead.'
+                .format(padding))
+        return func()
+
+    @signature(
+        {'type': 'string'},
+        {'type': 'string'},
+        {'type': 'string'},
+        {'type': 'number', 'optional': True})
+    def _func_replace(self, text, search, replacement, count = None):
+        self._ensure_non_negative_integer(
+            'replace',
+            'count',
+            count)
+
+        if count != None:
+            return text.replace(search, replacement, int(count))
+        return text.replace(search, replacement)
+
+    @signature(
+        {'type': 'string'},
+        {'type': 'string'},
+        {'type': 'number', 'optional': True})
+    def _func_split(self, text, search, count = None):
+        self._ensure_non_negative_integer(
+            'split',
+            'count',
+            count)
+
+        if len(search) == 0:
+            chars = list(text)
+            if count == None:
+                return chars
+
+            head = [c for c in chars[:count]]
+            tail =  [''.join(chars[count:])]
+            return head + tail
+
+        if count != None:
+            return text.split(search, count)
+        return text.split(search)
+    
+    def _ensure_integer(
+        self,
+        func_name,
+        param_name,
+        param_value):
+
+        if param_value != None: 
+            if int(param_value) != param_value:
+                raise exceptions.JMESPathValueError(
+                    func_name,
+                    param_value,
+                    "integer")
+
+    def _ensure_non_negative_integer(
+        self,
+        func_name,
+        param_name,
+        param_value):
+
+        if param_value != None: 
+            if int(param_value) != param_value or int(param_value) < 0:
+                raise exceptions.JMESPathValueError(
+                    func_name,
+                    param_name,
+                    "non-negative integer")
+
+    @signature({'type': 'string'}, {'type': 'string', 'optional': True})
+    def _func_trim(self, text, chars = None):
+        if chars == None or len(chars) == 0:
+            return text.strip()
+        return text.strip(chars)
+
+    @signature({'type': 'string'}, {'type': 'string', 'optional': True})
+    def _func_trim_left(self, text, chars = None):
+        if chars == None or len(chars) == 0:
+            return text.lstrip()
+        return text.lstrip(chars)
+
+    @signature({'type': 'string'}, {'type': 'string', 'optional': True})
+    def _func_trim_right(self, text, chars = None):
+        if chars == None or len(chars) == 0:
+            return text.rstrip()
+        return text.rstrip(chars)
+
     @signature({"types": ['object']})
     def _func_values(self, arg):
         return list(arg.values())
@@ -348,6 +531,10 @@ def _func_max_by(self, array, expref):
         else:
             return None
 
+    @signature({'types': ['array'], 'variadic': True})
+    def _func_zip(self, *arguments):
+        return list(map(list, zip(*arguments)))
+
     @signature({'types': ['array']}, {'types': ['expref']})
     def _func_group_by(self, array, expref):
         keyfunc = self._create_key_func(expref, ['null', 'string'], 'group_by')

jmespath/lexer.py

@@ -21,6 +21,11 @@ class Lexer(object):
         ')': 'rparen',
         '{': 'lbrace',
         '}': 'rbrace',
+        '+': 'plus',
+        '%': 'modulo',
+        u'\u2212': 'minus',
+        u'\u00d7': 'multiply',
+        u'\u00f7': 'divide',
     }
 
     def tokenize(self, expression):
@@ -68,16 +73,30 @@ def tokenize(self, expression):
                 yield {'type': 'number', 'value': int(buff),
                        'start': start, 'end': start + len(buff)}
             elif self._current == '-':
-                # Negative number.
-                start = self._position
-                buff = self._consume_number()
-                if len(buff) > 1:
-                    yield {'type': 'number', 'value': int(buff),
-                           'start': start, 'end': start + len(buff)}
+                if not self._peek_is_next_digit():
+                    self._next()
+                    yield {'type': 'minus', 'value': '-',
+                       'start': self._position - 1, 'end': self._position}
+                else:
+                    # Negative number.
+                    start = self._position
+                    buff = self._consume_number()
+                    if len(buff) > 1:
+                        yield {'type': 'number', 'value': int(buff),
+                            'start': start, 'end': start + len(buff)}
+                    else:
+                        raise LexerError(lexer_position=start,
+                                        lexer_value=buff,
+                                        message="Unknown token '%s'" % buff)
+            elif self._current == '/':
+                self._next()
+                if self._current == '/':
+                    self._next()
+                    yield {'type': 'div', 'value': '//',
+                        'start': self._position - 1, 'end': self._position}
                 else:
-                    raise LexerError(lexer_position=start,
-                                     lexer_value=buff,
-                                     message="Unknown token '%s'" % buff)
+                    yield {'type': 'divide', 'value': '/',
+                        'start': self._position, 'end': self._position + 1}
             elif self._current == '"':
                 yield self._consume_quoted_identifier()
             elif self._current == '<':
@@ -117,6 +136,13 @@ def _consume_number(self):
             buff += self._current
         return buff
 
+    def _peek_is_next_digit(self):
+        if (self._position == self._length - 1):
+            return False
+        else:
+            next = self._chars[self._position + 1]
+            return next in self.VALID_NUMBER
+
     def _initialize_for_expression(self, expression):
         if not expression:
             raise EmptyExpressionError()