Fix lexer error values

Updated the hypothesis tests to check additional
properties of lexer errors being raised.
This caught a few more issues that I've fixed here,
primarily related to the lexer position and the token
value that triggered the error.

Author: jamesls

extra/test_hypothesis.py

@@ -40,11 +40,31 @@
 def test_lexer_api(expr):
     try:
         tokens = list(lexer.Lexer().tokenize(expr))
-    except exceptions.JMESPathError as e:
+    except exceptions.EmptyExpressionError:
+        return
+    except exceptions.LexerError as e:
+        assert e.lex_position >= 0, e.lex_position
+        assert e.lex_position < len(expr), e.lex_position
+        if expr:
+            assert expr[e.lex_position] == e.token_value[0], (
+                "Lex position does not match first token char.\n"
+                "Expression: %s\n%s != %s" % (expr, expr[e.lex_position],
+                                              e.token_value[0])
+            )
         return
     except Exception as e:
         raise AssertionError("Non JMESPathError raised: %s" % e)
     assert isinstance(tokens, list)
+    # Token starting positions must be unique, can't have two
+    # tokens with the same start position.
+    start_locations = [t['start'] for t in tokens]
+    assert len(set(start_locations)) == len(start_locations), (
+        "Tokens must have unique starting locations.")
+    # Starting positions must be increasing (i.e sorted).
+    assert sorted(start_locations) == start_locations, (
+        "Tokens must have increasing start locations.")
+    # Last token is always EOF.
+    assert tokens[-1]['type'] == 'eof'
 
 
 @settings(**BASE_SETTINGS)

jmespath/lexer.py

@@ -92,10 +92,17 @@ def tokenize(self, expression):
                         'start': self._position - 1, 'end': self._position}
                     self._next()
                 else:
+                    if self._current is None:
+                        # If we're at the EOF, we never advanced
+                        # the position so we don't need to rewind
+                        # it back one location.
+                        position = self._position
+                    else:
+                        position = self._position - 1
                     raise LexerError(
-                        lexer_position=self._position - 1,
+                        lexer_position=position,
                         lexer_value='=',
-                        message="Unknown token =")
+                        message="Unknown token '='")
             else:
                 raise LexerError(lexer_position=self._position,
                                  lexer_value=self._current,
@@ -138,8 +145,9 @@ def _consume_until(self, delimiter):
                 buff += '\\'
                 self._next()
             if self._current is None:
+                # We're at the EOF.
                 raise LexerError(lexer_position=start,
-                                 lexer_value=self._expression,
+                                 lexer_value=self._expression[start:],
                                  message="Unclosed %s delimiter" % delimiter)
             buff += self._current
             self._next()
@@ -162,7 +170,7 @@ def _consume_literal(self):
                               PendingDeprecationWarning)
             except ValueError:
                 raise LexerError(lexer_position=start,
-                                 lexer_value=self._expression,
+                                 lexer_value=self._expression[start:],
                                  message="Bad token %s" % lexeme)
         token_len = self._position - start
         return {'type': 'literal', 'value': parsed_json,