JEP12-compatibility is now a flag

Author: springcomp

jmespath.test

@@ -4,9 +4,9 @@
 __version__ = '1.1.0rc1'
 
 
-def compile(expression):
-    return parser.Parser().parse(expression)
+def compile(expression, options=None):
+    return parser.Parser().parse(expression, options=options)
 
 
 def search(expression, data, options=None):
-    return parser.Parser().parse(expression).search(data, options=options)
+    return compile(expression, options).search(data, options=options)

jmespath/__init__.py

@@ -2,6 +2,7 @@
 import warnings
 from json import loads
 
+from jmespath.visitor import Options
 from jmespath.exceptions import LexerError, EmptyExpressionError
 
 
@@ -29,7 +30,14 @@ class Lexer(object):
         u'\u00f7': 'divide',
     }
 
-    def tokenize(self, expression):
+    def __init__(self):
+        self._enable_legacy_literals = False
+
+    def tokenize(self, expression, options=None):
+        if (options is not None):
+            self._enable_legacy_literals= \
+                options.enable_legacy_literals
+
         self._initialize_for_expression(expression)
         while self._current is not None:
             if self._current in self.SIMPLE_TOKENS:
@@ -184,21 +192,30 @@ def _consume_until(self, delimiter):
 
     def _consume_literal(self):
         start = self._position
-        lexeme = self._consume_until('`').replace('\\`', '`')
+        token = self._consume_until('`')
+        lexeme = token.replace('\\`', '`')
+        parsed_json = None
         try:
             # Assume it is valid JSON and attempt to parse.
             parsed_json = loads(lexeme)
         except ValueError:
+            error = LexerError(lexer_position=start,
+                        lexer_value=self._expression[start:],
+                        message="Bad token %s `{}`".format(token))
+
+            if not self._enable_legacy_literals:
+                raise error
+                return
+
             try:
                 # Invalid JSON values should be converted to quoted
                 # JSON strings during the JEP-12 deprecation period.
                 parsed_json = loads('"%s"' % lexeme.lstrip())
                 warnings.warn("deprecated string literal syntax",
                               PendingDeprecationWarning)
             except ValueError:
-                raise LexerError(lexer_position=start,
-                                 lexer_value=self._expression[start:],
-                                 message="Bad token %s" % lexeme)
+                raise error
+
         token_len = self._position - start
         return {'type': 'literal', 'value': parsed_json,
                 'start': start, 'end': token_len}

jmespath/lexer.py

@@ -89,19 +89,19 @@ def __init__(self, lookahead=2):
         self._buffer_size = lookahead
         self._index = 0
 
-    def parse(self, expression):
+    def parse(self, expression, options=None):
         cached = self._CACHE.get(expression)
         if cached is not None:
             return cached
-        parsed_result = self._do_parse(expression)
+        parsed_result = self._do_parse(expression, options)
         self._CACHE[expression] = parsed_result
         if len(self._CACHE) > self._MAX_SIZE:
             self._free_cache_entries()
         return parsed_result
 
-    def _do_parse(self, expression):
+    def _do_parse(self, expression, options=None):
         try:
-            return self._parse(expression)
+            return self._parse(expression, options)
         except exceptions.LexerError as e:
             e.expression = expression
             raise
@@ -112,8 +112,8 @@ def _do_parse(self, expression):
             e.expression = expression
             raise
 
-    def _parse(self, expression):
-        self.tokenizer = lexer.Lexer().tokenize(expression)
+    def _parse(self, expression, options=None):
+        self.tokenizer = lexer.Lexer().tokenize(expression, options)
         self._tokens = list(self.tokenizer)
         self._index = 0
         parsed = self._expression(binding_power=0)

jmespath/parser.py

@@ -59,7 +59,10 @@ def _is_actual_number(x):
 
 class Options(object):
     """Options to control how a JMESPath function is evaluated."""
-    def __init__(self, dict_cls=None, custom_functions=None):
+    def __init__(self, dict_cls=None,
+        custom_functions=None,
+        enable_legacy_literals=False):
+
         #: The class to use when creating a dict.  The interpreter
         #  may create dictionaries during the evaluation of a JMESPath
         #  expression.  For example, a multi-select hash will
@@ -71,6 +74,12 @@ def __init__(self, dict_cls=None, custom_functions=None):
         self.dict_cls = dict_cls
         self.custom_functions = custom_functions
 
+        #: The flag to enable pre-JEP-12 literal compatibility.
+        #  JEP-12 deprecates `foo` -> "foo" syntax.
+        #  Valid expressions MUST use: `"foo"` -> "foo"
+        #  Setting this flag to `True` enables support for legacy syntax.
+        self.enable_legacy_literals = enable_legacy_literals
+
 
 class _Expression(object):
     def __init__(self, expression, interpreter, context):

jmespath/visitor.py

@@ -1 +1,2 @@
+legacy/
 *\.json

tests/compliance/.gitignore

@@ -0,0 +1 @@
+import unittest

tests/legacy/__init__.py

@@ -0,0 +1,44 @@
+from tests.legacy import unittest
+from tests import test_lexer
+
+from jmespath import lexer
+from jmespath.visitor import Options
+from jmespath.exceptions import LexerError
+
+
+class TestLegacyRegexLexer(test_lexer.LexerUtils):
+
+    def setUp(self):
+        self.lexer = lexer.Lexer()
+        self.options = Options(enable_legacy_literals=True)
+
+    def tokenize(self, expression):
+        return self.lexer.tokenize(expression, self.options)
+
+    def test_literal_string(self):
+        tokens = list(self.tokenize('`foobar`'))
+        self.assert_tokens(tokens, [
+            {'type': 'literal', 'value': "foobar"},
+        ])
+
+    def test_literal_with_invalid_json(self):
+        with self.assertRaises(LexerError):
+            list(self.tokenize('`foo"bar`'))
+
+    def test_literal_with_empty_string(self):
+        tokens = list(self.tokenize('``'))
+        self.assert_tokens(tokens, [{'type': 'literal', 'value': ''}])
+
+    def test_adds_quotes_when_invalid_json(self):
+        tokens = list(self.tokenize('`{{}`'))
+        self.assertEqual(
+            tokens,
+            [{'type': 'literal', 'value': '{{}',
+              'start': 0, 'end': 4},
+             {'type': 'eof', 'value': '',
+              'start': 5, 'end': 5}
+             ]
+        )
+
+if __name__ == '__main__':
+    unittest.main()

tests/legacy/literal.json

@@ -12,7 +12,8 @@
 COMPLIANCE_DIR = os.path.join(TEST_DIR, 'compliance')
 LEGACY_DIR = os.path.join(TEST_DIR, 'legacy')
 NOT_SPECIFIED = object()
-OPTIONS = Options(dict_cls=OrderedDict)
+COMPLIANCE_OPTIONS = Options(dict_cls=OrderedDict)
+LEGACY_OPTIONS = Options(dict_cls=OrderedDict, enable_legacy_literals=True)
 
 
 def _compliance_tests(requested_test_type):
@@ -69,16 +70,22 @@ def load_cases(full_path):
 )
 def test_expression(given, expression, expected, filename):
     import jmespath.parser
+
+    options = COMPLIANCE_OPTIONS \
+        if filename.find('legacy_') == -1 \
+        else LEGACY_OPTIONS
+
     try:
-        parsed = jmespath.compile(expression)
+        parsed = jmespath.compile(expression, options)
     except ValueError as e:
         raise AssertionError(
             'jmespath expression failed to compile: "%s", error: %s"' %
             (expression, e))
-    actual = parsed.search(given, options=OPTIONS)
+
+    actual = parsed.search(given, options=options)
     expected_repr = json.dumps(expected, indent=4)
     actual_repr = json.dumps(actual, indent=4)
-    error_msg = ("\n\n  (%s) The expression '%s' was suppose to give:\n%s\n"
+    error_msg = ("\n\n  (%s) The expression '%s' was supposed to give:\n%s\n"
                  "Instead it matched:\n%s\nparsed as:\n%s\ngiven:\n%s" % (
                      filename, expression, expected_repr,
                      actual_repr, pformat(parsed.parsed),