Use RecursionError to check for circular objects

Other fast JSON encoders rely on RecursionError to detect circular
references:

- ultrajson -- OverflowError: Maximum recursion level reached
- orjson -- TypeError: Recursion limit reached

Python's json module kept a dictionary with visited objects. It is too
much work for a nicer error message. Instead raise a ValueError on
RecursionError to keep the API but do not track the objects.

Author: aivarsk

Lib/json/encoder.py

@@ -194,12 +194,23 @@ def encode(self, o):
                 return encode_basestring_ascii(o)
             else:
                 return encode_basestring(o)
-        # This doesn't pass the iterator directly to ''.join() because the
-        # exceptions aren't as detailed.  The list call should be roughly
-        # equivalent to the PySequence_Fast that ''.join() would do.
-        chunks = self.iterencode(o, _one_shot=True)
-        if not isinstance(chunks, (list, tuple)):
-            chunks = list(chunks)
+
+        # There are tests for bad bool args
+        bool(self.check_circular)
+        try:
+            # This doesn't pass the iterator directly to ''.join() because the
+            # exceptions aren't as detailed.  The list call should be roughly
+            # equivalent to the PySequence_Fast that ''.join() would do.
+            chunks = self.iterencode(o, _one_shot=True)
+            if not isinstance(chunks, (list, tuple)):
+                chunks = list(chunks)
+        except RecursionError as exc:
+            if self.check_circular:
+                err = ValueError("Circular reference detected")
+                if notes := getattr(exc, "__notes__", None):
+                    err.__notes__ = notes
+                raise err
+            raise
         return ''.join(chunks)
 
     def iterencode(self, o, _one_shot=False):
@@ -212,10 +223,7 @@ def iterencode(self, o, _one_shot=False):
                 mysocket.write(chunk)
 
         """
-        if self.check_circular:
-            markers = {}
-        else:
-            markers = None
+        markers = None
         if self.ensure_ascii:
             _encoder = encode_basestring_ascii
         else:
@@ -279,11 +287,6 @@ def _iterencode_list(lst, _current_indent_level):
         if not lst:
             yield '[]'
             return
-        if markers is not None:
-            markerid = id(lst)
-            if markerid in markers:
-                raise ValueError("Circular reference detected")
-            markers[markerid] = lst
         buf = '['
         if _indent is not None:
             _current_indent_level += 1
@@ -331,18 +334,11 @@ def _iterencode_list(lst, _current_indent_level):
             _current_indent_level -= 1
             yield '\n' + _indent * _current_indent_level
         yield ']'
-        if markers is not None:
-            del markers[markerid]
 
     def _iterencode_dict(dct, _current_indent_level):
         if not dct:
             yield '{}'
             return
-        if markers is not None:
-            markerid = id(dct)
-            if markerid in markers:
-                raise ValueError("Circular reference detected")
-            markers[markerid] = dct
         yield '{'
         if _indent is not None:
             _current_indent_level += 1
@@ -417,8 +413,6 @@ def _iterencode_dict(dct, _current_indent_level):
             _current_indent_level -= 1
             yield '\n' + _indent * _current_indent_level
         yield '}'
-        if markers is not None:
-            del markers[markerid]
 
     def _iterencode(o, _current_indent_level):
         if isinstance(o, str):
@@ -440,11 +434,6 @@ def _iterencode(o, _current_indent_level):
         elif isinstance(o, dict):
             yield from _iterencode_dict(o, _current_indent_level)
         else:
-            if markers is not None:
-                markerid = id(o)
-                if markerid in markers:
-                    raise ValueError("Circular reference detected")
-                markers[markerid] = o
             newobj = _default(o)
             try:
                 yield from _iterencode(newobj, _current_indent_level)
@@ -453,6 +442,4 @@ def _iterencode(o, _current_indent_level):
             except BaseException as exc:
                 exc.add_note(f'when serializing {type(o).__name__} object')
                 raise
-            if markers is not None:
-                del markers[markerid]
     return _iterencode

Lib/test/test_json/test_recursion.py

@@ -13,7 +13,7 @@ def test_listrecursion(self):
         try:
             self.dumps(x)
         except ValueError as exc:
-            self.assertEqual(exc.__notes__, ["when serializing list item 0"])
+            self.assertEqual(exc.__notes__[:1], ["when serializing list item 0"])
         else:
             self.fail("didn't raise ValueError on list recursion")
         x = []
@@ -22,7 +22,7 @@ def test_listrecursion(self):
         try:
             self.dumps(x)
         except ValueError as exc:
-            self.assertEqual(exc.__notes__, ["when serializing list item 0"]*2)
+            self.assertEqual(exc.__notes__[:2], ["when serializing list item 0"]*2)
         else:
             self.fail("didn't raise ValueError on alternating list recursion")
         y = []
@@ -36,7 +36,7 @@ def test_dictrecursion(self):
         try:
             self.dumps(x)
         except ValueError as exc:
-            self.assertEqual(exc.__notes__, ["when serializing dict item 'test'"])
+            self.assertEqual(exc.__notes__[:1], ["when serializing dict item 'test'"])
         else:
             self.fail("didn't raise ValueError on dict recursion")
         x = {}
@@ -61,7 +61,7 @@ def default(self, o):
         try:
             enc.encode(JSONTestObject)
         except ValueError as exc:
-            self.assertEqual(exc.__notes__,
+            self.assertEqual(exc.__notes__[:2],
                              ["when serializing list item 0",
                               "when serializing type object"])
         else:
@@ -94,10 +94,10 @@ def test_highly_nested_objects_encoding(self):
             l, d = [l], {'k':d}
         with self.assertRaises(RecursionError):
             with support.infinite_recursion(5000):
-                self.dumps(l)
+                self.dumps(l, check_circular=False)
         with self.assertRaises(RecursionError):
             with support.infinite_recursion(5000):
-                self.dumps(d)
+                self.dumps(d, check_circular=False)
 
     @support.skip_emscripten_stack_overflow()
     @support.skip_wasi_stack_overflow()

Modules/_json.c

@@ -1242,7 +1242,6 @@ encoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
     if (s == NULL)
         return NULL;
 
-    s->markers = Py_NewRef(markers);
     s->defaultfn = Py_NewRef(defaultfn);
     s->encoder = Py_NewRef(encoder);
     s->indent = Py_NewRef(indent);
@@ -1521,33 +1520,13 @@ encoder_listencode_obj(PyEncoderObject *s, PyUnicodeWriter *writer,
         return rv;
     }
     else {
-        PyObject *ident = NULL;
-        if (s->markers != Py_None) {
-            int has_key;
-            ident = PyLong_FromVoidPtr(obj);
-            if (ident == NULL)
-                return -1;
-            has_key = PyDict_Contains(s->markers, ident);
-            if (has_key) {
-                if (has_key != -1)
-                    PyErr_SetString(PyExc_ValueError, "Circular reference detected");
-                Py_DECREF(ident);
-                return -1;
-            }
-            if (PyDict_SetItem(s->markers, ident, obj)) {
-                Py_DECREF(ident);
-                return -1;
-            }
-        }
         newobj = PyObject_CallOneArg(s->defaultfn, obj);
         if (newobj == NULL) {
-            Py_XDECREF(ident);
             return -1;
         }
 
         if (_Py_EnterRecursiveCall(" while encoding a JSON object")) {
             Py_DECREF(newobj);
-            Py_XDECREF(ident);
             return -1;
         }
         rv = encoder_listencode_obj(s, writer, newobj, indent_level, indent_cache);
@@ -1556,16 +1535,8 @@ encoder_listencode_obj(PyEncoderObject *s, PyUnicodeWriter *writer,
         Py_DECREF(newobj);
         if (rv) {
             _PyErr_FormatNote("when serializing %T object", obj);
-            Py_XDECREF(ident);
             return -1;
         }
-        if (ident != NULL) {
-            if (PyDict_DelItem(s->markers, ident)) {
-                Py_XDECREF(ident);
-                return -1;
-            }
-            Py_XDECREF(ident);
-        }
         return rv;
     }
 }
@@ -1642,7 +1613,6 @@ encoder_listencode_dict(PyEncoderObject *s, PyUnicodeWriter *writer,
                        Py_ssize_t indent_level, PyObject *indent_cache)
 {
     /* Encode Python dict dct a JSON term */
-    PyObject *ident = NULL;
     PyObject *items = NULL;
     PyObject *key, *value;
     bool first = true;
@@ -1652,22 +1622,6 @@ encoder_listencode_dict(PyEncoderObject *s, PyUnicodeWriter *writer,
         return PyUnicodeWriter_WriteASCII(writer, "{}", 2);
     }
 
-    if (s->markers != Py_None) {
-        int has_key;
-        ident = PyLong_FromVoidPtr(dct);
-        if (ident == NULL)
-            goto bail;
-        has_key = PyDict_Contains(s->markers, ident);
-        if (has_key) {
-            if (has_key != -1)
-                PyErr_SetString(PyExc_ValueError, "Circular reference detected");
-            goto bail;
-        }
-        if (PyDict_SetItem(s->markers, ident, dct)) {
-            goto bail;
-        }
-    }
-
     if (PyUnicodeWriter_WriteChar(writer, '{')) {
         goto bail;
     }
@@ -1715,11 +1669,6 @@ encoder_listencode_dict(PyEncoderObject *s, PyUnicodeWriter *writer,
         }
     }
 
-    if (ident != NULL) {
-        if (PyDict_DelItem(s->markers, ident))
-            goto bail;
-        Py_CLEAR(ident);
-    }
     if (s->indent != Py_None) {
         indent_level--;
         if (write_newline_indent(writer, indent_level, indent_cache) < 0) {
@@ -1734,7 +1683,6 @@ encoder_listencode_dict(PyEncoderObject *s, PyUnicodeWriter *writer,
 
 bail:
     Py_XDECREF(items);
-    Py_XDECREF(ident);
     return -1;
 }
 
@@ -1743,11 +1691,9 @@ encoder_listencode_list(PyEncoderObject *s, PyUnicodeWriter *writer,
                         PyObject *seq,
                         Py_ssize_t indent_level, PyObject *indent_cache)
 {
-    PyObject *ident = NULL;
     PyObject *s_fast = NULL;
     Py_ssize_t i;
 
-    ident = NULL;
     s_fast = PySequence_Fast(seq, "_iterencode_list needs a sequence");
     if (s_fast == NULL)
         return -1;
@@ -1756,22 +1702,6 @@ encoder_listencode_list(PyEncoderObject *s, PyUnicodeWriter *writer,
         return PyUnicodeWriter_WriteASCII(writer, "[]", 2);
     }
 
-    if (s->markers != Py_None) {
-        int has_key;
-        ident = PyLong_FromVoidPtr(seq);
-        if (ident == NULL)
-            goto bail;
-        has_key = PyDict_Contains(s->markers, ident);
-        if (has_key) {
-            if (has_key != -1)
-                PyErr_SetString(PyExc_ValueError, "Circular reference detected");
-            goto bail;
-        }
-        if (PyDict_SetItem(s->markers, ident, seq)) {
-            goto bail;
-        }
-    }
-
     if (PyUnicodeWriter_WriteChar(writer, '[')) {
         goto bail;
     }
@@ -1797,11 +1727,6 @@ encoder_listencode_list(PyEncoderObject *s, PyUnicodeWriter *writer,
             goto bail;
         }
     }
-    if (ident != NULL) {
-        if (PyDict_DelItem(s->markers, ident))
-            goto bail;
-        Py_CLEAR(ident);
-    }
 
     if (s->indent != Py_None) {
         indent_level--;
@@ -1817,7 +1742,6 @@ encoder_listencode_list(PyEncoderObject *s, PyUnicodeWriter *writer,
     return 0;
 
 bail:
-    Py_XDECREF(ident);
     Py_DECREF(s_fast);
     return -1;
 }
@@ -1838,7 +1762,6 @@ encoder_traverse(PyObject *op, visitproc visit, void *arg)
 {
     PyEncoderObject *self = PyEncoderObject_CAST(op);
     Py_VISIT(Py_TYPE(self));
-    Py_VISIT(self->markers);
     Py_VISIT(self->defaultfn);
     Py_VISIT(self->encoder);
     Py_VISIT(self->indent);
@@ -1852,7 +1775,6 @@ encoder_clear(PyObject *op)
 {
     PyEncoderObject *self = PyEncoderObject_CAST(op);
     /* Deallocate Encoder */
-    Py_CLEAR(self->markers);
     Py_CLEAR(self->defaultfn);
     Py_CLEAR(self->encoder);
     Py_CLEAR(self->indent);