Skip to content

Commit 685ad02

Browse files
pythongh-155006: Encode an error handler's replacement strictly (pythonGH-155008)
Encoding it with the same error handler could never terminate: a replacement that is itself unencodable calls the handler again. A replacement that does not fit is now reported against the input character. Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
1 parent 04a1fcf commit 685ad02

2 files changed

Lines changed: 22 additions & 2 deletions

File tree

Lib/test/test_codecs.py

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import _codecs
12
import codecs
23
import contextlib
34
import copy
@@ -3735,6 +3736,17 @@ def test_encode_errors(self):
37353736
self.assertEqual(codecs.iconv_encode(enc, 'a€b', 'xmlcharrefreplace')[0],
37363737
b'a&#8364;b')
37373738

3739+
def test_encode_errors_unencodable_replacement(self):
3740+
# Encoding the replacement must not call the error handler again.
3741+
enc = self.require('ASCII')
3742+
codecs.register_error('test.iconv', lambda exc: ('€', exc.end))
3743+
self.addCleanup(_codecs._unregister_error, 'test.iconv')
3744+
with self.assertRaises(UnicodeEncodeError) as cm:
3745+
codecs.iconv_encode(enc, 'a€b', 'test.iconv')
3746+
self.assertEqual((cm.exception.start, cm.exception.end), (1, 2))
3747+
self.assertEqual(cm.exception.reason,
3748+
'unable to encode error handler result')
3749+
37383750
def test_decode_errors(self):
37393751
enc = self.require('ASCII')
37403752
bad = b'a\xffb'

Objects/unicodeobject.c

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8520,11 +8520,19 @@ _PyUnicode_EncodeIconv(const char *encoding, PyObject *unicode,
85208520
replen = PyBytes_GET_SIZE(rep);
85218521
}
85228522
else {
8523-
/* A str replacement is encoded through the same codec. */
8523+
/* A str replacement is encoded through the same codec, but
8524+
strictly: handling its errors in turn could never terminate. */
85248525
assert(PyUnicode_Check(rep));
8525-
repbytes = _PyUnicode_EncodeIconv(encoding, rep, errors);
8526+
repbytes = _PyUnicode_EncodeIconv(encoding, rep, NULL);
85268527
Py_DECREF(rep);
85278528
if (repbytes == NULL) {
8529+
if (PyErr_ExceptionMatches(PyExc_UnicodeEncodeError)) {
8530+
/* Report the input the caller knows about, not the
8531+
replacement. */
8532+
PyErr_Clear();
8533+
raise_encode_exception(&exc, encoding, unicode, pos, pos + 1,
8534+
"unable to encode error handler result");
8535+
}
85288536
goto done;
85298537
}
85308538
repdata = PyBytes_AS_STRING(repbytes);

0 commit comments

Comments
 (0)