Skip to content

[mypyc] feat: cache ids for fallback pythonic method lookups [1/1] #19870

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 7 commits into
base: master
Choose a base branch
from
Open

[mypyc] feat: cache ids for fallback pythonic method lookups [1/1] #19870

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
f77f297
[mypyc] feat: cache ids for fallback pythonic method lookups
BobTheBuidler Sep 17, 2025
1f3d409
fix declared in 2 places
BobTheBuidler Sep 17, 2025
fd66205
fix linker
BobTheBuidler Sep 17, 2025
d574135
fix segfault
BobTheBuidler Sep 17, 2025
3267aea
Merge branch 'master' into cache-ids
BobTheBuidler Sep 30, 2025
83e35de
Merge branch 'master' into cache-ids
BobTheBuidler Oct 2, 2025
22af2bc
Merge branch 'master' into cache-ids
BobTheBuidler Oct 4, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 9 additions & 5 deletions mypyc/lib-rt/bytes_ops.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,18 +95,22 @@ PyObject *CPyBytes_GetSlice(PyObject *obj, CPyTagged start, CPyTagged end) {
return CPyObject_GetSlice(obj, start, end);
}

static PyObject *join_id_unicode = NULL;

// Like _PyBytes_Join but fallback to dynamic call if 'sep' is not bytes
// (mostly commonly, for bytearrays)
PyObject *CPyBytes_Join(PyObject *sep, PyObject *iter) {
if (PyBytes_CheckExact(sep)) {
return PyBytes_Join(sep, iter);
} else {
_Py_IDENTIFIER(join);
PyObject *name = _PyUnicode_FromId(&PyId_join); /* borrowed */
if (name == NULL) {
return NULL;
if (join_id_unicode == NULL) {
_Py_IDENTIFIER(join);
join_id_unicode = _PyUnicode_FromId(&PyId_join); /* borrowed */
if (join_id_unicode == NULL) {
return NULL;
}
}
return PyObject_CallMethodOneArg(sep, name, iter);
return PyObject_CallMethodOneArg(sep, join_id_unicode, iter);
}
}

Expand Down
154 changes: 94 additions & 60 deletions mypyc/lib-rt/dict_ops.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,16 @@
#define Py_TPFLAGS_MAPPING (1 << 6)
#endif

// Caching _PyUnicode_FromId results for identifiers
static PyObject *setdefault_id_unicode = NULL;
static PyObject *update_id_unicode = NULL;
static PyObject *keys_id_unicode = NULL;
static PyObject *values_id_unicode = NULL;
static PyObject *items_id_unicode = NULL;
// dict_ prefix prevents a name conflict with list_ops
static PyObject *dict_copy_id_unicode = NULL;
static PyObject *dict_clear_id_unicode = NULL;

// Dict subclasses like defaultdict override things in interesting
// ways, so we don't want to just directly use the dict methods. Not
// sure if it is actually worth doing all this stuff, but it saves
Expand Down Expand Up @@ -77,12 +87,14 @@ PyObject *CPyDict_SetDefault(PyObject *dict, PyObject *key, PyObject *value) {
Py_XINCREF(ret);
return ret;
}
_Py_IDENTIFIER(setdefault);
PyObject *name = _PyUnicode_FromId(&PyId_setdefault); /* borrowed */
if (name == NULL) {
return NULL;
if (setdefault_id_unicode == NULL) {
_Py_IDENTIFIER(setdefault);
setdefault_id_unicode = _PyUnicode_FromId(&PyId_setdefault); /* borrowed */
if (setdefault_id_unicode == NULL) {
return NULL;
}
}
return PyObject_CallMethodObjArgs(dict, name, key, value, NULL);
return PyObject_CallMethodObjArgs(dict, setdefault_id_unicode, key, value, NULL);
}

PyObject *CPyDict_SetDefaultWithNone(PyObject *dict, PyObject *key) {
Expand Down Expand Up @@ -136,12 +148,14 @@ static inline int CPy_ObjectToStatus(PyObject *obj) {
}

static int CPyDict_UpdateGeneral(PyObject *dict, PyObject *stuff) {
_Py_IDENTIFIER(update);
PyObject *name = _PyUnicode_FromId(&PyId_update); /* borrowed */
if (name == NULL) {
return -1;
if (update_id_unicode == NULL) {
_Py_IDENTIFIER(update);
update_id_unicode = _PyUnicode_FromId(&PyId_update); /* borrowed */
if (update_id_unicode == NULL) {
return -1;
}
}
PyObject *res = PyObject_CallMethodOneArg(dict, name, stuff);
PyObject *res = PyObject_CallMethodOneArg(dict, update_id_unicode, stuff);
return CPy_ObjectToStatus(res);
}

Expand Down Expand Up @@ -207,36 +221,42 @@ PyObject *CPyDict_KeysView(PyObject *dict) {
if (PyDict_CheckExact(dict)){
return _CPyDictView_New(dict, &PyDictKeys_Type);
}
_Py_IDENTIFIER(keys);
PyObject *name = _PyUnicode_FromId(&PyId_keys); /* borrowed */
if (name == NULL) {
return NULL;
if (keys_id_unicode == NULL) {
_Py_IDENTIFIER(keys);
keys_id_unicode = _PyUnicode_FromId(&PyId_keys); /* borrowed */
if (keys_id_unicode == NULL) {
return NULL;
}
}
return PyObject_CallMethodNoArgs(dict, name);
return PyObject_CallMethodNoArgs(dict, keys_id_unicode);
}

PyObject *CPyDict_ValuesView(PyObject *dict) {
if (PyDict_CheckExact(dict)){
return _CPyDictView_New(dict, &PyDictValues_Type);
}
_Py_IDENTIFIER(values);
PyObject *name = _PyUnicode_FromId(&PyId_values); /* borrowed */
if (name == NULL) {
return NULL;
if (values_id_unicode == NULL) {
_Py_IDENTIFIER(values);
values_id_unicode = _PyUnicode_FromId(&PyId_values); /* borrowed */
if (values_id_unicode == NULL) {
return NULL;
}
}
return PyObject_CallMethodNoArgs(dict, name);
return PyObject_CallMethodNoArgs(dict, values_id_unicode);
}

PyObject *CPyDict_ItemsView(PyObject *dict) {
if (PyDict_CheckExact(dict)){
return _CPyDictView_New(dict, &PyDictItems_Type);
}
_Py_IDENTIFIER(items);
PyObject *name = _PyUnicode_FromId(&PyId_items); /* borrowed */
if (name == NULL) {
return NULL;
if (items_id_unicode == NULL) {
_Py_IDENTIFIER(items);
items_id_unicode = _PyUnicode_FromId(&PyId_items); /* borrowed */
if (items_id_unicode == NULL) {
return NULL;
}
}
return PyObject_CallMethodNoArgs(dict, name);
return PyObject_CallMethodNoArgs(dict, items_id_unicode);
}

PyObject *CPyDict_Keys(PyObject *dict) {
Expand All @@ -245,12 +265,14 @@ PyObject *CPyDict_Keys(PyObject *dict) {
}
// Inline generic fallback logic to also return a list.
PyObject *list = PyList_New(0);
_Py_IDENTIFIER(keys);
PyObject *name = _PyUnicode_FromId(&PyId_keys); /* borrowed */
if (name == NULL) {
return NULL;
if (keys_id_unicode == NULL) {
_Py_IDENTIFIER(keys);
keys_id_unicode = _PyUnicode_FromId(&PyId_keys); /* borrowed */
if (keys_id_unicode == NULL) {
return NULL;
}
}
PyObject *view = PyObject_CallMethodNoArgs(dict, name);
PyObject *view = PyObject_CallMethodNoArgs(dict, keys_id_unicode);
if (view == NULL) {
return NULL;
}
Expand All @@ -268,12 +290,14 @@ PyObject *CPyDict_Values(PyObject *dict) {
}
// Inline generic fallback logic to also return a list.
PyObject *list = PyList_New(0);
_Py_IDENTIFIER(values);
PyObject *name = _PyUnicode_FromId(&PyId_values); /* borrowed */
if (name == NULL) {
return NULL;
if (values_id_unicode == NULL) {
_Py_IDENTIFIER(values);
values_id_unicode = _PyUnicode_FromId(&PyId_values); /* borrowed */
if (values_id_unicode == NULL) {
return NULL;
}
}
PyObject *view = PyObject_CallMethodNoArgs(dict, name);
PyObject *view = PyObject_CallMethodNoArgs(dict, values_id_unicode);
if (view == NULL) {
return NULL;
}
Expand All @@ -291,12 +315,14 @@ PyObject *CPyDict_Items(PyObject *dict) {
}
// Inline generic fallback logic to also return a list.
PyObject *list = PyList_New(0);
_Py_IDENTIFIER(items);
PyObject *name = _PyUnicode_FromId(&PyId_items); /* borrowed */
if (name == NULL) {
return NULL;
if (items_id_unicode == NULL) {
_Py_IDENTIFIER(items);
items_id_unicode = _PyUnicode_FromId(&PyId_items); /* borrowed */
if (items_id_unicode == NULL) {
return NULL;
}
}
PyObject *view = PyObject_CallMethodNoArgs(dict, name);
PyObject *view = PyObject_CallMethodNoArgs(dict, items_id_unicode);
if (view == NULL) {
return NULL;
}
Expand All @@ -312,12 +338,14 @@ char CPyDict_Clear(PyObject *dict) {
if (PyDict_CheckExact(dict)) {
PyDict_Clear(dict);
} else {
_Py_IDENTIFIER(clear);
PyObject *name = _PyUnicode_FromId(&PyId_clear); /* borrowed */
if (name == NULL) {
return 0;
if (dict_clear_id_unicode == NULL) {
_Py_IDENTIFIER(clear);
dict_clear_id_unicode = _PyUnicode_FromId(&PyId_clear); /* borrowed */
if (dict_clear_id_unicode == NULL) {
return 0;
}
}
PyObject *res = PyObject_CallMethodNoArgs(dict, name);
PyObject *res = PyObject_CallMethodNoArgs(dict, dict_clear_id_unicode);
if (res == NULL) {
return 0;
}
Expand All @@ -329,12 +357,14 @@ PyObject *CPyDict_Copy(PyObject *dict) {
if (PyDict_CheckExact(dict)) {
return PyDict_Copy(dict);
}
_Py_IDENTIFIER(copy);
PyObject *name = _PyUnicode_FromId(&PyId_copy); /* borrowed */
if (name == NULL) {
return NULL;
if (dict_copy_id_unicode == NULL) {
_Py_IDENTIFIER(copy);
dict_copy_id_unicode = _PyUnicode_FromId(&PyId_copy); /* borrowed */
if (dict_copy_id_unicode == NULL) {
return NULL;
}
}
return PyObject_CallMethodNoArgs(dict, name);
return PyObject_CallMethodNoArgs(dict, dict_copy_id_unicode);
}

PyObject *CPyDict_GetKeysIter(PyObject *dict) {
Expand All @@ -352,12 +382,14 @@ PyObject *CPyDict_GetItemsIter(PyObject *dict) {
Py_INCREF(dict);
return dict;
}
_Py_IDENTIFIER(items);
PyObject *name = _PyUnicode_FromId(&PyId_items); /* borrowed */
if (name == NULL) {
return NULL;
if (items_id_unicode == NULL) {
_Py_IDENTIFIER(items);
items_id_unicode = _PyUnicode_FromId(&PyId_items); /* borrowed */
if (items_id_unicode == NULL) {
return NULL;
}
}
PyObject *view = PyObject_CallMethodNoArgs(dict, name);
PyObject *view = PyObject_CallMethodNoArgs(dict, items_id_unicode);
if (view == NULL) {
return NULL;
}
Expand All @@ -372,12 +404,14 @@ PyObject *CPyDict_GetValuesIter(PyObject *dict) {
Py_INCREF(dict);
return dict;
}
_Py_IDENTIFIER(values);
PyObject *name = _PyUnicode_FromId(&PyId_values); /* borrowed */
if (name == NULL) {
return NULL;
if (values_id_unicode == NULL) {
_Py_IDENTIFIER(values);
values_id_unicode = _PyUnicode_FromId(&PyId_values); /* borrowed */
if (values_id_unicode == NULL) {
return NULL;
}
}
PyObject *view = PyObject_CallMethodNoArgs(dict, name);
PyObject *view = PyObject_CallMethodNoArgs(dict, values_id_unicode);
if (view == NULL) {
return NULL;
}
Expand Down
29 changes: 18 additions & 11 deletions mypyc/lib-rt/list_ops.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,10 @@
#define Py_TPFLAGS_SEQUENCE (1 << 5)
#endif

// list_ prefix prevents a name conflict with dict_ops
static PyObject *list_clear_id_unicode = NULL;
static PyObject *list_copy_id_unicode = NULL;

PyObject *CPyList_Build(Py_ssize_t len, ...) {
Py_ssize_t i;

Expand All @@ -33,12 +37,14 @@ char CPyList_Clear(PyObject *list) {
if (PyList_CheckExact(list)) {
PyList_Clear(list);
} else {
_Py_IDENTIFIER(clear);
PyObject *name = _PyUnicode_FromId(&PyId_clear);
if (name == NULL) {
return 0;
if (list_clear_id_unicode == NULL) {
_Py_IDENTIFIER(clear);
list_clear_id_unicode = _PyUnicode_FromId(&PyId_clear);
if (list_clear_id_unicode == NULL) {
return 0;
}
}
PyObject *res = PyObject_CallMethodNoArgs(list, name);
PyObject *res = PyObject_CallMethodNoArgs(list, list_clear_id_unicode);
if (res == NULL) {
return 0;
}
Expand All @@ -50,13 +56,14 @@ PyObject *CPyList_Copy(PyObject *list) {
if(PyList_CheckExact(list)) {
return PyList_GetSlice(list, 0, PyList_GET_SIZE(list));
}
_Py_IDENTIFIER(copy);

PyObject *name = _PyUnicode_FromId(&PyId_copy);
if (name == NULL) {
return NULL;
if (list_copy_id_unicode == NULL) {
_Py_IDENTIFIER(copy);
list_copy_id_unicode = _PyUnicode_FromId(&PyId_copy);
if (list_copy_id_unicode == NULL) {
return NULL;
}
}
return PyObject_CallMethodNoArgs(list, name);
return PyObject_CallMethodNoArgs(list, list_copy_id_unicode);
}

PyObject *CPyList_GetItemShort(PyObject *list, CPyTagged index) {
Expand Down
27 changes: 19 additions & 8 deletions mypyc/lib-rt/misc_ops.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,19 +20,23 @@ PyObject *CPy_GetCoro(PyObject *obj)
}
}

static PyObject *send_id_unicode = NULL;

PyObject *CPyIter_Send(PyObject *iter, PyObject *val)
{
// Do a send, or a next if second arg is None.
// (This behavior is to match the PEP 380 spec for yield from.)
if (Py_IsNone(val)) {
return CPyIter_Next(iter);
} else {
_Py_IDENTIFIER(send);
PyObject *name = _PyUnicode_FromId(&PyId_send); /* borrowed */
if (name == NULL) {
return NULL;
if (send_id_unicode == NULL) {
_Py_IDENTIFIER(send);
send_id_unicode = _PyUnicode_FromId(&PyId_send); /* borrowed */
if (send_id_unicode == NULL) {
return NULL;
}
}
return PyObject_CallMethodOneArg(iter, name, val);
return PyObject_CallMethodOneArg(iter, send_id_unicode, val);
}
}

Expand Down Expand Up @@ -1058,14 +1062,21 @@ PyObject *CPy_GetANext(PyObject *aiter)

#if CPY_3_11_FEATURES

static PyObject *__name___id_unicode = NULL;

// Return obj.__name__ (specialized to type objects, which are the most common target).
PyObject *CPy_GetName(PyObject *obj) {
if (PyType_Check(obj)) {
return PyType_GetName((PyTypeObject *)obj);
}
_Py_IDENTIFIER(__name__);
PyObject *name = _PyUnicode_FromId(&PyId___name__); /* borrowed */
return PyObject_GetAttr(obj, name);
if (__name___id_unicode == NULL) {
_Py_IDENTIFIER(__name__);
__name___id_unicode = _PyUnicode_FromId(&PyId___name__); /* borrowed */
if (__name___id_unicode == NULL) {
return NULL;
}
}
return PyObject_GetAttr(obj, __name___id_unicode);
}

#endif
Expand Down