Skip to content

Commit 49ff8b6

Browse files
kumaraditya303kumaraditya303
authored
gh-140795: fetch thread state once on fast path for critical sections (#141406)
1 parent cb4c87d commit 49ff8b6

File tree

2 files changed

+65
-37
lines changed

2 files changed

+65
-37
lines changed

‎Include/internal/pycore_critical_section.h‎

Lines changed: 57 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ extern "C" {
3232
const bool _should_lock_cs = PyList_CheckExact(_orig_seq); \
3333
PyCriticalSection _cs; \
3434
if (_should_lock_cs) { \
35-
_PyCriticalSection_Begin(&_cs, _orig_seq); \
35+
PyCriticalSection_Begin(&_cs, _orig_seq); \
3636
}
3737

3838
# define Py_END_CRITICAL_SECTION_SEQUENCE_FAST() \
@@ -77,10 +77,10 @@ _PyCriticalSection_Resume(PyThreadState *tstate);
7777

7878
// (private) slow path for locking the mutex
7979
PyAPI_FUNC(void)
80-
_PyCriticalSection_BeginSlow(PyCriticalSection *c, PyMutex *m);
80+
_PyCriticalSection_BeginSlow(PyThreadState *tstate, PyCriticalSection *c, PyMutex *m);
8181

8282
PyAPI_FUNC(void)
83-
_PyCriticalSection2_BeginSlow(PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2,
83+
_PyCriticalSection2_BeginSlow(PyThreadState *tstate, PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2,
8484
int is_m1_locked);
8585

8686
PyAPI_FUNC(void)
@@ -95,34 +95,30 @@ _PyCriticalSection_IsActive(uintptr_t tag)
9595
}
9696

9797
static inline void
98-
_PyCriticalSection_BeginMutex(PyCriticalSection *c, PyMutex *m)
98+
_PyCriticalSection_BeginMutex(PyThreadState *tstate, PyCriticalSection *c, PyMutex *m)
9999
{
100100
if (PyMutex_LockFast(m)) {
101-
PyThreadState *tstate = _PyThreadState_GET();
102101
c->_cs_mutex = m;
103102
c->_cs_prev = tstate->critical_section;
104103
tstate->critical_section = (uintptr_t)c;
105104
}
106105
else {
107-
_PyCriticalSection_BeginSlow(c, m);
106+
_PyCriticalSection_BeginSlow(tstate, c, m);
108107
}
109108
}
110-
#define PyCriticalSection_BeginMutex _PyCriticalSection_BeginMutex
111109

112110
static inline void
113-
_PyCriticalSection_Begin(PyCriticalSection *c, PyObject *op)
111+
_PyCriticalSection_Begin(PyThreadState *tstate, PyCriticalSection *c, PyObject *op)
114112
{
115-
_PyCriticalSection_BeginMutex(c, &op->ob_mutex);
113+
_PyCriticalSection_BeginMutex(tstate, c, &op->ob_mutex);
116114
}
117-
#define PyCriticalSection_Begin _PyCriticalSection_Begin
118115

119116
// Removes the top-most critical section from the thread's stack of critical
120117
// sections. If the new top-most critical section is inactive, then it is
121118
// resumed.
122119
static inline void
123-
_PyCriticalSection_Pop(PyCriticalSection *c)
120+
_PyCriticalSection_Pop(PyThreadState *tstate, PyCriticalSection *c)
124121
{
125-
PyThreadState *tstate = _PyThreadState_GET();
126122
uintptr_t prev = c->_cs_prev;
127123
tstate->critical_section = prev;
128124

@@ -132,7 +128,7 @@ _PyCriticalSection_Pop(PyCriticalSection *c)
132128
}
133129

134130
static inline void
135-
_PyCriticalSection_End(PyCriticalSection *c)
131+
_PyCriticalSection_End(PyThreadState *tstate, PyCriticalSection *c)
136132
{
137133
// If the mutex is NULL, we used the fast path in
138134
// _PyCriticalSection_BeginSlow for locks already held in the top-most
@@ -141,18 +137,17 @@ _PyCriticalSection_End(PyCriticalSection *c)
141137
return;
142138
}
143139
PyMutex_Unlock(c->_cs_mutex);
144-
_PyCriticalSection_Pop(c);
140+
_PyCriticalSection_Pop(tstate, c);
145141
}
146-
#define PyCriticalSection_End _PyCriticalSection_End
147142

148143
static inline void
149-
_PyCriticalSection2_BeginMutex(PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2)
144+
_PyCriticalSection2_BeginMutex(PyThreadState *tstate, PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2)
150145
{
151146
if (m1 == m2) {
152147
// If the two mutex arguments are the same, treat this as a critical
153148
// section with a single mutex.
154149
c->_cs_mutex2 = NULL;
155-
_PyCriticalSection_BeginMutex(&c->_cs_base, m1);
150+
_PyCriticalSection_BeginMutex(tstate, &c->_cs_base, m1);
156151
return;
157152
}
158153

@@ -167,7 +162,6 @@ _PyCriticalSection2_BeginMutex(PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2)
167162

168163
if (PyMutex_LockFast(m1)) {
169164
if (PyMutex_LockFast(m2)) {
170-
PyThreadState *tstate = _PyThreadState_GET();
171165
c->_cs_base._cs_mutex = m1;
172166
c->_cs_mutex2 = m2;
173167
c->_cs_base._cs_prev = tstate->critical_section;
@@ -176,24 +170,22 @@ _PyCriticalSection2_BeginMutex(PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2)
176170
tstate->critical_section = p;
177171
}
178172
else {
179-
_PyCriticalSection2_BeginSlow(c, m1, m2, 1);
173+
_PyCriticalSection2_BeginSlow(tstate, c, m1, m2, 1);
180174
}
181175
}
182176
else {
183-
_PyCriticalSection2_BeginSlow(c, m1, m2, 0);
177+
_PyCriticalSection2_BeginSlow(tstate, c, m1, m2, 0);
184178
}
185179
}
186-
#define PyCriticalSection2_BeginMutex _PyCriticalSection2_BeginMutex
187180

188181
static inline void
189-
_PyCriticalSection2_Begin(PyCriticalSection2 *c, PyObject *a, PyObject *b)
182+
_PyCriticalSection2_Begin(PyThreadState *tstate, PyCriticalSection2 *c, PyObject *a, PyObject *b)
190183
{
191-
_PyCriticalSection2_BeginMutex(c, &a->ob_mutex, &b->ob_mutex);
184+
_PyCriticalSection2_BeginMutex(tstate, c, &a->ob_mutex, &b->ob_mutex);
192185
}
193-
#define PyCriticalSection2_Begin _PyCriticalSection2_Begin
194186

195187
static inline void
196-
_PyCriticalSection2_End(PyCriticalSection2 *c)
188+
_PyCriticalSection2_End(PyThreadState *tstate, PyCriticalSection2 *c)
197189
{
198190
// if mutex1 is NULL, we used the fast path in
199191
// _PyCriticalSection_BeginSlow for mutexes that are already held,
@@ -207,9 +199,8 @@ _PyCriticalSection2_End(PyCriticalSection2 *c)
207199
PyMutex_Unlock(c->_cs_mutex2);
208200
}
209201
PyMutex_Unlock(c->_cs_base._cs_mutex);
210-
_PyCriticalSection_Pop(&c->_cs_base);
202+
_PyCriticalSection_Pop(tstate, &c->_cs_base);
211203
}
212-
#define PyCriticalSection2_End _PyCriticalSection2_End
213204

214205
static inline void
215206
_PyCriticalSection_AssertHeld(PyMutex *mutex)
@@ -251,6 +242,45 @@ _PyCriticalSection_AssertHeldObj(PyObject *op)
251242

252243
#endif
253244
}
245+
246+
#undef Py_BEGIN_CRITICAL_SECTION
247+
# define Py_BEGIN_CRITICAL_SECTION(op) \
248+
{ \
249+
PyCriticalSection _py_cs; \
250+
PyThreadState *_cs_tstate = _PyThreadState_GET(); \
251+
_PyCriticalSection_Begin(_cs_tstate, &_py_cs, _PyObject_CAST(op))
252+
253+
#undef Py_BEGIN_CRITICAL_SECTION_MUTEX
254+
# define Py_BEGIN_CRITICAL_SECTION_MUTEX(mutex) \
255+
{ \
256+
PyCriticalSection _py_cs; \
257+
PyThreadState *_cs_tstate = _PyThreadState_GET(); \
258+
_PyCriticalSection_BeginMutex(_cs_tstate, &_py_cs, mutex)
259+
260+
#undef Py_END_CRITICAL_SECTION
261+
# define Py_END_CRITICAL_SECTION() \
262+
_PyCriticalSection_End(_cs_tstate, &_py_cs); \
263+
}
264+
265+
#undef Py_BEGIN_CRITICAL_SECTION2
266+
# define Py_BEGIN_CRITICAL_SECTION2(a, b) \
267+
{ \
268+
PyCriticalSection2 _py_cs2; \
269+
PyThreadState *_cs_tstate = _PyThreadState_GET(); \
270+
_PyCriticalSection2_Begin(_cs_tstate, &_py_cs2, _PyObject_CAST(a), _PyObject_CAST(b))
271+
272+
#undef Py_BEGIN_CRITICAL_SECTION2_MUTEX
273+
# define Py_BEGIN_CRITICAL_SECTION2_MUTEX(m1, m2) \
274+
{ \
275+
PyCriticalSection2 _py_cs2; \
276+
PyThreadState *_cs_tstate = _PyThreadState_GET(); \
277+
_PyCriticalSection2_BeginMutex(_cs_tstate, &_py_cs2, m1, m2)
278+
279+
#undef Py_END_CRITICAL_SECTION2
280+
# define Py_END_CRITICAL_SECTION2() \
281+
_PyCriticalSection2_End(_cs_tstate, &_py_cs2); \
282+
}
283+
254284
#endif /* Py_GIL_DISABLED */
255285

256286
#ifdef __cplusplus

‎Python/critical_section.c‎

Lines changed: 8 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,9 @@ untag_critical_section(uintptr_t tag)
1717
#endif
1818

1919
void
20-
_PyCriticalSection_BeginSlow(PyCriticalSection *c, PyMutex *m)
20+
_PyCriticalSection_BeginSlow(PyThreadState *tstate, PyCriticalSection *c, PyMutex *m)
2121
{
2222
#ifdef Py_GIL_DISABLED
23-
PyThreadState *tstate = _PyThreadState_GET();
2423
// As an optimisation for locking the same object recursively, skip
2524
// locking if the mutex is currently locked by the top-most critical
2625
// section.
@@ -53,11 +52,10 @@ _PyCriticalSection_BeginSlow(PyCriticalSection *c, PyMutex *m)
5352
}
5453

5554
void
56-
_PyCriticalSection2_BeginSlow(PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2,
55+
_PyCriticalSection2_BeginSlow(PyThreadState *tstate, PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2,
5756
int is_m1_locked)
5857
{
5958
#ifdef Py_GIL_DISABLED
60-
PyThreadState *tstate = _PyThreadState_GET();
6159
c->_cs_base._cs_mutex = NULL;
6260
c->_cs_mutex2 = NULL;
6361
c->_cs_base._cs_prev = tstate->critical_section;
@@ -139,7 +137,7 @@ void
139137
PyCriticalSection_Begin(PyCriticalSection *c, PyObject *op)
140138
{
141139
#ifdef Py_GIL_DISABLED
142-
_PyCriticalSection_Begin(c, op);
140+
_PyCriticalSection_Begin(_PyThreadState_GET(), c, op);
143141
#endif
144142
}
145143

@@ -148,7 +146,7 @@ void
148146
PyCriticalSection_BeginMutex(PyCriticalSection *c, PyMutex *m)
149147
{
150148
#ifdef Py_GIL_DISABLED
151-
_PyCriticalSection_BeginMutex(c, m);
149+
_PyCriticalSection_BeginMutex(_PyThreadState_GET(), c, m);
152150
#endif
153151
}
154152

@@ -157,7 +155,7 @@ void
157155
PyCriticalSection_End(PyCriticalSection *c)
158156
{
159157
#ifdef Py_GIL_DISABLED
160-
_PyCriticalSection_End(c);
158+
_PyCriticalSection_End(_PyThreadState_GET(), c);
161159
#endif
162160
}
163161

@@ -166,7 +164,7 @@ void
166164
PyCriticalSection2_Begin(PyCriticalSection2 *c, PyObject *a, PyObject *b)
167165
{
168166
#ifdef Py_GIL_DISABLED
169-
_PyCriticalSection2_Begin(c, a, b);
167+
_PyCriticalSection2_Begin(_PyThreadState_GET(), c, a, b);
170168
#endif
171169
}
172170

@@ -175,7 +173,7 @@ void
175173
PyCriticalSection2_BeginMutex(PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2)
176174
{
177175
#ifdef Py_GIL_DISABLED
178-
_PyCriticalSection2_BeginMutex(c, m1, m2);
176+
_PyCriticalSection2_BeginMutex(_PyThreadState_GET(), c, m1, m2);
179177
#endif
180178
}
181179

@@ -184,6 +182,6 @@ void
184182
PyCriticalSection2_End(PyCriticalSection2 *c)
185183
{
186184
#ifdef Py_GIL_DISABLED
187-
_PyCriticalSection2_End(c);
185+
_PyCriticalSection2_End(_PyThreadState_GET(), c);
188186
#endif
189187
}

0 commit comments

Comments
 (0)