|
| 1 | +/* |
| 2 | + * Implementation of the Global Interpreter Lock (GIL). |
| 3 | + */ |
| 4 | + |
| 5 | +#include <stdlib.h> |
| 6 | +#include <errno.h> |
| 7 | + |
| 8 | + |
| 9 | +/* First some general settings */ |
| 10 | + |
| 11 | +/* microseconds (the Python API uses seconds, though) */ |
| 12 | +#define DEFAULT_INTERVAL 5000 |
| 13 | +static unsigned long gil_interval = DEFAULT_INTERVAL; |
| 14 | +#define INTERVAL (gil_interval >= 1 ? gil_interval : 1) |
| 15 | + |
| 16 | +/* Enable if you want to force the switching of threads at least every `gil_interval` */ |
| 17 | +#undef FORCE_SWITCHING |
| 18 | +#define FORCE_SWITCHING |
| 19 | + |
| 20 | + |
| 21 | +/* |
| 22 | + Notes about the implementation: |
| 23 | +
|
| 24 | + - The GIL is just a boolean variable (gil_locked) whose access is protected |
| 25 | + by a mutex (gil_mutex), and whose changes are signalled by a condition |
| 26 | + variable (gil_cond). gil_mutex is taken for short periods of time, |
| 27 | + and therefore mostly uncontended. |
| 28 | +
|
| 29 | + - In the GIL-holding thread, the main loop (PyEval_EvalFrameEx) must be |
| 30 | + able to release the GIL on demand by another thread. A volatile boolean |
| 31 | + variable (gil_drop_request) is used for that purpose, which is checked |
| 32 | + at every turn of the eval loop. That variable is set after a wait of |
| 33 | + `interval` microseconds on `gil_cond` has timed out. |
| 34 | +
|
| 35 | + [Actually, another volatile boolean variable (eval_breaker) is used |
| 36 | + which ORs several conditions into one. Volatile booleans are |
| 37 | + sufficient as inter-thread signalling means since Python is run |
| 38 | + on cache-coherent architectures only.] |
| 39 | +
|
| 40 | + - A thread wanting to take the GIL will first let pass a given amount of |
| 41 | + time (`interval` microseconds) before setting gil_drop_request. This |
| 42 | + encourages a defined switching period, but doesn't enforce it since |
| 43 | + opcodes can take an arbitrary time to execute. |
| 44 | +
|
| 45 | + The `interval` value is available for the user to read and modify |
| 46 | + using the Python API `sys.{get,set}switchinterval()`. |
| 47 | +
|
| 48 | + - When a thread releases the GIL and gil_drop_request is set, that thread |
| 49 | + ensures that another GIL-awaiting thread gets scheduled. |
| 50 | + It does so by waiting on a condition variable (switch_cond) until |
| 51 | + the value of gil_last_holder is changed to something else than its |
| 52 | + own thread state pointer, indicating that another thread was able to |
| 53 | + take the GIL. |
| 54 | +
|
| 55 | + This is meant to prohibit the latency-adverse behaviour on multi-core |
| 56 | + machines where one thread would speculatively release the GIL, but still |
| 57 | + run and end up being the first to re-acquire it, making the "timeslices" |
| 58 | + much longer than expected. |
| 59 | + (Note: this mechanism is enabled with FORCE_SWITCHING above) |
| 60 | +*/ |
| 61 | + |
| 62 | +#include "condvar.h" |
| 63 | +#ifndef Py_HAVE_CONDVAR |
| 64 | +#error You need either a POSIX-compatible or a Windows system! |
| 65 | +#endif |
| 66 | + |
| 67 | +#define MUTEX_T PyMUTEX_T |
| 68 | +#define MUTEX_INIT(mut) \ |
| 69 | + if (PyMUTEX_INIT(&(mut))) { \ |
| 70 | + Py_FatalError("PyMUTEX_INIT(" #mut ") failed"); }; |
| 71 | +#define MUTEX_FINI(mut) \ |
| 72 | + if (PyMUTEX_FINI(&(mut))) { \ |
| 73 | + Py_FatalError("PyMUTEX_FINI(" #mut ") failed"); }; |
| 74 | +#define MUTEX_LOCK(mut) \ |
| 75 | + if (PyMUTEX_LOCK(&(mut))) { \ |
| 76 | + Py_FatalError("PyMUTEX_LOCK(" #mut ") failed"); }; |
| 77 | +#define MUTEX_UNLOCK(mut) \ |
| 78 | + if (PyMUTEX_UNLOCK(&(mut))) { \ |
| 79 | + Py_FatalError("PyMUTEX_UNLOCK(" #mut ") failed"); }; |
| 80 | + |
| 81 | +#define COND_T PyCOND_T |
| 82 | + |
| 83 | +#ifdef SWITCH |
| 84 | +#define COND_INIT(cond, mut) \ |
| 85 | + if (PyCOND_INIT(&(cond), &(mut))) { \ |
| 86 | + Py_FatalError("PyCOND_INIT(" #cond ") failed"); }; |
| 87 | +#else |
| 88 | +#define COND_INIT(cond) \ |
| 89 | + if (PyCOND_INIT(&(cond))) { \ |
| 90 | + Py_FatalError("PyCOND_INIT(" #cond ") failed"); }; |
| 91 | +#endif |
| 92 | + |
| 93 | +#define COND_FINI(cond) \ |
| 94 | + if (PyCOND_FINI(&(cond))) { \ |
| 95 | + Py_FatalError("PyCOND_FINI(" #cond ") failed"); }; |
| 96 | +#define COND_SIGNAL(cond) \ |
| 97 | + if (PyCOND_SIGNAL(&(cond))) { \ |
| 98 | + Py_FatalError("PyCOND_SIGNAL(" #cond ") failed"); }; |
| 99 | +#define COND_WAIT(cond, mut) \ |
| 100 | + if (PyCOND_WAIT(&(cond), &(mut))) { \ |
| 101 | + Py_FatalError("PyCOND_WAIT(" #cond ") failed"); }; |
| 102 | +#define COND_TIMED_WAIT(cond, mut, microseconds, timeout_result) \ |
| 103 | + { \ |
| 104 | + int r = PyCOND_TIMEDWAIT(&(cond), &(mut), (microseconds)); \ |
| 105 | + if (r < 0) \ |
| 106 | + Py_FatalError("PyCOND_WAIT(" #cond ") failed"); \ |
| 107 | + if (r) /* 1 == timeout, 2 == impl. can't say, so assume timeout */ \ |
| 108 | + timeout_result = 1; \ |
| 109 | + else \ |
| 110 | + timeout_result = 0; \ |
| 111 | + } \ |
| 112 | + |
| 113 | + |
| 114 | + |
| 115 | +/* Whether the GIL is already taken (-1 if uninitialized). This is atomic |
| 116 | + because it can be read without any lock taken in ceval.c. */ |
| 117 | +static _Py_atomic_int gil_locked = {-1}; |
| 118 | +/* Number of GIL switches since the beginning. */ |
| 119 | +static unsigned long gil_switch_number = 0; |
| 120 | +/* Last PyThreadState holding / having held the GIL. This helps us know |
| 121 | + whether anyone else was scheduled after we dropped the GIL. */ |
| 122 | +static _Py_atomic_address gil_last_holder = {0}; |
| 123 | + |
| 124 | +/* This condition variable allows one or several threads to wait until |
| 125 | + the GIL is released. In addition, the mutex also protects the above |
| 126 | + variables. */ |
| 127 | +static COND_T gil_cond; |
| 128 | +static MUTEX_T gil_mutex; |
| 129 | + |
| 130 | +#ifdef FORCE_SWITCHING |
| 131 | +/* This condition variable helps the GIL-releasing thread wait for |
| 132 | + a GIL-awaiting thread to be scheduled and take the GIL. */ |
| 133 | +static COND_T switch_cond; |
| 134 | +static MUTEX_T switch_mutex; |
| 135 | +#endif |
| 136 | + |
| 137 | + |
| 138 | +static int gil_created(void) |
| 139 | +{ |
| 140 | + return _Py_atomic_load_explicit(&gil_locked, _Py_memory_order_acquire) >= 0; |
| 141 | +} |
| 142 | + |
| 143 | +static void create_gil(void) |
| 144 | +{ |
| 145 | + MUTEX_INIT(gil_mutex); |
| 146 | +#ifdef FORCE_SWITCHING |
| 147 | + MUTEX_INIT(switch_mutex); |
| 148 | +#endif |
| 149 | + COND_INIT(gil_cond, gil_mutex); |
| 150 | +#ifdef FORCE_SWITCHING |
| 151 | + COND_INIT(switch_cond, switch_mutex); |
| 152 | +#endif |
| 153 | + _Py_atomic_store_relaxed(&gil_last_holder, 0); |
| 154 | + _Py_ANNOTATE_RWLOCK_CREATE(&gil_locked); |
| 155 | + _Py_atomic_store_explicit(&gil_locked, 0, _Py_memory_order_release); |
| 156 | +} |
| 157 | + |
| 158 | +static void destroy_gil(void) |
| 159 | +{ |
| 160 | + /* some pthread-like implementations tie the mutex to the cond |
| 161 | + * and must have the cond destroyed first. |
| 162 | + */ |
| 163 | + COND_FINI(gil_cond); |
| 164 | + MUTEX_FINI(gil_mutex); |
| 165 | +#ifdef FORCE_SWITCHING |
| 166 | + COND_FINI(switch_cond); |
| 167 | + MUTEX_FINI(switch_mutex); |
| 168 | +#endif |
| 169 | + _Py_atomic_store_explicit(&gil_locked, -1, _Py_memory_order_release); |
| 170 | + _Py_ANNOTATE_RWLOCK_DESTROY(&gil_locked); |
| 171 | +} |
| 172 | + |
| 173 | +static void recreate_gil(void) |
| 174 | +{ |
| 175 | + _Py_ANNOTATE_RWLOCK_DESTROY(&gil_locked); |
| 176 | + /* XXX should we destroy the old OS resources here? */ |
| 177 | + create_gil(); |
| 178 | +} |
| 179 | + |
| 180 | +static void drop_gil(PyThreadState *tstate) |
| 181 | +{ |
| 182 | + if (!_Py_atomic_load_relaxed(&gil_locked)) |
| 183 | + Py_FatalError("drop_gil: GIL is not locked"); |
| 184 | + /* tstate is allowed to be NULL (early interpreter init) */ |
| 185 | + if (tstate != NULL) { |
| 186 | + /* Sub-interpreter support: threads might have been switched |
| 187 | + under our feet using PyThreadState_Swap(). Fix the GIL last |
| 188 | + holder variable so that our heuristics work. */ |
| 189 | + _Py_atomic_store_relaxed(&gil_last_holder, (Py_uintptr_t)tstate); |
| 190 | + } |
| 191 | + |
| 192 | + MUTEX_LOCK(gil_mutex); |
| 193 | + _Py_ANNOTATE_RWLOCK_RELEASED(&gil_locked, /*is_write=*/1); |
| 194 | + _Py_atomic_store_relaxed(&gil_locked, 0); |
| 195 | + COND_SIGNAL(gil_cond); |
| 196 | + MUTEX_UNLOCK(gil_mutex); |
| 197 | + |
| 198 | +#ifdef FORCE_SWITCHING |
| 199 | + if (_Py_atomic_load_relaxed(&gil_drop_request) && tstate != NULL) { |
| 200 | + MUTEX_LOCK(switch_mutex); |
| 201 | + /* Not switched yet => wait */ |
| 202 | + if ((PyThreadState*)_Py_atomic_load_relaxed(&gil_last_holder) == tstate) { |
| 203 | + RESET_GIL_DROP_REQUEST(); |
| 204 | + /* NOTE: if COND_WAIT does not atomically start waiting when |
| 205 | + releasing the mutex, another thread can run through, take |
| 206 | + the GIL and drop it again, and reset the condition |
| 207 | + before we even had a chance to wait for it. */ |
| 208 | + COND_WAIT(switch_cond, switch_mutex); |
| 209 | + } |
| 210 | + MUTEX_UNLOCK(switch_mutex); |
| 211 | + } |
| 212 | +#endif |
| 213 | +} |
| 214 | + |
| 215 | +static void take_gil(PyThreadState *tstate) |
| 216 | +{ |
| 217 | + int err; |
| 218 | + if (tstate == NULL) |
| 219 | + Py_FatalError("take_gil: NULL tstate"); |
| 220 | + |
| 221 | + err = errno; |
| 222 | + MUTEX_LOCK(gil_mutex); |
| 223 | + |
| 224 | + if (!_Py_atomic_load_relaxed(&gil_locked)) |
| 225 | + goto _ready; |
| 226 | + |
| 227 | + while (_Py_atomic_load_relaxed(&gil_locked)) { |
| 228 | + int timed_out = 0; |
| 229 | + unsigned long saved_switchnum; |
| 230 | + |
| 231 | + saved_switchnum = gil_switch_number; |
| 232 | + COND_TIMED_WAIT(gil_cond, gil_mutex, INTERVAL, timed_out); |
| 233 | + /* If we timed out and no switch occurred in the meantime, it is time |
| 234 | + to ask the GIL-holding thread to drop it. */ |
| 235 | + if (timed_out && |
| 236 | + _Py_atomic_load_relaxed(&gil_locked) && |
| 237 | + gil_switch_number == saved_switchnum) { |
| 238 | + SET_GIL_DROP_REQUEST(); |
| 239 | + } |
| 240 | + } |
| 241 | +_ready: |
| 242 | +#ifdef FORCE_SWITCHING |
| 243 | + /* This mutex must be taken before modifying gil_last_holder (see drop_gil()). */ |
| 244 | + MUTEX_LOCK(switch_mutex); |
| 245 | +#endif |
| 246 | + /* We now hold the GIL */ |
| 247 | + _Py_atomic_store_relaxed(&gil_locked, 1); |
| 248 | + _Py_ANNOTATE_RWLOCK_ACQUIRED(&gil_locked, /*is_write=*/1); |
| 249 | + |
| 250 | + if (tstate != (PyThreadState*)_Py_atomic_load_relaxed(&gil_last_holder)) { |
| 251 | + _Py_atomic_store_relaxed(&gil_last_holder, (Py_uintptr_t)tstate); |
| 252 | + ++gil_switch_number; |
| 253 | + } |
| 254 | + |
| 255 | +#ifdef FORCE_SWITCHING |
| 256 | + COND_SIGNAL(switch_cond); |
| 257 | + MUTEX_UNLOCK(switch_mutex); |
| 258 | +#endif |
| 259 | + if (_Py_atomic_load_relaxed(&gil_drop_request)) { |
| 260 | + RESET_GIL_DROP_REQUEST(); |
| 261 | + } |
| 262 | + if (tstate->async_exc != NULL) { |
| 263 | + _PyEval_SignalAsyncExc(); |
| 264 | + } |
| 265 | + |
| 266 | + MUTEX_UNLOCK(gil_mutex); |
| 267 | + errno = err; |
| 268 | +} |
| 269 | + |
| 270 | +void _PyEval_SetSwitchInterval(unsigned long microseconds) |
| 271 | +{ |
| 272 | + gil_interval = microseconds; |
| 273 | +} |
| 274 | + |
| 275 | +unsigned long _PyEval_GetSwitchInterval() |
| 276 | +{ |
| 277 | + return gil_interval; |
| 278 | +} |
0 commit comments