From c7e4b833e63d4321c65eb2c5b95f5fba4764b9b4 Mon Sep 17 00:00:00 2001 From: Ivan Pozdeev Date: Tue, 10 Apr 2018 15:23:30 +0300 Subject: [PATCH 01/15] Fix race conditions for non-threaded Tcl when passing data to/from it --- Modules/_tkinter.c | 72 +++++++++++++++++++++++++++------------------- 1 file changed, 42 insertions(+), 30 deletions(-) diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c index 93d4dbc5f659a0..a811d8b976b4ca 100644 --- a/Modules/_tkinter.c +++ b/Modules/_tkinter.c @@ -240,6 +240,9 @@ static PyThreadState *tcl_tstate = NULL; #define ENTER_OVERLAP \ Py_END_ALLOW_THREADS +#define LEAVE_OVERLAP \ + Py_BEGIN_ALLOW_THREADS + #define LEAVE_OVERLAP_TCL \ tcl_tstate = NULL; if(tcl_lock)PyThread_release_lock(tcl_lock); } @@ -1499,24 +1502,28 @@ Tkapp_Call(PyObject *selfptr, PyObject *args) else { + ENTER_TCL + ENTER_OVERLAP + objv = Tkapp_CallArgs(args, objStore, &objc); - if (!objv) - return NULL; - ENTER_TCL + if (objv) { - i = Tcl_EvalObjv(self->interp, objc, objv, flags); + LEAVE_OVERLAP - ENTER_OVERLAP + i = Tcl_EvalObjv(self->interp, objc, objv, flags); - if (i == TCL_ERROR) - Tkinter_Error(selfptr); - else - res = Tkapp_CallResult(self); + ENTER_OVERLAP - LEAVE_OVERLAP_TCL + if (i == TCL_ERROR) + Tkinter_Error(selfptr); + else + res = Tkapp_CallResult(self); - Tkapp_CallDeallocArgs(objv, objStore, objc); + Tkapp_CallDeallocArgs(objv, objStore, objc); + + } + LEAVE_OVERLAP_TCL } return res; } @@ -1790,19 +1797,20 @@ SetVar(PyObject *self, PyObject *args, int flags) if (!PyArg_ParseTuple(args, "O&O:setvar", varname_converter, &name1, &newValue)) return NULL; - /* XXX Acquire tcl lock??? */ - newval = AsObj(newValue); - if (newval == NULL) - return NULL; ENTER_TCL - ok = Tcl_SetVar2Ex(Tkapp_Interp(self), name1, NULL, - newval, flags); ENTER_OVERLAP - if (!ok) - Tkinter_Error(self); - else { - res = Py_None; - Py_INCREF(res); + newval = AsObj(newValue); + if (newval) { + LEAVE_OVERLAP + ok = Tcl_SetVar2Ex(Tkapp_Interp(self), name1, NULL, + newval, flags); + ENTER_OVERLAP + if (!ok) + Tkinter_Error(self); + else { + res = Py_None; + Py_INCREF(res); + } } LEAVE_OVERLAP_TCL break; @@ -1812,16 +1820,20 @@ SetVar(PyObject *self, PyObject *args, int flags) return NULL; CHECK_STRING_LENGTH(name1); CHECK_STRING_LENGTH(name2); - /* XXX must hold tcl lock already??? */ - newval = AsObj(newValue); ENTER_TCL - ok = Tcl_SetVar2Ex(Tkapp_Interp(self), name1, name2, newval, flags); ENTER_OVERLAP - if (!ok) - Tkinter_Error(self); - else { - res = Py_None; - Py_INCREF(res); + newval = AsObj(newValue); + if (newval) { + LEAVE_OVERLAP + ok = Tcl_SetVar2Ex(Tkapp_Interp(self), name1, name2, + newval, flags); + ENTER_OVERLAP + if (!ok) + Tkinter_Error(self); + else { + res = Py_None; + Py_INCREF(res); + } } LEAVE_OVERLAP_TCL break; From 6696aae9730cab1af89a53959bcfc07407587a6e Mon Sep 17 00:00:00 2001 From: Ivan Pozdeev Date: Tue, 10 Apr 2018 19:03:42 +0300 Subject: [PATCH 02/15] Document the added complexity of TCL bracket macros more clearly --- Modules/_tkinter.c | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c index a811d8b976b4ca..88e75257d64149 100644 --- a/Modules/_tkinter.c +++ b/Modules/_tkinter.c @@ -168,24 +168,24 @@ _get_tcl_lib_path() waiting for events. To solve this problem, a separate lock for Tcl is introduced. - Holding it is incompatible with holding Python's interpreter lock. - The following four macros manipulate both locks together. - - ENTER_TCL and LEAVE_TCL are brackets, just like - Py_BEGIN_ALLOW_THREADS and Py_END_ALLOW_THREADS. They should be - used whenever a call into Tcl is made that could call an event - handler, or otherwise affect the state of a Tcl interpreter. These - assume that the surrounding code has the Python interpreter lock; - inside the brackets, the Python interpreter lock has been released - and the lock for Tcl has been acquired. - - Sometimes, it is necessary to have both the Python lock and the Tcl - lock. (For example, when transferring data from the Tcl - interpreter result to a Python string object.) This can be done by - using different macros to close the ENTER_TCL block: ENTER_OVERLAP - reacquires the Python lock (and restores the thread state) but - doesn't release the Tcl lock; LEAVE_OVERLAP_TCL releases the Tcl - lock. + We normally treat holding it as incompatible with holding Python's + interpreter lock. The following macros manipulate both locks together. + + ENTER_TCL and LEAVE_TCL are brackets, just like Py_BEGIN_ALLOW_THREADS and + Py_END_ALLOW_THREADS. They should be used whenever a call into Tcl is made + that could call an event handler, or otherwise affect the state of a Tcl + interpreter. These assume that the surrounding code has the Python + interpreter lock: ENTER_TCL releases the Python lock, then acquires the Tcl + lock, and LEAVE_TCL does the reverse. + + Sometimes, it is necessary to have both the Python lock and the Tcl lock. + (For example, when transferring data between the Tcl interpreter and/or + objects and Python objects.) To avoid deadlocks, when acquiring, we always + acquire the Tcl lock first, then the Python lock. The additional macros for + finer lock control are: ENTER_OVERLAP acquires the Python lock (and restores + the thread state) when already holding the Tcl lock; LEAVE_OVERLAP releases + the Python lock and keeps the Tcl lock; and LEAVE_OVERLAP_TCL releases the + Tcl lock and keeps the Python lock. By contrast, ENTER_PYTHON and LEAVE_PYTHON are used in Tcl event handlers when the handler needs to use Python. Such event handlers From 2c4c3e95d02a12e5f938c38caa69fecb6395a636 Mon Sep 17 00:00:00 2001 From: Ivan Pozdeev Date: Sat, 14 Apr 2018 01:27:28 +0300 Subject: [PATCH 03/15] Add NEWS entry for bpo-33257 --- .../NEWS.d/next/Library/2018-04-14-01-26-10.bpo-33257.i32qOW.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Library/2018-04-14-01-26-10.bpo-33257.i32qOW.rst diff --git a/Misc/NEWS.d/next/Library/2018-04-14-01-26-10.bpo-33257.i32qOW.rst b/Misc/NEWS.d/next/Library/2018-04-14-01-26-10.bpo-33257.i32qOW.rst new file mode 100644 index 00000000000000..d8611ec82b046f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-04-14-01-26-10.bpo-33257.i32qOW.rst @@ -0,0 +1 @@ +Fixed race conditions in Tkinter when passing data to/from nonthreaded Tcl From e828ea7ff6becc178dd7611a721d0e53b605c54a Mon Sep 17 00:00:00 2001 From: Ivan Pozdeev Date: Sun, 15 Apr 2018 19:59:21 +0300 Subject: [PATCH 04/15] * Tkapp_CallArgs, Tkapp_CallResult, AsObj require both locks * Tkapp_CallDeallocArgs requires Tcl lock --- Modules/_tkinter.c | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c index 88e75257d64149..475ca70256b581 100644 --- a/Modules/_tkinter.c +++ b/Modules/_tkinter.c @@ -255,6 +255,13 @@ static PyThreadState *tcl_tstate = NULL; { PyThreadState *tstate = PyEval_SaveThread(); \ if(tcl_lock)PyThread_acquire_lock(tcl_lock, 1); tcl_tstate = tstate; } +#define ENTER_PYTHON_OVERLAP \ +{ PyThreadState *tstate = tcl_tstate; tcl_tstate = NULL; PyEval_RestoreThread((tstate)); } + +#define LEAVE_PYTHON_OVERLAP \ +{ PyThreadState *tstate = PyEval_SaveThread(); tcl_tstate = tstate; } + + #define CHECK_TCL_APPARTMENT \ if (((TkappObject *)self)->threaded && \ ((TkappObject *)self)->thread_id != Tcl_GetCurrentThread()) { \ @@ -1405,17 +1412,17 @@ Tkapp_CallProc(Tkapp_CallEvent *e, int flags) Tcl_Obj **objv; int objc; int i; - ENTER_PYTHON + ENTER_PYTHON_OVERLAP objv = Tkapp_CallArgs(e->args, objStore, &objc); if (!objv) { PyErr_Fetch(e->exc_type, e->exc_value, e->exc_tb); *(e->res) = NULL; } - LEAVE_PYTHON + LEAVE_PYTHON_OVERLAP if (!objv) goto done; i = Tcl_EvalObjv(e->self->interp, objc, objv, e->flags); - ENTER_PYTHON + ENTER_PYTHON_OVERLAP if (i == TCL_ERROR) { *(e->res) = NULL; *(e->exc_type) = NULL; @@ -1427,7 +1434,7 @@ Tkapp_CallProc(Tkapp_CallEvent *e, int flags) else { *(e->res) = Tkapp_CallResult(e->self); } - LEAVE_PYTHON + LEAVE_PYTHON_OVERLAP Tkapp_CallDeallocArgs(objv, objStore, objc); done: @@ -1520,7 +1527,11 @@ Tkapp_Call(PyObject *selfptr, PyObject *args) else res = Tkapp_CallResult(self); + LEAVE_OVERLAP + Tkapp_CallDeallocArgs(objv, objStore, objc); + + ENTER_OVERLAP } LEAVE_OVERLAP_TCL @@ -2379,15 +2390,18 @@ PythonCmd(ClientData clientData, Tcl_Interp *interp, int argc, const char *argv[ if (res == NULL) return PythonCmd_Error(interp); + ENTER_TCL + ENTER_OVERLAP obj_res = AsObj(res); + if (obj_res) { + Tcl_SetObjResult(interp, obj_res); + rv = TCL_OK; + } + LEAVE_OVERLAP_TCL if (obj_res == NULL) { Py_DECREF(res); return PythonCmd_Error(interp); } - else { - Tcl_SetObjResult(interp, obj_res); - rv = TCL_OK; - } Py_DECREF(res); From a67255a8f141305db9598fab5723f4f7995fae51 Mon Sep 17 00:00:00 2001 From: Ivan Pozdeev Date: Sat, 12 May 2018 03:49:15 +0300 Subject: [PATCH 05/15] Regrtest bpo-33257 --- Lib/tkinter/test/test_tkinter/test_threads.py | 144 ++++++++++++++++++ 1 file changed, 144 insertions(+) create mode 100644 Lib/tkinter/test/test_tkinter/test_threads.py diff --git a/Lib/tkinter/test/test_tkinter/test_threads.py b/Lib/tkinter/test/test_tkinter/test_threads.py new file mode 100644 index 00000000000000..2dd18fa0f2b5f4 --- /dev/null +++ b/Lib/tkinter/test/test_tkinter/test_threads.py @@ -0,0 +1,144 @@ +#!/usr/bin/python3 + +import sys +import time +import threading +import random +import tkinter + +import unittest +class TestThreads(unittest.TestCase): + def test_threads(self): + import subprocess + p = subprocess.Popen([sys.executable, __file__, "run"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + try: + #Test code is designed to complete in a few seconds + stdout, stderr = p.communicate(timeout=10) + except subprocess.TimeoutExpired: + p.kill() + stdout, stderr = p.communicate() + self.fail("Test code hang. Stderr: " + repr(stderr)) + rc = p.returncode + self.assertTrue(rc == 0, + "Nonzero exit status: " + str(rc) + "; stderr:" + repr(stderr)) + self.assertTrue(len(stderr) == 0, "stderr: " + repr(stderr)) + + + +running = True + +class Track(threading.Thread): + """ + Calculates coordinates for a ballistic track + with random angle and velocity + and fires the callback to draw each consecutive segment. + """ + def __init__(self, track_number, draw_callback): + """ + :param track_number: ordinal + :param draw_callback: fn(track_number, x, y) + that draws the extention of the specified track + to the specified point. The callback must keep track + of any previous coordinates itself. + """ + threading.Thread.__init__(self) + #self.setDaemon(True) + self.track_number = track_number + self.draw_callback = draw_callback + + def run(self): + #starting point for the track + y = 0.0001 #height + x = 999.0 #range + #initial velocities + yVel = 400. + random.random() * 200. + xVel = -200. - random.random() * 150. + # Stop drawing when the track hits the ground + while y > 0 and running: + #How long to sleep, in seconds, between track updates + time.sleep(0.01) #realism: >1 Fun: 0.01 + + yVel -= 9.8 #gravity, more or less + xVel *= .9998 #air resistance + + y += yVel + x += xVel + + if running: + self.draw_callback(self.track_number, x, y) + + +class App: + """ + The main app logic + """ + def __init__(self, window): + self.track_no = 0 #last index of track added + self.track_coordinates = {} #coords of track tips + self.threads = [] + + self.window = window + self.frame = tkinter.Frame(window) + self.frame.pack() + self.graph = tkinter.Canvas(self.frame) + self.graph.pack() + + self.t_cleanup = threading.Thread(target=self.tf_cleanup) + self.window.after(0, self.add_tracks, 5) + self.window.after(1000, self.t_cleanup.start) + + def add_tracks(self,depth): + if depth>0: self.window.after(5, self.add_tracks, depth-1) + tracks_to_add = 40 + start_no = self.track_no + self.track_no += tracks_to_add + for self.track_no in range(start_no, self.track_no): + #self.window.after(t, self.add_tracks) + #self.track_no += 1 #Make a new track number + #if (self.track_no > 40): + t = Track(self.track_no, self.draw_track_segment) + self.threads.append(t) + t.start() + + def tf_cleanup(self): + global running + running = False + for t in self.threads: t.join() + self.window.after(0,self.window.destroy) + + def draw_track_segment(self, track_no, x, y): + # x & y are virtual coordinates for the purpose of simulation. + #To convert them to screen coordinates, + # we scale them down so the graphs fit into the window, and move the origin. + # Y is also inverted because in canvas, the UL corner is the origin. + newsx, newsy = (250.+x/100., 150.-y/100.) + + try: + (oldsx, oldsy) = self.track_coordinates[track_no] + except KeyError: + pass + else: + self.graph.create_line(oldsx, oldsy, + newsx, newsy) + self.track_coordinates[track_no] = (newsx, newsy) + + def go(self): + self.window.mainloop() + +tests_gui = (TestThreads,) + +if __name__=='__main__': + import sys,os + if sys.argv[1:]==['run']: + if os.name == 'nt': + import ctypes + #the bug causes crashes + ctypes.windll.kernel32.SetErrorMode(3) + App(tkinter.Tk()).go() + else: + from test.support import run_unittest + run_unittest(*tests_gui) From 6ccdccbcd91608fe5def56825f39756ac0c00f13 Mon Sep 17 00:00:00 2001 From: Ivan Pozdeev Date: Sun, 15 Apr 2018 20:09:05 +0300 Subject: [PATCH 06/15] * PythonCmd_Error doesn't use the arg and thus can do without Tcl lock --- Modules/_tkinter.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c index 475ca70256b581..2e950801c4359e 100644 --- a/Modules/_tkinter.c +++ b/Modules/_tkinter.c @@ -2346,7 +2346,7 @@ typedef struct { } PythonCmd_ClientData; static int -PythonCmd_Error(Tcl_Interp *interp) +PythonCmd_Error(void) { errorInCmd = 1; PyErr_Fetch(&excInCmd, &valInCmd, &trbInCmd); @@ -2374,13 +2374,13 @@ PythonCmd(ClientData clientData, Tcl_Interp *interp, int argc, const char *argv[ /* Create argument list (argv1, ..., argvN) */ if (!(arg = PyTuple_New(argc - 1))) - return PythonCmd_Error(interp); + return PythonCmd_Error(); for (i = 0; i < (argc - 1); i++) { PyObject *s = unicodeFromTclString(argv[i + 1]); if (!s) { Py_DECREF(arg); - return PythonCmd_Error(interp); + return PythonCmd_Error(); } PyTuple_SET_ITEM(arg, i, s); } @@ -2388,7 +2388,7 @@ PythonCmd(ClientData clientData, Tcl_Interp *interp, int argc, const char *argv[ Py_DECREF(arg); if (res == NULL) - return PythonCmd_Error(interp); + return PythonCmd_Error(); ENTER_TCL ENTER_OVERLAP @@ -2400,7 +2400,7 @@ PythonCmd(ClientData clientData, Tcl_Interp *interp, int argc, const char *argv[ LEAVE_OVERLAP_TCL if (obj_res == NULL) { Py_DECREF(res); - return PythonCmd_Error(interp); + return PythonCmd_Error(); } Py_DECREF(res); From 0caf3559b0480e887d076e1cd9bca13d37ebc7f6 Mon Sep 17 00:00:00 2001 From: Ivan Pozdeev Date: Wed, 25 Apr 2018 01:57:25 +0300 Subject: [PATCH 07/15] Make all Tcl lock operations into macros for easier tracking --- Modules/_tkinter.c | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c index 2e950801c4359e..348a56ab2fa29b 100644 --- a/Modules/_tkinter.c +++ b/Modules/_tkinter.c @@ -230,13 +230,25 @@ static PyThreadState *tcl_tstate = NULL; #endif #define ENTER_TCL \ - { PyThreadState *tstate = PyThreadState_Get(); Py_BEGIN_ALLOW_THREADS \ - if(tcl_lock)PyThread_acquire_lock(tcl_lock, 1); tcl_tstate = tstate; + { PyThreadState *tstate = PyThreadState_Get();\ + ENTER_TCL_CUSTOM_TSTATE(tstate) + +#define ENTER_TCL_CUSTOM_TSTATE(tstate) \ + Py_BEGIN_ALLOW_THREADS\ + if (tcl_lock)PyThread_acquire_lock(tcl_lock, 1); tcl_tstate = tstate; #define LEAVE_TCL \ tcl_tstate = NULL; \ if(tcl_lock)PyThread_release_lock(tcl_lock); Py_END_ALLOW_THREADS} +#define LEAVE_TCL_WITH_BUSYWAIT(result) \ + tcl_tstate = NULL; \ + if (tcl_lock)PyThread_release_lock(tcl_lock);\ + if (result == 0)\ + Sleep(Tkinter_busywaitinterval);\ + Py_END_ALLOW_THREADS + + #define ENTER_OVERLAP \ Py_END_ALLOW_THREADS @@ -2879,15 +2891,9 @@ _tkinter_tkapp_mainloop_impl(TkappObject *self, int threshold) LEAVE_TCL } else { - Py_BEGIN_ALLOW_THREADS - if(tcl_lock)PyThread_acquire_lock(tcl_lock, 1); - tcl_tstate = tstate; + ENTER_TCL_CUSTOM_TSTATE(tstate) result = Tcl_DoOneEvent(TCL_DONT_WAIT); - tcl_tstate = NULL; - if(tcl_lock)PyThread_release_lock(tcl_lock); - if (result == 0) - Sleep(Tkinter_busywaitinterval); - Py_END_ALLOW_THREADS + LEAVE_TCL_WITH_BUSYWAIT(result) } if (PyErr_CheckSignals() != 0) { @@ -3345,17 +3351,11 @@ EventHook(void) break; } #endif - Py_BEGIN_ALLOW_THREADS - if(tcl_lock)PyThread_acquire_lock(tcl_lock, 1); - tcl_tstate = event_tstate; + ENTER_TCL_CUSTOM_TSTATE(event_tstate) result = Tcl_DoOneEvent(TCL_DONT_WAIT); - tcl_tstate = NULL; - if(tcl_lock)PyThread_release_lock(tcl_lock); - if (result == 0) - Sleep(Tkinter_busywaitinterval); - Py_END_ALLOW_THREADS + LEAVE_TCL_WITH_BUSYWAIT(result) if (result < 0) break; From 028b32ae266917af7085a3904b331dc58e9d3882 Mon Sep 17 00:00:00 2001 From: Ivan Pozdeev Date: Wed, 16 May 2018 05:29:54 +0300 Subject: [PATCH 08/15] Make Tcl lock reentrant --- Modules/_tkinter.c | 39 ++++++++++++++++++++++++++++++++------- 1 file changed, 32 insertions(+), 7 deletions(-) diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c index 348a56ab2fa29b..dac4d180464bd7 100644 --- a/Modules/_tkinter.c +++ b/Modules/_tkinter.c @@ -219,6 +219,9 @@ _get_tcl_lib_path() */ static PyThread_type_lock tcl_lock = 0; +/* Only the thread holding the lock can modify these */ +static unsigned long tcl_lock_thread_ident = 0; +static unsigned int tcl_lock_reentry_count = 0; #ifdef TCL_THREADS static Tcl_ThreadDataKey state_key; @@ -229,21 +232,43 @@ typedef PyThreadState *ThreadSpecificData; static PyThreadState *tcl_tstate = NULL; #endif +#define ACQUIRE_TCL_LOCK \ +if (tcl_lock) {\ + if (tcl_lock_thread_ident == PyThread_get_thread_ident()) {\ + tcl_lock_reentry_count++; \ + if(!tcl_lock_reentry_count) \ + Py_FatalError("Tcl lock reentry count overflow"); \ + } else { \ + PyThread_acquire_lock(tcl_lock, 1); \ + tcl_lock_thread_ident = PyThread_get_thread_ident(); \ + }\ +} + +#define RELEASE_TCL_LOCK \ +if (tcl_lock){\ + if (tcl_lock_reentry_count) { \ + tcl_lock_reentry_count--; \ + } else { \ + tcl_lock_thread_ident = 0; \ + PyThread_release_lock(tcl_lock); \ + }\ +} + #define ENTER_TCL \ { PyThreadState *tstate = PyThreadState_Get();\ ENTER_TCL_CUSTOM_TSTATE(tstate) #define ENTER_TCL_CUSTOM_TSTATE(tstate) \ Py_BEGIN_ALLOW_THREADS\ - if (tcl_lock)PyThread_acquire_lock(tcl_lock, 1); tcl_tstate = tstate; + ACQUIRE_TCL_LOCK; tcl_tstate = tstate; #define LEAVE_TCL \ tcl_tstate = NULL; \ - if(tcl_lock)PyThread_release_lock(tcl_lock); Py_END_ALLOW_THREADS} + RELEASE_TCL_LOCK; Py_END_ALLOW_THREADS} #define LEAVE_TCL_WITH_BUSYWAIT(result) \ tcl_tstate = NULL; \ - if (tcl_lock)PyThread_release_lock(tcl_lock);\ + RELEASE_TCL_LOCK;\ if (result == 0)\ Sleep(Tkinter_busywaitinterval);\ Py_END_ALLOW_THREADS @@ -256,16 +281,16 @@ static PyThreadState *tcl_tstate = NULL; Py_BEGIN_ALLOW_THREADS #define LEAVE_OVERLAP_TCL \ - tcl_tstate = NULL; if(tcl_lock)PyThread_release_lock(tcl_lock); } + tcl_tstate = NULL; RELEASE_TCL_LOCK; } #define ENTER_PYTHON \ { PyThreadState *tstate = tcl_tstate; tcl_tstate = NULL; \ - if(tcl_lock) \ - PyThread_release_lock(tcl_lock); PyEval_RestoreThread((tstate)); } + RELEASE_TCL_LOCK; PyEval_RestoreThread((tstate)); } #define LEAVE_PYTHON \ { PyThreadState *tstate = PyEval_SaveThread(); \ - if(tcl_lock)PyThread_acquire_lock(tcl_lock, 1); tcl_tstate = tstate; } + ACQUIRE_TCL_LOCK;\ + tcl_tstate = tstate; } #define ENTER_PYTHON_OVERLAP \ { PyThreadState *tstate = tcl_tstate; tcl_tstate = NULL; PyEval_RestoreThread((tstate)); } From ccf51ece19d629de5c42b40feedc02500e8c13f6 Mon Sep 17 00:00:00 2001 From: Ivan Pozdeev Date: Sun, 29 Apr 2018 17:12:43 +0300 Subject: [PATCH 09/15] Don't release Tcl lock in Tcl custom commands --- Modules/_tkinter.c | 26 +++++++------------------- 1 file changed, 7 insertions(+), 19 deletions(-) diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c index dac4d180464bd7..c9773322eb1ae5 100644 --- a/Modules/_tkinter.c +++ b/Modules/_tkinter.c @@ -284,18 +284,9 @@ if (tcl_lock){\ tcl_tstate = NULL; RELEASE_TCL_LOCK; } #define ENTER_PYTHON \ - { PyThreadState *tstate = tcl_tstate; tcl_tstate = NULL; \ - RELEASE_TCL_LOCK; PyEval_RestoreThread((tstate)); } - -#define LEAVE_PYTHON \ - { PyThreadState *tstate = PyEval_SaveThread(); \ - ACQUIRE_TCL_LOCK;\ - tcl_tstate = tstate; } - -#define ENTER_PYTHON_OVERLAP \ { PyThreadState *tstate = tcl_tstate; tcl_tstate = NULL; PyEval_RestoreThread((tstate)); } -#define LEAVE_PYTHON_OVERLAP \ +#define LEAVE_PYTHON \ { PyThreadState *tstate = PyEval_SaveThread(); tcl_tstate = tstate; } @@ -1449,17 +1440,17 @@ Tkapp_CallProc(Tkapp_CallEvent *e, int flags) Tcl_Obj **objv; int objc; int i; - ENTER_PYTHON_OVERLAP + ENTER_PYTHON objv = Tkapp_CallArgs(e->args, objStore, &objc); if (!objv) { PyErr_Fetch(e->exc_type, e->exc_value, e->exc_tb); *(e->res) = NULL; } - LEAVE_PYTHON_OVERLAP + LEAVE_PYTHON if (!objv) goto done; i = Tcl_EvalObjv(e->self->interp, objc, objv, e->flags); - ENTER_PYTHON_OVERLAP + ENTER_PYTHON if (i == TCL_ERROR) { *(e->res) = NULL; *(e->exc_type) = NULL; @@ -1471,7 +1462,7 @@ Tkapp_CallProc(Tkapp_CallEvent *e, int flags) else { *(e->res) = Tkapp_CallResult(e->self); } - LEAVE_PYTHON_OVERLAP + LEAVE_PYTHON Tkapp_CallDeallocArgs(objv, objStore, objc); done: @@ -2405,8 +2396,8 @@ PythonCmd(ClientData clientData, Tcl_Interp *interp, int argc, const char *argv[ ENTER_PYTHON /* TBD: no error checking here since we know, via the - * Tkapp_CreateCommand() that the client data is a two-tuple - */ + * Tkapp_CreateCommand() that the client data is a two-tuple + */ func = data->func; /* Create argument list (argv1, ..., argvN) */ @@ -2427,14 +2418,11 @@ PythonCmd(ClientData clientData, Tcl_Interp *interp, int argc, const char *argv[ if (res == NULL) return PythonCmd_Error(); - ENTER_TCL - ENTER_OVERLAP obj_res = AsObj(res); if (obj_res) { Tcl_SetObjResult(interp, obj_res); rv = TCL_OK; } - LEAVE_OVERLAP_TCL if (obj_res == NULL) { Py_DECREF(res); return PythonCmd_Error(); From 10c6711485df0678ecf41c51725dc73ab92daa0a Mon Sep 17 00:00:00 2001 From: Ivan Pozdeev Date: Wed, 16 May 2018 08:35:01 +0300 Subject: [PATCH 10/15] Document locking changes for handlers --- Modules/_tkinter.c | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c index c9773322eb1ae5..ca79f5ebc6ea60 100644 --- a/Modules/_tkinter.c +++ b/Modules/_tkinter.c @@ -190,15 +190,14 @@ _get_tcl_lib_path() By contrast, ENTER_PYTHON and LEAVE_PYTHON are used in Tcl event handlers when the handler needs to use Python. Such event handlers are entered while the lock for Tcl is held; the event handler - presumably needs to use Python. ENTER_PYTHON releases the lock for - Tcl and acquires the Python interpreter lock, restoring the - appropriate thread state, and LEAVE_PYTHON releases the Python - interpreter lock and re-acquires the lock for Tcl. It is okay for - ENTER_TCL/LEAVE_TCL pairs to be contained inside the code between - ENTER_PYTHON and LEAVE_PYTHON. - - These locks expand to several statements and brackets; they should - not be used in branches of if statements and the like. + presumably needs to use Python. ENTER_PYTHON acquires the Python + interpreter lock, restoring the appropriate thread state, and + LEAVE_PYTHON releases the Python interpreter lock. Tcl lock is not + released because that would lead to a race condition: another, + unrelated call that uses these macros may start (but not finish) in + the meantime, and a wrong Tcl stack frame will be on top when the original + handler regains the lock. Since a handler's Python payload may need to make + Tkinter calls itself, the Tcl lock is made reentrant. If Tcl is threaded, this approach won't work anymore. The Tcl interpreter is only valid in the thread that created it, and all Tk From 41ac64c9b7d240c7771450a3ed7f5fe8fcdfb61a Mon Sep 17 00:00:00 2001 From: Ivan Pozdeev Date: Thu, 17 May 2018 14:52:58 +0300 Subject: [PATCH 11/15] cosmetic changes for readability --- Modules/_tkinter.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c index ca79f5ebc6ea60..2694cd59c2c82c 100644 --- a/Modules/_tkinter.c +++ b/Modules/_tkinter.c @@ -180,8 +180,8 @@ _get_tcl_lib_path() Sometimes, it is necessary to have both the Python lock and the Tcl lock. (For example, when transferring data between the Tcl interpreter and/or - objects and Python objects.) To avoid deadlocks, when acquiring, we always - acquire the Tcl lock first, then the Python lock. The additional macros for + objects and Python objects.) To avoid deadlocks, when acquiring, we always + acquire the Tcl lock first, then the Python lock. The additional macros for finer lock control are: ENTER_OVERLAP acquires the Python lock (and restores the thread state) when already holding the Tcl lock; LEAVE_OVERLAP releases the Python lock and keeps the Tcl lock; and LEAVE_OVERLAP_TCL releases the @@ -222,6 +222,7 @@ static PyThread_type_lock tcl_lock = 0; static unsigned long tcl_lock_thread_ident = 0; static unsigned int tcl_lock_reentry_count = 0; + #ifdef TCL_THREADS static Tcl_ThreadDataKey state_key; typedef PyThreadState *ThreadSpecificData; @@ -231,9 +232,10 @@ typedef PyThreadState *ThreadSpecificData; static PyThreadState *tcl_tstate = NULL; #endif + #define ACQUIRE_TCL_LOCK \ if (tcl_lock) {\ - if (tcl_lock_thread_ident == PyThread_get_thread_ident()) {\ + if (tcl_lock_thread_ident == PyThread_get_thread_ident()) { \ tcl_lock_reentry_count++; \ if(!tcl_lock_reentry_count) \ Py_FatalError("Tcl lock reentry count overflow"); \ @@ -253,12 +255,13 @@ if (tcl_lock){\ }\ } + #define ENTER_TCL \ - { PyThreadState *tstate = PyThreadState_Get();\ + { PyThreadState *tstate = PyThreadState_Get(); \ ENTER_TCL_CUSTOM_TSTATE(tstate) #define ENTER_TCL_CUSTOM_TSTATE(tstate) \ - Py_BEGIN_ALLOW_THREADS\ + Py_BEGIN_ALLOW_THREADS \ ACQUIRE_TCL_LOCK; tcl_tstate = tstate; #define LEAVE_TCL \ From 534c57d47e35871deff459dd2f593f436e2605e7 Mon Sep 17 00:00:00 2001 From: Ivan Pozdeev Date: Wed, 16 May 2018 19:16:09 +0300 Subject: [PATCH 12/15] replace test case --- Lib/tkinter/test/test_tkinter/test_threads.py | 131 +++++------------- 1 file changed, 37 insertions(+), 94 deletions(-) diff --git a/Lib/tkinter/test/test_tkinter/test_threads.py b/Lib/tkinter/test/test_tkinter/test_threads.py index 2dd18fa0f2b5f4..1d0b05e66bde45 100644 --- a/Lib/tkinter/test/test_tkinter/test_threads.py +++ b/Lib/tkinter/test/test_tkinter/test_threads.py @@ -1,10 +1,10 @@ -#!/usr/bin/python3 - +from __future__ import print_function +import tkinter +import random +import string import sys -import time import threading -import random -import tkinter +import time import unittest class TestThreads(unittest.TestCase): @@ -31,103 +31,46 @@ def test_threads(self): running = True -class Track(threading.Thread): - """ - Calculates coordinates for a ballistic track - with random angle and velocity - and fires the callback to draw each consecutive segment. - """ - def __init__(self, track_number, draw_callback): - """ - :param track_number: ordinal - :param draw_callback: fn(track_number, x, y) - that draws the extention of the specified track - to the specified point. The callback must keep track - of any previous coordinates itself. - """ - threading.Thread.__init__(self) - #self.setDaemon(True) - self.track_number = track_number - self.draw_callback = draw_callback - +class EventThread(threading.Thread): + def __init__(self,target): + super(EventThread,self).__init__() + self.target = target def run(self): - #starting point for the track - y = 0.0001 #height - x = 999.0 #range - #initial velocities - yVel = 400. + random.random() * 200. - xVel = -200. - random.random() * 150. - # Stop drawing when the track hits the ground - while y > 0 and running: - #How long to sleep, in seconds, between track updates - time.sleep(0.01) #realism: >1 Fun: 0.01 - - yVel -= 9.8 #gravity, more or less - xVel *= .9998 #air resistance - - y += yVel - x += xVel - - if running: - self.draw_callback(self.track_number, x, y) + while running: + time.sleep(0.02) + c = random.choice(string.ascii_letters) + self.target.event_generate(c) +class Main(object): + def __init__(self): + self.root = tkinter.Tk() + self.root.bind('',dummy_handler) + self.threads=[] -class App: - """ - The main app logic - """ - def __init__(self, window): - self.track_no = 0 #last index of track added - self.track_coordinates = {} #coords of track tips - self.threads = [] + self.t_cleanup = threading.Thread(target=self.tf_stop) - self.window = window - self.frame = tkinter.Frame(window) - self.frame.pack() - self.graph = tkinter.Canvas(self.frame) - self.graph.pack() - - self.t_cleanup = threading.Thread(target=self.tf_cleanup) - self.window.after(0, self.add_tracks, 5) - self.window.after(1000, self.t_cleanup.start) - - def add_tracks(self,depth): - if depth>0: self.window.after(5, self.add_tracks, depth-1) - tracks_to_add = 40 - start_no = self.track_no - self.track_no += tracks_to_add - for self.track_no in range(start_no, self.track_no): - #self.window.after(t, self.add_tracks) - #self.track_no += 1 #Make a new track number - #if (self.track_no > 40): - t = Track(self.track_no, self.draw_track_segment) - self.threads.append(t) - t.start() - - def tf_cleanup(self): + def go(self): + self.root.after(0,self.add_threads) + self.root.after(500,self.stop) + self.root.protocol("WM_DELETE_WINDOW",self.stop) + self.root.mainloop() + self.t_cleanup.join() + def stop(self): + self.t_cleanup.start() + def tf_stop(self): global running running = False for t in self.threads: t.join() - self.window.after(0,self.window.destroy) + self.root.after(0,self.root.destroy) + def add_threads(self): + for _ in range(20): + t = EventThread(self.root) + self.threads.append(t) + t.start() - def draw_track_segment(self, track_no, x, y): - # x & y are virtual coordinates for the purpose of simulation. - #To convert them to screen coordinates, - # we scale them down so the graphs fit into the window, and move the origin. - # Y is also inverted because in canvas, the UL corner is the origin. - newsx, newsy = (250.+x/100., 150.-y/100.) +def dummy_handler(event): + pass - try: - (oldsx, oldsy) = self.track_coordinates[track_no] - except KeyError: - pass - else: - self.graph.create_line(oldsx, oldsy, - newsx, newsy) - self.track_coordinates[track_no] = (newsx, newsy) - - def go(self): - self.window.mainloop() tests_gui = (TestThreads,) @@ -138,7 +81,7 @@ def go(self): import ctypes #the bug causes crashes ctypes.windll.kernel32.SetErrorMode(3) - App(tkinter.Tk()).go() + Main().go() else: from test.support import run_unittest run_unittest(*tests_gui) From 0b1a8b6c580b864679db8189a224cca7e68d6a1f Mon Sep 17 00:00:00 2001 From: Ivan Pozdeev Date: Thu, 17 May 2018 22:09:05 +0300 Subject: [PATCH 13/15] code style --- Lib/tkinter/test/test_tkinter/test_threads.py | 43 ++++++++++++------- Modules/_tkinter.c | 37 ++++++++-------- 2 files changed, 46 insertions(+), 34 deletions(-) diff --git a/Lib/tkinter/test/test_tkinter/test_threads.py b/Lib/tkinter/test/test_tkinter/test_threads.py index 1d0b05e66bde45..35dc7b73f48d1c 100644 --- a/Lib/tkinter/test/test_tkinter/test_threads.py +++ b/Lib/tkinter/test/test_tkinter/test_threads.py @@ -7,16 +7,18 @@ import time import unittest + + class TestThreads(unittest.TestCase): def test_threads(self): import subprocess p = subprocess.Popen([sys.executable, __file__, "run"], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) self.addCleanup(p.stdout.close) self.addCleanup(p.stderr.close) try: - #Test code is designed to complete in a few seconds + # Test code is designed to complete in a few seconds stdout, stderr = p.communicate(timeout=10) except subprocess.TimeoutExpired: p.kill() @@ -28,60 +30,69 @@ def test_threads(self): self.assertTrue(len(stderr) == 0, "stderr: " + repr(stderr)) - running = True + class EventThread(threading.Thread): - def __init__(self,target): - super(EventThread,self).__init__() + def __init__(self, target): + super(EventThread, self).__init__() self.target = target + def run(self): while running: time.sleep(0.02) c = random.choice(string.ascii_letters) self.target.event_generate(c) + class Main(object): def __init__(self): self.root = tkinter.Tk() - self.root.bind('',dummy_handler) - self.threads=[] + self.root.bind('', dummy_handler) + self.threads = [] self.t_cleanup = threading.Thread(target=self.tf_stop) def go(self): - self.root.after(0,self.add_threads) - self.root.after(500,self.stop) - self.root.protocol("WM_DELETE_WINDOW",self.stop) + self.root.after(0, self.add_threads) + self.root.after(500, self.stop) + self.root.protocol("WM_DELETE_WINDOW", self.stop) self.root.mainloop() self.t_cleanup.join() + def stop(self): self.t_cleanup.start() + def tf_stop(self): global running running = False for t in self.threads: t.join() - self.root.after(0,self.root.destroy) + self.root.after(0, self.root.destroy) + def add_threads(self): for _ in range(20): t = EventThread(self.root) self.threads.append(t) t.start() + def dummy_handler(event): pass tests_gui = (TestThreads,) -if __name__=='__main__': - import sys,os - if sys.argv[1:]==['run']: +if __name__ == '__main__': + import sys, os + + if sys.argv[1:] == ['run']: if os.name == 'nt': import ctypes - #the bug causes crashes + + # the bug causes crashes ctypes.windll.kernel32.SetErrorMode(3) Main().go() else: from test.support import run_unittest + run_unittest(*tests_gui) diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c index 2694cd59c2c82c..bdda1d4f2f3d77 100644 --- a/Modules/_tkinter.c +++ b/Modules/_tkinter.c @@ -234,7 +234,7 @@ static PyThreadState *tcl_tstate = NULL; #define ACQUIRE_TCL_LOCK \ -if (tcl_lock) {\ +if (tcl_lock) { \ if (tcl_lock_thread_ident == PyThread_get_thread_ident()) { \ tcl_lock_reentry_count++; \ if(!tcl_lock_reentry_count) \ @@ -242,17 +242,17 @@ if (tcl_lock) {\ } else { \ PyThread_acquire_lock(tcl_lock, 1); \ tcl_lock_thread_ident = PyThread_get_thread_ident(); \ - }\ + } \ } #define RELEASE_TCL_LOCK \ -if (tcl_lock){\ +if (tcl_lock) { \ if (tcl_lock_reentry_count) { \ tcl_lock_reentry_count--; \ } else { \ tcl_lock_thread_ident = 0; \ PyThread_release_lock(tcl_lock); \ - }\ + } \ } @@ -261,8 +261,9 @@ if (tcl_lock){\ ENTER_TCL_CUSTOM_TSTATE(tstate) #define ENTER_TCL_CUSTOM_TSTATE(tstate) \ - Py_BEGIN_ALLOW_THREADS \ - ACQUIRE_TCL_LOCK; tcl_tstate = tstate; + Py_BEGIN_ALLOW_THREADS \ + ACQUIRE_TCL_LOCK \ + tcl_tstate = tstate; #define LEAVE_TCL \ tcl_tstate = NULL; \ @@ -270,9 +271,9 @@ if (tcl_lock){\ #define LEAVE_TCL_WITH_BUSYWAIT(result) \ tcl_tstate = NULL; \ - RELEASE_TCL_LOCK;\ - if (result == 0)\ - Sleep(Tkinter_busywaitinterval);\ + RELEASE_TCL_LOCK \ + if (result == 0) \ + Sleep(Tkinter_busywaitinterval); \ Py_END_ALLOW_THREADS @@ -283,7 +284,7 @@ if (tcl_lock){\ Py_BEGIN_ALLOW_THREADS #define LEAVE_OVERLAP_TCL \ - tcl_tstate = NULL; RELEASE_TCL_LOCK; } + tcl_tstate = NULL; RELEASE_TCL_LOCK } #define ENTER_PYTHON \ { PyThreadState *tstate = tcl_tstate; tcl_tstate = NULL; PyEval_RestoreThread((tstate)); } @@ -1844,7 +1845,7 @@ SetVar(PyObject *self, PyObject *args, int flags) if (newval) { LEAVE_OVERLAP ok = Tcl_SetVar2Ex(Tkapp_Interp(self), name1, NULL, - newval, flags); + newval, flags); ENTER_OVERLAP if (!ok) Tkinter_Error(self); @@ -1867,7 +1868,7 @@ SetVar(PyObject *self, PyObject *args, int flags) if (newval) { LEAVE_OVERLAP ok = Tcl_SetVar2Ex(Tkapp_Interp(self), name1, name2, - newval, flags); + newval, flags); ENTER_OVERLAP if (!ok) Tkinter_Error(self); @@ -2398,8 +2399,8 @@ PythonCmd(ClientData clientData, Tcl_Interp *interp, int argc, const char *argv[ ENTER_PYTHON /* TBD: no error checking here since we know, via the - * Tkapp_CreateCommand() that the client data is a two-tuple - */ + * Tkapp_CreateCommand() that the client data is a two-tuple + */ func = data->func; /* Create argument list (argv1, ..., argvN) */ @@ -2421,14 +2422,14 @@ PythonCmd(ClientData clientData, Tcl_Interp *interp, int argc, const char *argv[ return PythonCmd_Error(); obj_res = AsObj(res); - if (obj_res) { - Tcl_SetObjResult(interp, obj_res); - rv = TCL_OK; - } if (obj_res == NULL) { Py_DECREF(res); return PythonCmd_Error(); } + else { + Tcl_SetObjResult(interp, obj_res); + rv = TCL_OK; + } Py_DECREF(res); From c50090835a6b3a5760bb9e987de2f6da9df152dc Mon Sep 17 00:00:00 2001 From: Ivan Pozdeev Date: Fri, 18 May 2018 05:09:07 +0300 Subject: [PATCH 14/15] alter NEWS to include the 2nd fix --- .../next/Library/2018-04-14-01-26-10.bpo-33257.i32qOW.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2018-04-14-01-26-10.bpo-33257.i32qOW.rst b/Misc/NEWS.d/next/Library/2018-04-14-01-26-10.bpo-33257.i32qOW.rst index d8611ec82b046f..b061786ae808af 100644 --- a/Misc/NEWS.d/next/Library/2018-04-14-01-26-10.bpo-33257.i32qOW.rst +++ b/Misc/NEWS.d/next/Library/2018-04-14-01-26-10.bpo-33257.i32qOW.rst @@ -1 +1 @@ -Fixed race conditions in Tkinter when passing data to/from nonthreaded Tcl +Fixed race conditions in Tkinter with nonthreaded Tcl From 90e73a978a25471e232944653e89da52fa5ee90c Mon Sep 17 00:00:00 2001 From: Ivan Pozdeev Date: Sun, 6 Apr 2025 18:10:07 +0300 Subject: [PATCH 15/15] lint error fix --- Modules/_tkinter.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c index 9de2f155b6455a..6ef80f4bc81f7d 100644 --- a/Modules/_tkinter.c +++ b/Modules/_tkinter.c @@ -1590,7 +1590,7 @@ Tkapp_Call(PyObject *selfptr, PyObject *args) Tkapp_CallDeallocArgs(objv, objStore, objc); ENTER_OVERLAP - + } LEAVE_OVERLAP_TCL }