Skip to content

Commit cbf32f0

Browse files
frazerclementfrazerclement
committed
Bug#37023661 (1/3) Use TCRELEASEREQ to force release in scan close() timeout cases
Backport to 7.6 New testcase showing leaks in key access timeout cases. Where the API protocol timeout elapses, key operation execution (primary key, unique key) will mark the NdbTransaction object involved as 'ReleaseOnClose' and this will result in a leaked ApiConnectRecord at TC. New test ndb_key_protocol_timeout shows these leaks for such operations. Change-Id: I83f2032c88aab295c9621364bf52fff1221561a3 Bug#37023661 (2/3) Use TCRELEASEREQ to force release in scan close() timeout cases Use TCRELEASEREQ to force release of kernel side when Scan close times out This patch adds a new option to force release in the kernel if a normal close times out. This involves sending a TCRELEASEREQ signal to TC which will handle the ApiConnectRecord as though there were a node failure. The API side record will be free to be reused, but for a different ApiConnectRecord. This removes the leaks observed in the 'timeout due to bug' case where the close request does not complete. Modifications : - Add theForceReleaseOnClose flag - Set theForceReleaseOnClose flag : - In ordered/unordered NdbApi scan close_impl() timeout path - In SPJ scan close timeout path - Handle theForceReleaseOnClose flag in NdbTransaction.cpp - Map to function sending TCRELEASEREQ to TC as part of releasing the NdbTransaction object The ndb_scan_protocol_timeout testcase shows the effect of the change, where the 'BUG' scenario leaks are removed. Change-Id: Ie708db32f95b9d4649dc2c8640bf6e2bbabf2097 Bug#37023661 (3/3) Use TCRELEASEREQ to force release in scan close() timeout cases Use theForceReleaseOnClose mechanism for key op timeouts Also make use of release on close for handling key op timeouts. Note that key op timeouts already send a rollback request which is similar in intent to the force close with TCRELEASEREQ. Testcase ndb_key_protocol_timeout result is updated to show the ApiConnectRecord leaks removed by this change. Change-Id: Ieffe8a3197c239ad3648c56d0136d25862b291d3
1 parent 0d50abb commit cbf32f0

File tree

10 files changed

+378
-17
lines changed

10 files changed

+378
-17
lines changed

‎mysql-test/suite/ndb/r/ndb_key_protocol_timeout.result‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
# We are using some debug-only features in this test
2+
call mtr.add_suppression("Receive from NDB failed");
3+
Create some tables suitable for executing operations on
4+
-------------------------------------------------------
5+
use test;
6+
create table test.t1 (a int primary key, b int, key(b), c int, unique(c)) engine=ndb;
7+
Insert 40 rows
8+
insert into test.t1 values (0, 0, 0);;
9+
insert into test.t1 values (1, 1, 1);;
10+
insert into test.t1 values (2, 2, 2);;
11+
insert into test.t1 values (3, 3, 3);;
12+
insert into test.t1 values (4, 4, 4);;
13+
insert into test.t1 values (5, 5, 5);;
14+
insert into test.t1 values (6, 6, 6);;
15+
insert into test.t1 values (7, 7, 7);;
16+
insert into test.t1 values (8, 8, 8);;
17+
insert into test.t1 values (9, 9, 9);;
18+
insert into test.t1 values (10, 10, 10);;
19+
insert into test.t1 values (11, 11, 11);;
20+
insert into test.t1 values (12, 12, 12);;
21+
insert into test.t1 values (13, 13, 13);;
22+
insert into test.t1 values (14, 14, 14);;
23+
insert into test.t1 values (15, 15, 15);;
24+
insert into test.t1 values (16, 16, 16);;
25+
insert into test.t1 values (17, 17, 17);;
26+
insert into test.t1 values (18, 18, 18);;
27+
insert into test.t1 values (19, 19, 19);;
28+
insert into test.t1 values (20, 20, 20);;
29+
insert into test.t1 values (21, 21, 21);;
30+
insert into test.t1 values (22, 22, 22);;
31+
insert into test.t1 values (23, 23, 23);;
32+
insert into test.t1 values (24, 24, 24);;
33+
insert into test.t1 values (25, 25, 25);;
34+
insert into test.t1 values (26, 26, 26);;
35+
insert into test.t1 values (27, 27, 27);;
36+
insert into test.t1 values (28, 28, 28);;
37+
insert into test.t1 values (29, 29, 29);;
38+
insert into test.t1 values (30, 30, 30);;
39+
insert into test.t1 values (31, 31, 31);;
40+
insert into test.t1 values (32, 32, 32);;
41+
insert into test.t1 values (33, 33, 33);;
42+
insert into test.t1 values (34, 34, 34);;
43+
insert into test.t1 values (35, 35, 35);;
44+
insert into test.t1 values (36, 36, 36);;
45+
insert into test.t1 values (37, 37, 37);;
46+
insert into test.t1 values (38, 38, 38);;
47+
insert into test.t1 values (39, 39, 39);;
48+
Take row locks in other session
49+
begin;
50+
select * from test.t1 where a < 10 order by a for update;
51+
a b c
52+
0 0 0
53+
1 1 1
54+
2 2 2
55+
3 3 3
56+
4 4 4
57+
5 5 5
58+
6 6 6
59+
7 7 7
60+
8 8 8
61+
9 9 9
62+
set @save_debug = @@session.debug;
63+
SET SESSION debug="+d,ndb_reduced_api_protocol_timeout";
64+
----------------------------
65+
Autocommit PK SELECT timeout
66+
----------------------------
67+
select * from test.t1 where a=0 for update;
68+
ERROR HY000: Got error 4012 'Request ndbd time-out, maybe due to high load or communication problems' from NDBCLUSTER
69+
Check pk lookups
70+
Check transaction leaks
71+
Leaks
72+
0
73+
select count(1) as ops from ndbinfo.cluster_operations;
74+
ops
75+
10
76+
--------------------------------------
77+
Stateful transaction PK SELECT timeout
78+
--------------------------------------
79+
begin;
80+
insert into test.t1 values (100,100,100);
81+
select * from test.t1 where a=2 for update;
82+
ERROR HY000: Got error 4012 'Request ndbd time-out, maybe due to high load or communication problems' from NDBCLUSTER
83+
Check pk lookups
84+
Check transaction leaks
85+
Leaks
86+
0
87+
select count(1) as ops from ndbinfo.cluster_operations;
88+
ops
89+
10
90+
----------------------------
91+
Autocommit UK SELECT timeout
92+
----------------------------
93+
select * from test.t1 where c=0 for update;
94+
ERROR HY000: Got error 4012 'Request ndbd time-out, maybe due to high load or communication problems' from NDBCLUSTER
95+
Check pk lookups
96+
Check transaction leaks
97+
Leaks
98+
0
99+
select count(1) as ops from ndbinfo.cluster_operations;
100+
ops
101+
10
102+
--------------------------------------
103+
Stateful transaction UK SELECT timeout
104+
--------------------------------------
105+
begin;
106+
insert into test.t1 values (100,100,100);
107+
select * from test.t1 where c=2 for update;
108+
ERROR HY000: Got error 4012 'Request ndbd time-out, maybe due to high load or communication problems' from NDBCLUSTER
109+
Check pk lookups
110+
Check transaction leaks
111+
Leaks
112+
0
113+
select count(1) as ops from ndbinfo.cluster_operations;
114+
ops
115+
10
116+
rollback;
117+
SET SESSION debug=@save_debug;
118+
drop table test.t1;

‎mysql-test/suite/ndb/r/ndb_scan_protocol_timeout.result‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -222,7 +222,7 @@ ERROR HY000: Got error 4008 'Receive from NDB failed' from NDBCLUSTER
222222
Check pk lookups
223223
Check transaction leaks
224224
Leaks
225-
1
225+
0
226226
select count(1) as ops from ndbinfo.cluster_operations;
227227
ops
228228
0
@@ -238,7 +238,7 @@ ERROR HY000: Got error 4008 'Receive from NDB failed' from NDBCLUSTER
238238
Check pk lookups
239239
Check transaction leaks
240240
Leaks
241-
1
241+
0
242242
select count(1) as ops from ndbinfo.cluster_operations;
243243
ops
244244
0
@@ -284,7 +284,7 @@ ERROR HY000: Got error 4008 'Receive from NDB failed' from NDBCLUSTER
284284
Check pk lookups
285285
Check transaction leaks
286286
Leaks
287-
1
287+
0
288288
select count(1) as ops from ndbinfo.cluster_operations;
289289
ops
290290
0
@@ -300,7 +300,7 @@ ERROR HY000: Got error 4008 'Receive from NDB failed' from NDBCLUSTER
300300
Check pk lookups
301301
Check transaction leaks
302302
Leaks
303-
1
303+
0
304304
select count(1) as ops from ndbinfo.cluster_operations;
305305
ops
306306
0
@@ -347,7 +347,7 @@ Check pk lookups
347347
Clear error condition
348348
Check transaction leaks
349349
Leaks
350-
1
350+
0
351351
select count(1) as ops from ndbinfo.cluster_operations;
352352
ops
353353
0
@@ -364,7 +364,7 @@ Check pk lookups
364364
Clear error condition
365365
Check transaction leaks
366366
Leaks
367-
1
367+
0
368368
select count(1) as ops from ndbinfo.cluster_operations;
369369
ops
370370
0

‎mysql-test/suite/ndb/t/ndb_key_protocol_timeout.cnf‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
!include suite/ndb/my.cnf
2+
3+
[cluster_config.1]
4+
# Increase TDDT to avoid it interfering with
5+
# API side timeout + cleanup handling
6+
TransactionDeadlockDetectionTimeout = 60000
7+
8+

‎mysql-test/suite/ndb/t/ndb_key_protocol_timeout.test‎

Lines changed: 205 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,205 @@
1+
-- source include/have_multi_ndb.inc
2+
--echo # We are using some debug-only features in this test
3+
-- source include/have_debug.inc
4+
-- source have_ndb_error_insert.inc
5+
-- source include/have_query_cache_disabled.inc
6+
7+
8+
#
9+
# Test of key protocol timeout behaviour
10+
#
11+
# Checking behaviour of system over key (PK/UK) request
12+
# timeouts
13+
#
14+
# This test is testing the behaviour of the system
15+
# when timeouts occur at the NdbApi level
16+
#
17+
# Multiple error insert mechanisms are used for testing
18+
#
19+
# API side : ndb_reduced_api_protocol_timeout
20+
# This DBUG option causes the NdbApi
21+
# timeout to be set to a low value
22+
# (currently 2000 millis) so that blocking
23+
# NdbApi calls typically return in
24+
# 3 * 2 = 6s
25+
#
26+
# Kernel side :
27+
#
28+
# Key protocol timeout is expected to mostly be due
29+
# to bugs
30+
#
31+
# A .cnf file is used to avoid TDDT getting in the
32+
# way of API timeout testing
33+
34+
# STABILISATION_SECS is time taken for error insert
35+
# cancellation to cause delayed signal loops to end
36+
# and system to reach a stable state
37+
#
38+
let STABILISATION_SECS=4;
39+
40+
call mtr.add_suppression("Receive from NDB failed");
41+
42+
--echo Create some tables suitable for executing operations on
43+
--echo -------------------------------------------------------
44+
use test;
45+
create table test.t1 (a int primary key, b int, key(b), c int, unique(c)) engine=ndb;
46+
47+
--echo Insert 40 rows
48+
--let $keycount=40
49+
--let $i = 0
50+
while ($i < $keycount)
51+
{
52+
--eval insert into test.t1 values ($i, $i, $i);
53+
--inc $i
54+
}
55+
56+
--echo Take row locks in other session
57+
--connection server2
58+
begin;
59+
select * from test.t1 where a < 10 order by a for update;
60+
61+
--connection server1
62+
63+
# Track # Api Connection records allocated
64+
--source ndb_get_api_connect_count.inc
65+
--let $start_api_conn_count = $api_conn_count
66+
67+
# Cause reduced API timeout
68+
set @save_debug = @@session.debug;
69+
SET SESSION debug="+d,ndb_reduced_api_protocol_timeout";
70+
71+
--echo ----------------------------
72+
--echo Autocommit PK SELECT timeout
73+
--echo ----------------------------
74+
75+
--error 1296
76+
select * from test.t1 where a=0 for update;
77+
78+
# Checking pk lookups checks that all usable ApiConnectRecords can
79+
# be used for a different transaction
80+
--echo Check pk lookups
81+
--disable_query_log
82+
--disable_result_log
83+
--let $i=0
84+
while ($i < $keycount)
85+
{
86+
--eval select * from test.t1 where a=$i;
87+
--inc $i
88+
}
89+
--enable_result_log
90+
--enable_query_log
91+
92+
--echo Check transaction leaks
93+
--source ndb_get_api_connect_count.inc
94+
--disable_query_log
95+
eval select $api_conn_count - $start_api_conn_count as Leaks;
96+
--let $start_api_conn_count = $api_conn_count
97+
--enable_query_log
98+
select count(1) as ops from ndbinfo.cluster_operations;
99+
100+
--echo --------------------------------------
101+
--echo Stateful transaction PK SELECT timeout
102+
--echo --------------------------------------
103+
104+
begin;
105+
insert into test.t1 values (100,100,100);
106+
--error 1296
107+
select * from test.t1 where a=2 for update;
108+
109+
# Checking pk lookups checks that all usable ApiConnectRecords can
110+
# be used for a different transaction
111+
--echo Check pk lookups
112+
--disable_query_log
113+
--disable_result_log
114+
--let $i=0
115+
while ($i < $keycount)
116+
{
117+
--eval select * from test.t1 where a=$i;
118+
--inc $i
119+
}
120+
--enable_result_log
121+
--enable_query_log
122+
123+
--echo Check transaction leaks
124+
--source ndb_get_api_connect_count.inc
125+
--disable_query_log
126+
eval select $api_conn_count - $start_api_conn_count as Leaks;
127+
--let $start_api_conn_count = $api_conn_count
128+
--enable_query_log
129+
select count(1) as ops from ndbinfo.cluster_operations;
130+
131+
132+
--echo ----------------------------
133+
--echo Autocommit UK SELECT timeout
134+
--echo ----------------------------
135+
136+
--error 1296
137+
select * from test.t1 where c=0 for update;
138+
139+
# Checking pk lookups checks that all usable ApiConnectRecords can
140+
# be used for a different transaction
141+
--echo Check pk lookups
142+
--disable_query_log
143+
--disable_result_log
144+
--let $i=0
145+
while ($i < $keycount)
146+
{
147+
--eval select * from test.t1 where a=$i;
148+
--inc $i
149+
}
150+
--enable_result_log
151+
--enable_query_log
152+
153+
--echo Check transaction leaks
154+
--source ndb_get_api_connect_count.inc
155+
--disable_query_log
156+
eval select $api_conn_count - $start_api_conn_count as Leaks;
157+
--let $start_api_conn_count = $api_conn_count
158+
--enable_query_log
159+
select count(1) as ops from ndbinfo.cluster_operations;
160+
161+
--echo --------------------------------------
162+
--echo Stateful transaction UK SELECT timeout
163+
--echo --------------------------------------
164+
165+
begin;
166+
insert into test.t1 values (100,100,100);
167+
--error 1296
168+
select * from test.t1 where c=2 for update;
169+
170+
# Checking pk lookups checks that all usable ApiConnectRecords can
171+
# be used for a different transaction
172+
--echo Check pk lookups
173+
--disable_query_log
174+
--disable_result_log
175+
--let $i=0
176+
while ($i < $keycount)
177+
{
178+
--eval select * from test.t1 where a=$i;
179+
--inc $i
180+
}
181+
--enable_result_log
182+
--enable_query_log
183+
184+
185+
--echo Check transaction leaks
186+
--source ndb_get_api_connect_count.inc
187+
--disable_query_log
188+
eval select $api_conn_count - $start_api_conn_count as Leaks;
189+
--let $start_api_conn_count = $api_conn_count
190+
--enable_query_log
191+
select count(1) as ops from ndbinfo.cluster_operations;
192+
193+
194+
195+
196+
197+
--connection server2
198+
rollback;
199+
200+
--connection server1
201+
202+
SET SESSION debug=@save_debug;
203+
204+
drop table test.t1;
205+

0 commit comments

Comments
 (0)