libpython3.11.so.1.0 segv using mariadb connection pool #128934
Description
Crash report
What happened?
Using the mariadb package for python, I am getting occasional segv which seems to be related to running out of connections from the connection pool a number of times.
A couple of examples of output:
Fatal Python error: PyEval_SaveThread: the function must be called with the GIL held, but the GIL is released (the current Python thread state is NULL)
Python runtime state: initialized
Thread 0x00007fffe3fff640 (most recent call first):
File sqldb execute error: No connection available SELECT * FROM table1 ORDER BY id DESC LIMIT 10
"/home/ec2-user/venv/lib64/python3.11/site-packages/mariadb/cursors.py"got exception No connection available
, line 310 in execute
File "/home/ec2-user/test_sqldb_segv-bug.py", line 74 in execute
File "/home/ec2-user/test_sqldb_segv-bug.py", line 117 in test_sqldb1
File "/usr/lib64/python3.11/threading.py", line 982 in run
File "/usr/lib64/python3.11/threading.py", line 1045 in _bootstrap_inner
File "/usr/lib64/python3.11/threading.py", line 1002 in _bootstrap
Thread 0x00007fffe8ccb640 (most recent call first):
File "/home/ec2-user/test_sqldb_segv-bug.py", line 127 in test_sqldb1
File "/usr/lib64/python3.11/threading.py", line 982 in run
File "/usr/lib64/python3.11/threading.py", line 1045 in _bootstrap_inner
File "/usr/lib64/python3.11/threading.py", line 1002 in _bootstrap
Current thread 0x00007fffe96cc640 (most recent call first):
File "/home/ec2-user/test_sqldb_segv-bug.py", line 107 in test_sqldb_insert
File "/usr/lib64/python3.11/threading.py", line 982 in run
File "/usr/lib64/python3.11/threading.py", line 1045 in _bootstrap_inner
File "/usr/lib64/python3.11/threading.py", line 1002 in _bootstrap
Thread 0x00007ffff7fb7740 (most recent call first):
File "/usr/lib64/python3.11/threading.py", line 1590 in _shutdown
Extension modules: mariadb._mariadb (total: 1)
Thread 2 "python3" received signal SIGABRT, Aborted.
[Switching to Thread 0x7fffe96cc640 (LWP 42106)]
0x00007ffff76a157c in __pthread_kill_implementation () from /lib64/libc.so.6
Missing separate debuginfos, use: dnf debuginfo-install glibc-2.34-117.amzn2023.0.1.x86_64 libuuid-2.37.4-1.amzn2023.0.4.x86_64 mpdecimal-2.5.1-3.amzn2023.0.3.x86_64 openssl-libs-3.0.8-1.amzn2023.0.18.x86_64 zlib-1.2.11-33.amzn2023.0.5.x86_64
(gdb)
Other try outputs:
hread 3 "python3" received signal SIGSEGV, Segmentation fault.
[Switching to Thread 0x7fffe8ccb640 (LWP 42274)]
method_vectorcall_NOARGS (func=<method_descriptor at remote 0x7fffe9d3bbf0>, args=0x7ffff7538358, nargsf=<optimized out>, kwnames=0x0) at /usr/src/debug/python3.11-3.11.6-1.amzn2023.0.5.x86_64/Objects/descrobject.c:449
449 PyCFunction meth = (PyCFunction)method_enter_call(tstate, func);
(gdb) info locals
tstate = 0x0
nargs = 1
meth = <optimized out>
result = <optimized out>
(gdb)
import os
import sys
import time
import uuid
from threading import Thread
import mariadb
###
#
# The SEGV seems to occur when we hit limit of connections from pool a number of times
#
# Run this script with 2 threads and 100ms sleep :
#
# python3 test_sqldb_segv-bug.py 2 100
#
# Also tried using python 3.12.4, this also segvs
#
###
DEBUG=0
POOL_NUM=1
def debug(s, *args, **kwargs):
if DEBUG:
print(s, *args, **kwargs)
pool = mariadb.ConnectionPool(pool_name="sqldb", pool_size=5)
pool.set_config(
user="test",
password="password",
host="127.0.0.1",
database="TEST",
ssl=False,
autocommit=True,
)
for num in range(POOL_NUM):
try:
conn = mariadb.connect(
user="test",
password="password",
host="127.0.0.1",
database="TEST",
ssl=False,
)
conn.autocommit = True
conn.auto_reconnect = True
pool.add_connection(conn)
print(f"startup - added pool {conn}")
except mariadb.PoolError:
print("got Pool error")
def get_cursor(_id):
try:
conn = pool.get_connection()
except mariadb.PoolError:
raise
except Exception as exc:
print(f"get_cursor: exception {exc}")
cur = conn.cursor(dictionary=False, buffered=True)
return cur
def execute(_id, sql, *args, **kwargs):
rows = []
try:
cur = get_cursor(_id)
except Exception as exc:
print(f"sqldb execute error: {exc} {sql}")
raise exc
try:
res = cur.execute(sql, *args, **kwargs)
if cur.description:
try:
rows = cur.fetchall()
except mariadb.Error as exc:
# probably "Cursor doesn't have a result set" which is OK for tasks like INSERT
if str(exc) != "Cursor doesn't have a result set":
print(f"sqldb cursor fetchall error: {exc}")
except Exception as exc:
print(f"sqldb execute error: {exc} {sql}")
cur.connection.close()
raise exc
cur.connection.close()
return rows
def test_sqldb_insert(_id, sleep_time):
print("start insert")
num = 0
while True:
debug("I",end='',flush=True)
try:
res = execute(_id, "INSERT INTO table1 (name, uid, call_from, clientid) VALUES (?,?,?,?)", (num, str(uuid.uuid1()), num, str(uuid.uuid1())))
except Exception as exc:
print(f"got exception {exc}")
num += 1
time.sleep(0.2)
def test_sqldb1(_id, sleep_time):
print(f"start {_id}")
while True:
debug(f".",end='',flush=True)
try:
res = execute(_id, "SELECT * FROM table1 ORDER BY id DESC LIMIT 10")
debug(f"{_id}",end='',flush=True)
if res:
for x in res:
pass
except Exception as exc:
print(f"got exception {exc}")
time.sleep(sleep_time)
if __name__ == "__main__":
if len(sys.argv) < 3:
print(f"Usage: {sys.argv[0]} <num threads> <sleep ms>")
sys.exit(1)
num_threads = int(sys.argv[1])
sleep_time = int(sys.argv[2]) / 1000
try:
res = execute('99', "CREATE TABLE IF NOT EXISTS table1 (id INT auto_increment primary key, name VARCHAR(255), uid VARCHAR(36), call_from VARCHAR(36), clientid VARCHAR(36)) CHARACTER SET 'utf8'")
except Exception as exc:
print(f"got exception {exc}")
tid = Thread(
target=test_sqldb_insert,
args=(99, sleep_time),
)
print(f"starting {99}")
tid.start()
for thread_id in range(num_threads):
tid = Thread(
target=test_sqldb1,
args=(thread_id, sleep_time),
)
print(f"starting {thread_id}")
tid.start()
This was using an AMI2023 on AWS install script is
#!/bin/bash
# This is for an AMI 2023 on AWS
sudo yum install -y pip
sudo yum install -y python3-devel python3-debug
sudo yum install -y python3.11-devel python3.11-debug python3.11-pip
sudo yum install -y python3-virtualenv
sudo yum install -y gdb
python3.11 -m venv ${HOME}/venv
source ${HOME}/venv/bin/activate
pip3.11 install --upgrade pip
sudo dnf install -y mariadb105 mariadb105-server mariadb-connector-c mariadb-connector-c-devel
sudo systemctl enable mariadb
sudo systemctl start mariadb
wget https://dlm.mariadb.com/3677127/Connectors/c/connector-c-3.3.8/mariadb-connector-c-3.3.8-src.tar.gz
tar zxf mariadb-connector-c-3.3.8-src.tar.gz
cd mariadb-connector-c-3.3.8-src
mkdir build
cd build
sudo yum install -y cmake
cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local
make
sudo make install
cd ../..
pip3.11 install mariadb
# Ok messy, the installation of mariadb requires the old mariadb-connector-c to be there, but the new mariadb python package will need to access new connector, so going to hijak softlink
pushd /lib64
sudo rm libmariadb.so.3
sudo ln -s /usr/local/lib/mariadb/libmariadb.so.3 libmariadb.so.3
popd
sudo mysqladmin password "password"
sudo mysql --user=root <<_EOF_
DELETE FROM mysql.user WHERE User='';
DELETE FROM mysql.user WHERE User='root' AND Host NOT IN ('localhost', '127.0.0.1', '::1');
DROP DATABASE IF EXISTS test;
DELETE FROM mysql.db WHERE Db='test' OR Db='test\\_%';
CREATE USER IF NOT EXISTS 'test'@'localhost' IDENTIFIED BY 'password';
GRANT ALL PRIVILEGES ON *.* TO 'test'@'localhost' IDENTIFIED BY 'password';
FLUSH PRIVILEGES;
CREATE DATABASE TEST;
_EOF_
# For debuginfo in gdb
sudo dnf debuginfo-install -y python3.11-3.11.6-1.amzn2023.0.5.x86_64
sudo dnf debuginfo-install -y glibc-2.34-117.amzn2023.0.1.x86_64 libuuid-2.37.4-1.amzn2023.0.4.x86_64 mpdecimal-2.5.1-3.amzn2023.0.3.x86_64 openssl-libs-3.0.8-1.amzn2023.0.18.x86_64 zlib-1.2.11-33.amzn2023.0.5.x86_64
CPython versions tested on:
3.11, 3.12
Operating systems tested on:
Linux
Output from running 'python -VV' on the command line:
Python 3.11.6 (main, Nov 6 2024, 00:00:00) [GCC 11.4.1 20230605 (Red Hat 11.4.1-2)]