Skip to content

Commit ba5287f

Browse files
authored
Allow setting transaction limit for db connections (matrix-org#10440)
Setting the value will help PostgreSQL free up memory by recycling the connections in the connection pool. Signed-off-by: Toni Spets <[email protected]>
1 parent 2afdb5c commit ba5287f

File tree

6 files changed

+69
-0
lines changed

6 files changed

+69
-0
lines changed

changelog.d/10440.feature

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Allow setting transaction limit for database connections.

docs/sample_config.yaml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -720,6 +720,9 @@ caches:
720720
# 'name' gives the database engine to use: either 'sqlite3' (for SQLite) or
721721
# 'psycopg2' (for PostgreSQL).
722722
#
723+
# 'txn_limit' gives the maximum number of transactions to run per connection
724+
# before reconnecting. Defaults to 0, which means no limit.
725+
#
723726
# 'args' gives options which are passed through to the database engine,
724727
# except for options starting 'cp_', which are used to configure the Twisted
725728
# connection pool. For a reference to valid arguments, see:
@@ -740,6 +743,7 @@ caches:
740743
#
741744
#database:
742745
# name: psycopg2
746+
# txn_limit: 10000
743747
# args:
744748
# user: synapse_user
745749
# password: secretpassword

synapse/config/database.py

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,9 @@
3333
# 'name' gives the database engine to use: either 'sqlite3' (for SQLite) or
3434
# 'psycopg2' (for PostgreSQL).
3535
#
36+
# 'txn_limit' gives the maximum number of transactions to run per connection
37+
# before reconnecting. Defaults to 0, which means no limit.
38+
#
3639
# 'args' gives options which are passed through to the database engine,
3740
# except for options starting 'cp_', which are used to configure the Twisted
3841
# connection pool. For a reference to valid arguments, see:
@@ -53,6 +56,7 @@
5356
#
5457
#database:
5558
# name: psycopg2
59+
# txn_limit: 10000
5660
# args:
5761
# user: synapse_user
5862
# password: secretpassword

synapse/storage/database.py

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
# limitations under the License.
1616
import logging
1717
import time
18+
from collections import defaultdict
1819
from sys import intern
1920
from time import monotonic as monotonic_time
2021
from typing import (
@@ -397,6 +398,7 @@ def __init__(
397398
):
398399
self.hs = hs
399400
self._clock = hs.get_clock()
401+
self._txn_limit = database_config.config.get("txn_limit", 0)
400402
self._database_config = database_config
401403
self._db_pool = make_pool(hs.get_reactor(), database_config, engine)
402404

@@ -406,6 +408,9 @@ def __init__(
406408
self._current_txn_total_time = 0.0
407409
self._previous_loop_ts = 0.0
408410

411+
# Transaction counter: key is the twisted thread id, value is the current count
412+
self._txn_counters: Dict[int, int] = defaultdict(int)
413+
409414
# TODO(paul): These can eventually be removed once the metrics code
410415
# is running in mainline, and we have some nice monitoring frontends
411416
# to watch it
@@ -750,10 +755,26 @@ def inner_func(conn, *args, **kwargs):
750755
sql_scheduling_timer.observe(sched_duration_sec)
751756
context.add_database_scheduled(sched_duration_sec)
752757

758+
if self._txn_limit > 0:
759+
tid = self._db_pool.threadID()
760+
self._txn_counters[tid] += 1
761+
762+
if self._txn_counters[tid] > self._txn_limit:
763+
logger.debug(
764+
"Reconnecting database connection over transaction limit"
765+
)
766+
conn.reconnect()
767+
opentracing.log_kv(
768+
{"message": "reconnected due to txn limit"}
769+
)
770+
self._txn_counters[tid] = 1
771+
753772
if self.engine.is_connection_closed(conn):
754773
logger.debug("Reconnecting closed database connection")
755774
conn.reconnect()
756775
opentracing.log_kv({"message": "reconnected"})
776+
if self._txn_limit > 0:
777+
self._txn_counters[tid] = 1
757778

758779
try:
759780
if db_autocommit:

tests/storage/test_txn_limit.py

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
# Copyright 2014-2021 The Matrix.org Foundation C.I.C.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
from tests import unittest
16+
17+
18+
class SQLTransactionLimitTestCase(unittest.HomeserverTestCase):
19+
"""Test SQL transaction limit doesn't break transactions."""
20+
21+
def make_homeserver(self, reactor, clock):
22+
return self.setup_test_homeserver(db_txn_limit=1000)
23+
24+
def test_config(self):
25+
db_config = self.hs.config.get_single_database()
26+
self.assertEqual(db_config.config["txn_limit"], 1000)
27+
28+
def test_select(self):
29+
def do_select(txn):
30+
txn.execute("SELECT 1")
31+
32+
db_pool = self.hs.get_datastores().databases[0]
33+
34+
# force txn limit to roll over at least once
35+
for i in range(0, 1001):
36+
self.get_success_or_raise(db_pool.runInteraction("test_select", do_select))

tests/utils.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -239,6 +239,9 @@ def setup_test_homeserver(
239239
"args": {"database": ":memory:", "cp_min": 1, "cp_max": 1},
240240
}
241241

242+
if "db_txn_limit" in kwargs:
243+
database_config["txn_limit"] = kwargs["db_txn_limit"]
244+
242245
database = DatabaseConnectionConfig("master", database_config)
243246
config.database.databases = [database]
244247

0 commit comments

Comments
 (0)