Skip to content

Commit fedd9cd

Browse files
yorekyorek
committed
improved connection recylcing
1 parent 3012ba2 commit fedd9cd

File tree

1 file changed

+50
-35
lines changed

1 file changed

+50
-35
lines changed

‎app.py

Lines changed: 50 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -16,18 +16,17 @@
1616
app = Flask(__name__)
1717

1818
# Setup Azure Monitor
19-
middleware = FlaskMiddleware(
20-
app,
21-
exporter=AzureExporter(connection_string="InstrumentationKey={0}".format(os.environ['APPINSIGHTS_KEY'])),
22-
sampler=ProbabilitySampler(rate=1.0),
23-
)
19+
# middleware = FlaskMiddleware(
20+
# app,
21+
# exporter=AzureExporter(connection_string="InstrumentationKey={0}".format(os.environ['APPINSIGHTS_KEY'])),
22+
# sampler=ProbabilitySampler(rate=1.0),
23+
# )
2424

2525
# Setup Flask Restful framework
2626
api = Api(app)
2727
parser = reqparse.RequestParser()
2828
parser.add_argument('customer')
2929

30-
3130
# Implement manual connection pooling
3231
class ConnectionManager(object):
3332
__instance = None
@@ -41,32 +40,34 @@ def __new__(cls):
4140
return ConnectionManager.__instance
4241

4342
def is_retriable(self, value):
43+
# https://docs.microsoft.com/en-us/sql/odbc/reference/appendixes/appendix-a-odbc-error-codes
4444
RETRY_CODES = [
45-
8001,
46-
42000,
47-
1204, # The instance of the SQL Server Database Engine cannot obtain a LOCK resource at this time. Rerun your statement when there are fewer active users. Ask the database administrator to check the lock and memory configuration for this instance, or to check for long-running transactions.
48-
1205, # Transaction (Process ID) was deadlocked on resources with another process and has been chosen as the deadlock victim. Rerun the transaction
49-
1222, # Lock request time out period exceeded.
50-
49918, # Cannot process request. Not enough resources to process request.
51-
49919, # Cannot process create or update request. Too many create or update operations in progress for subscription "%ld".
52-
49920, # Cannot process request. Too many operations in progress for subscription "%ld".
53-
4060, # Cannot open database "%.*ls" requested by the login. The login failed.
54-
4221, # Login to read-secondary failed due to long wait on 'HADR_DATABASE_WAIT_FOR_TRANSITION_TO_VERSIONING'. The replica is not available for login because row versions are missing for transactions that were in-flight when the replica was recycled. The issue can be resolved by rolling back or committing the active transactions on the primary replica. Occurrences of this condition can be minimized by avoiding long write transactions on the primary.
55-
56-
40143, # The service has encountered an error processing your request. Please try again.
57-
40613, # Database '%.*ls' on server '%.*ls' is not currently available. Please retry the connection later. If the problem persists, contact customer support, and provide them the session tracing ID of '%.*ls'.
58-
40501, # The service is currently busy. Retry the request after 10 seconds. Incident ID: %ls. Code: %d.
59-
40540, # The service has encountered an error processing your request. Please try again.
60-
40197, # The service has encountered an error processing your request. Please try again. Error code %d.
61-
10929, # Resource ID: %d. The %s minimum guarantee is %d, maximum limit is %d and the current usage for the database is %d. However, the server is currently too busy to support requests greater than %d for this database. For more information, see http://go.microsoft.com/fwlink/?LinkId=267637. Otherwise, please try again later.
62-
10928, # Resource ID: %d. The %s limit for the database is %d and has been reached. For more information, see http://go.microsoft.com/fwlink/?LinkId=267637.
63-
10060, # An error has occurred while establishing a connection to the server. When connecting to SQL Server, this failure may be caused by the fact that under the default settings SQL Server does not allow remote connections. (provider: TCP Provider, error: 0 - A connection attempt failed because the connected party did not properly respond after a period of time, or established connection failed because connected host has failed to respond.) (Microsoft SQL Server, Error: 10060)
64-
10054, # The data value for one or more columns overflowed the type used by the provider.
65-
10053, # Could not convert the data value due to reasons other than sign mismatch or overflow.
66-
233, # A connection was successfully established with the server, but then an error occurred during the login process. (provider: Shared Memory Provider, error: 0 - No process is on the other end of the pipe.) (Microsoft SQL Server, Error: 233)
67-
64,
68-
20,
69-
0
45+
"08S01", # ODBC - Communication link failure
46+
"8001",
47+
"42000",
48+
"1204", # The instance of the SQL Server Database Engine cannot obtain a LOCK resource at this time. Rerun your statement when there are fewer active users. Ask the database administrator to check the lock and memory configuration for this instance, or to check for long-running transactions.
49+
"1205", # Transaction (Process ID) was deadlocked on resources with another process and has been chosen as the deadlock victim. Rerun the transaction
50+
"1222", # Lock request time out period exceeded.
51+
"49918", # Cannot process request. Not enough resources to process request.
52+
"49919", # Cannot process create or update request. Too many create or update operations in progress for subscription "%ld".
53+
"49920", # Cannot process request. Too many operations in progress for subscription "%ld".
54+
"4060", # Cannot open database "%.*ls" requested by the login. The login failed.
55+
"4221", # Login to read-secondary failed due to long wait on 'HADR_DATABASE_WAIT_FOR_TRANSITION_TO_VERSIONING'. The replica is not available for login because row versions are missing for transactions that were in-flight when the replica was recycled. The issue can be resolved by rolling back or committing the active transactions on the primary replica. Occurrences of this condition can be minimized by avoiding long write transactions on the primary.
56+
57+
"40143", # The service has encountered an error processing your request. Please try again.
58+
"40613", # Database '%.*ls' on server '%.*ls' is not currently available. Please retry the connection later. If the problem persists, contact customer support, and provide them the session tracing ID of '%.*ls'.
59+
"40501", # The service is currently busy. Retry the request after 10 seconds. Incident ID: %ls. Code: %d.
60+
"40540", # The service has encountered an error processing your request. Please try again.
61+
"40197", # The service has encountered an error processing your request. Please try again. Error code %d.
62+
"10929", # Resource ID: %d. The %s minimum guarantee is %d, maximum limit is %d and the current usage for the database is %d. However, the server is currently too busy to support requests greater than %d for this database. For more information, see http://go.microsoft.com/fwlink/?LinkId=267637. Otherwise, please try again later.
63+
"10928", # Resource ID: %d. The %s limit for the database is %d and has been reached. For more information, see http://go.microsoft.com/fwlink/?LinkId=267637.
64+
"10060", # An error has occurred while establishing a connection to the server. When connecting to SQL Server, this failure may be caused by the fact that under the default settings SQL Server does not allow remote connections. (provider: TCP Provider, error: 0 - A connection attempt failed because the connected party did not properly respond after a period of time, or established connection failed because connected host has failed to respond.) (Microsoft SQL Server, Error: 10060)
65+
"10054", # The data value for one or more columns overflowed the type used by the provider.
66+
"10053", # Could not convert the data value due to reasons other than sign mismatch or overflow.
67+
"233", # A connection was successfully established with the server, but then an error occurred during the login process. (provider: Shared Memory Provider, error: 0 - No process is on the other end of the pipe.) (Microsoft SQL Server, Error: 233)
68+
"64",
69+
"20",
70+
"0"
7071
]
7172
result = value in RETRY_CODES
7273
return result
@@ -97,11 +98,18 @@ def __removeConnection(self, idx):
9798
del(self.__conn_dict[idx])
9899
self.__lock.release()
99100

101+
def __removeConnections(self):
102+
self.__lock.acquire()
103+
self.__conn_dict.clear()
104+
self.__lock.release()
105+
100106
@retry(stop=stop_after_attempt(3), wait=wait_fixed(10), after=after_log(app.logger, logging.DEBUG))
101107
def executeQueryJSON(self, procedure, payload=None):
102108
result = {}
103-
(idx, conn) = self.__getConnection()
104109
try:
110+
(idx, conn) = self.__getConnection()
111+
app.logger.info(f"Connection Index: {idx}")
112+
105113
cursor = conn.cursor()
106114

107115
if payload:
@@ -117,10 +125,17 @@ def executeQueryJSON(self, procedure, payload=None):
117125
result = {}
118126

119127
cursor.commit()
120-
except Exception as e:
121-
self.__removeConnection(idx)
128+
except pyodbc.Error as e:
122129
if isinstance(e, pyodbc.ProgrammingError) or isinstance(e, pyodbc.OperationalError):
123-
if self.is_retriable(int(e.args[0])):
130+
app.logger.error(f"Error: {e.args[0]}")
131+
if self.is_retriable(e.args[0]):
132+
# If there is a "Communication Link Failure" error,
133+
# then all connections must be removed
134+
# as all will be in an invalid state
135+
if (e.args[0] == "08S01"):
136+
self.__removeConnections()
137+
else:
138+
self.__removeConnection(idx)
124139
raise
125140

126141
return result

0 commit comments

Comments
 (0)