Skip to content

Commit d26753e

Browse files
swathipilswathipil
authored
[ServiceBus] stress optional rotate logs (Azure#38871)
* [ServiceBus] make rotating logs optional * add todo * only add handler to loggers if one does not exist * TimedRotatingFileHandler -> RotatingFileHandler
1 parent 90da0a6 commit d26753e

File tree

5 files changed

+76
-22
lines changed

5 files changed

+76
-22
lines changed

‎sdk/servicebus/azure-servicebus/stress/scripts/logger.py‎

Lines changed: 33 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -3,15 +3,14 @@
33
# Licensed under the MIT License. See License.txt in the project root for license information.
44
# --------------------------------------------------------------------------------------------
55

6-
import os
76
import sys
87
import logging
9-
from logging.handlers import TimedRotatingFileHandler
8+
from logging.handlers import RotatingFileHandler
109

1110
from opencensus.ext.azure.log_exporter import AzureLogHandler
1211

1312

14-
def get_base_logger(log_filename, logger_name, level=logging.ERROR, print_console=False, log_format=None):
13+
def get_base_logger(log_filename, logger_name, level=logging.ERROR, print_console=False, log_format=None, rotating_logs=True):
1514
logger = logging.getLogger(logger_name)
1615
logger.setLevel(level)
1716
formatter = log_format or logging.Formatter(
@@ -24,20 +23,24 @@ def get_base_logger(log_filename, logger_name, level=logging.ERROR, print_consol
2423
console_handler.setFormatter(formatter)
2524
logger.addHandler(console_handler)
2625
else:
27-
# rotated hourly if small file, o/w rotated bi-hourly
28-
if level == logging.DEBUG or level == logging.INFO:
29-
time = 30
26+
if rotating_logs:
27+
if not logger.handlers:
28+
# 5 MB max file size, 350 files max
29+
mb = 50
30+
bytes_in_mb = 1_048_576
31+
bytes = mb * bytes_in_mb
32+
file_handler = RotatingFileHandler(log_filename, maxBytes=bytes, backupCount=300)
33+
file_handler.setFormatter(formatter)
34+
logger.addHandler(file_handler)
3035
else:
31-
time = 60
32-
file_handler = TimedRotatingFileHandler(log_filename, when="M", interval=time, utc=True)
33-
if not logger.handlers:
36+
file_handler = logging.FileHandler(log_filename)
3437
file_handler.setFormatter(formatter)
3538
logger.addHandler(file_handler)
3639

3740
return logger
3841

3942

40-
def get_logger(log_filename, logger_name, level=logging.ERROR, print_console=False, log_format=None):
43+
def get_logger(log_filename, logger_name, level=logging.ERROR, print_console=False, log_format=None, rotating_logs=True):
4144
stress_logger = logging.getLogger(logger_name)
4245
stress_logger.setLevel(level)
4346
servicebus_logger = logging.getLogger("azure.servicebus")
@@ -48,16 +51,27 @@ def get_logger(log_filename, logger_name, level=logging.ERROR, print_console=Fal
4851
formatter = log_format or logging.Formatter(
4952
"%(asctime)s - [%(thread)d.%(threadName)s] - %(name)-12s %(levelname)-8s %(funcName)s(%(lineno)d) %(message)s"
5053
)
51-
# rotated hourly if small file, o/w rotated bi-hourly
52-
if level == logging.DEBUG or level == logging.INFO:
53-
time = 30
54+
if rotating_logs:
55+
# If any do not have handlers, create a new file handler and add.
56+
if not servicebus_logger.handlers or not pyamqp_logger.handlers or not stress_logger.handlers:
57+
# 5 MB max file size, 350 files max
58+
mb = 50
59+
bytes_in_mb = 1_048_576
60+
bytes = mb * bytes_in_mb
61+
file_handler = RotatingFileHandler(log_filename, maxBytes=bytes, backupCount=300)
62+
file_handler.setFormatter(formatter)
63+
if not servicebus_logger.handlers:
64+
servicebus_logger.addHandler(file_handler)
65+
if not pyamqp_logger.handlers:
66+
pyamqp_logger.addHandler(file_handler)
67+
if not stress_logger.handlers:
68+
stress_logger.addHandler(file_handler)
5469
else:
55-
time = 60
56-
file_handler = TimedRotatingFileHandler(log_filename, when="M", interval=time, utc=True)
57-
file_handler.setFormatter(formatter)
58-
servicebus_logger.addHandler(file_handler)
59-
pyamqp_logger.addHandler(file_handler)
60-
stress_logger.addHandler(file_handler)
70+
console_handler = logging.FileHandler(log_filename)
71+
console_handler.setFormatter(formatter)
72+
servicebus_logger.addHandler(console_handler)
73+
pyamqp_logger.addHandler(console_handler)
74+
stress_logger.addHandler(console_handler)
6175

6276
return stress_logger
6377

‎sdk/servicebus/azure-servicebus/stress/scripts/stress_test_base.py‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,7 @@ def __init__(
9696
process_monitor=None,
9797
logging_level=logging.ERROR,
9898
transport_type=False,
99+
rotating_logs=True,
99100
):
100101
self.senders = senders
101102
self.receivers = receivers
@@ -113,12 +114,13 @@ def __init__(
113114
self.send_session_id = send_session_id
114115
self.azure_monitor_metric = azure_monitor_metric or AbstractMonitorMetric("fake_test_name")
115116
self.logging_level = logging_level
117+
self.rotating_logs = rotating_logs
116118
logfile_name = LOGFILE_NAME
117119
if transport_type:
118120
logfile_name += "_ws.log"
119121
else:
120122
logfile_name += ".log"
121-
self.logger = get_logger(logfile_name, "stress_test", self.logging_level)
123+
self.logger = get_logger(logfile_name, "stress_test", self.logging_level, rotating_logs=self.rotating_logs)
122124
self.process_monitor = process_monitor or ProcessMonitor(
123125
"monitor_{}".format(logfile_name),
124126
"test_stress_queues",
@@ -349,6 +351,7 @@ def __init__(
349351
process_monitor=None,
350352
logging_level=logging.ERROR,
351353
transport_type=False,
354+
rotating_logs=True,
352355
):
353356
super(StressTestRunnerAsync, self).__init__(
354357
senders,
@@ -369,6 +372,7 @@ def __init__(
369372
process_monitor=process_monitor,
370373
logging_level=logging_level,
371374
transport_type=transport_type,
375+
rotating_logs=rotating_logs,
372376
)
373377

374378
async def _send_async(self, sender, end_time):

‎sdk/servicebus/azure-servicebus/stress/scripts/test_stress_queues.py‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ def test_stress_queue_send_and_receive(args):
3939
azure_monitor_metric=AzureMonitorMetric("test_stress_queue_send_and_receive"),
4040
logging_level=LOGGING_LEVEL,
4141
transport_type=args.transport,
42+
rotating_logs=args.rotating_logs,
4243
)
4344

4445
result = stress_test.run()
@@ -63,6 +64,7 @@ def test_stress_queue_send_and_pull_receive(args):
6364
azure_monitor_metric=AzureMonitorMetric("test_stress_queue_send_and_pull_receive"),
6465
logging_level=LOGGING_LEVEL,
6566
transport_type=args.transport,
67+
rotating_logs=args.rotating_logs,
6668
)
6769

6870
result = stress_test.run()
@@ -87,6 +89,7 @@ def test_stress_queue_batch_send_and_receive(args):
8789
azure_monitor_metric=AzureMonitorMetric("test_stress_queue_batch_send_and_receive"),
8890
logging_level=LOGGING_LEVEL,
8991
transport_type=args.transport,
92+
rotating_logs=args.rotating_logs,
9093
)
9194

9295
result = stress_test.run()
@@ -111,6 +114,7 @@ def test_stress_queue_slow_send_and_receive(args):
111114
azure_monitor_metric=AzureMonitorMetric("test_stress_queue_slow_send_and_receive"),
112115
logging_level=LOGGING_LEVEL,
113116
transport_type=args.transport,
117+
rotating_logs=args.rotating_logs,
114118
)
115119

116120
result = stress_test.run()
@@ -140,6 +144,7 @@ def test_stress_queue_receive_and_delete(args):
140144
azure_monitor_metric=AzureMonitorMetric("test_stress_queue_slow_send_and_receive"),
141145
logging_level=LOGGING_LEVEL,
142146
transport_type=args.transport,
147+
rotating_logs=args.rotating_logs,
143148
)
144149

145150
result = stress_test.run()
@@ -164,6 +169,7 @@ def test_stress_queue_unsettled_messages(args):
164169
azure_monitor_metric=AzureMonitorMetric("test_stress_queue_unsettled_messages"),
165170
logging_level=LOGGING_LEVEL,
166171
transport_type=args.transport,
172+
rotating_logs=args.rotating_logs,
167173
)
168174

169175
result = stress_test.run()
@@ -190,6 +196,7 @@ def test_stress_queue_receive_large_batch_size(args):
190196
azure_monitor_metric=AzureMonitorMetric("test_stress_queue_receive_large_batch_size"),
191197
logging_level=LOGGING_LEVEL,
192198
transport_type=args.transport,
199+
rotating_logs=args.rotating_logs,
193200
)
194201

195202
result = stress_test.run()
@@ -224,6 +231,7 @@ def test_stress_queue_pull_receive_timeout(args):
224231
azure_monitor_metric=AzureMonitorMetric("test_stress_queue_pull_receive_timeout"),
225232
logging_level=LOGGING_LEVEL,
226233
transport_type=args.transport,
234+
rotating_logs=args.rotating_logs,
227235
)
228236

229237
result = stress_test.run()
@@ -256,6 +264,7 @@ def test_stress_queue_long_renew_send_and_receive(args):
256264
azure_monitor_metric=AzureMonitorMetric("test_stress_queue_long_renew_send_and_receive"),
257265
logging_level=LOGGING_LEVEL,
258266
transport_type=args.transport,
267+
rotating_logs=args.rotating_logs,
259268
)
260269

261270
result = stress_test.run()
@@ -299,6 +308,7 @@ def test_stress_queue_long_renew_session_send_and_receive(args):
299308
azure_monitor_metric=AzureMonitorMetric("test_stress_queue_long_renew_session_send_and_receive"),
300309
logging_level=LOGGING_LEVEL,
301310
transport_type=args.transport,
311+
rotating_logs=args.rotating_logs,
302312
)
303313

304314
result = stress_test.run()
@@ -330,6 +340,7 @@ def test_stress_queue_peek_messages(args):
330340
azure_monitor_metric=AzureMonitorMetric("test_stress_queue_peek_messages"),
331341
logging_level=LOGGING_LEVEL,
332342
transport_type=args.transport,
343+
rotating_logs=args.rotating_logs,
333344
)
334345

335346
result = stress_test.run()
@@ -370,6 +381,7 @@ def test_stress_queue_close_and_reopen(args):
370381
azure_monitor_metric=AzureMonitorMetric("test_stress_queue_close_and_reopen"),
371382
logging_level=LOGGING_LEVEL,
372383
transport_type=args.transport,
384+
rotating_logs=args.rotating_logs,
373385
)
374386

375387
result = stress_test.run()
@@ -425,6 +437,7 @@ def test_stress_queue_check_for_dropped_messages(args):
425437
azure_monitor_metric=AzureMonitorMetric("test_stress_queue_check_for_dropped_messages"),
426438
logging_level=LOGGING_LEVEL,
427439
transport_type=args.transport,
440+
rotating_logs=args.rotating_logs,
428441
)
429442

430443
result = stress_test.run()
@@ -465,8 +478,12 @@ def test_stress_queue_check_for_dropped_messages(args):
465478
type=str,
466479
default="Error",
467480
)
481+
# rotate logs by default, if you want to disable it, use --no-rotating-logs flag
482+
parser.add_argument("--no-rotating-logs", action="store_true")
468483

469484
args, _ = parser.parse_known_args()
485+
# store rotating_logs in args for later use
486+
args.rotating_logs = not args.no_rotating_logs
470487

471488
if args.transport:
472489
TRANSPORT_TYPE = TransportType.AmqpOverWebsocket

‎sdk/servicebus/azure-servicebus/stress/scripts/test_stress_queues_async.py‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ async def test_stress_queue_send_and_receive(args):
4242
azure_monitor_metric=AzureMonitorMetric("test_stress_queue_send_and_receive"),
4343
logging_level=LOGGING_LEVEL,
4444
transport_type=args.transport,
45+
rotating_logs=args.rotating_logs,
4546
)
4647

4748
result = await stress_test.run_async()
@@ -66,6 +67,7 @@ async def test_stress_queue_send_and_pull_receive(args):
6667
azure_monitor_metric=AzureMonitorMetric("test_stress_queue_send_and_pull_receive"),
6768
logging_level=LOGGING_LEVEL,
6869
transport_type=args.transport,
70+
rotating_logs=args.rotating_logs,
6971
)
7072

7173
result = await stress_test.run_async()
@@ -90,6 +92,7 @@ async def test_stress_queue_batch_send_and_receive(args):
9092
azure_monitor_metric=AzureMonitorMetric("test_stress_queue_batch_send_and_receive"),
9193
logging_level=LOGGING_LEVEL,
9294
transport_type=args.transport,
95+
rotating_logs=args.rotating_logs,
9396
)
9497

9598
result = await stress_test.run_async()
@@ -114,6 +117,7 @@ async def test_stress_queue_slow_send_and_receive(args):
114117
azure_monitor_metric=AzureMonitorMetric("test_stress_queue_slow_send_and_receive"),
115118
logging_level=LOGGING_LEVEL,
116119
transport_type=args.transport,
120+
rotating_logs=args.rotating_logs,
117121
)
118122

119123
result = await stress_test.run_async()
@@ -144,6 +148,7 @@ async def test_stress_queue_receive_and_delete(args):
144148
azure_monitor_metric=AzureMonitorMetric("test_stress_queue_slow_send_and_receive"),
145149
logging_level=LOGGING_LEVEL,
146150
transport_type=args.transport,
151+
rotating_logs=args.rotating_logs,
147152
)
148153

149154
result = await stress_test.run_async()
@@ -168,6 +173,7 @@ async def test_stress_queue_unsettled_messages(args):
168173
azure_monitor_metric=AzureMonitorMetric("test_stress_queue_unsettled_messages"),
169174
logging_level=LOGGING_LEVEL,
170175
transport_type=args.transport,
176+
rotating_logs=args.rotating_logs,
171177
)
172178

173179
result = await stress_test.run_async()
@@ -194,6 +200,7 @@ async def test_stress_queue_receive_large_batch_size(args):
194200
azure_monitor_metric=AzureMonitorMetric("test_stress_queue_receive_large_batch_size"),
195201
logging_level=LOGGING_LEVEL,
196202
transport_type=args.transport,
203+
rotating_logs=args.rotating_logs,
197204
)
198205

199206
result = await stress_test.run_async()
@@ -228,6 +235,7 @@ async def test_stress_queue_pull_receive_timeout(args):
228235
azure_monitor_metric=AzureMonitorMetric("test_stress_queue_pull_receive_timeout"),
229236
logging_level=LOGGING_LEVEL,
230237
transport_type=args.transport,
238+
rotating_logs=args.rotating_logs,
231239
)
232240

233241
result = await stress_test.run_async()
@@ -260,6 +268,7 @@ async def test_stress_queue_long_renew_send_and_receive(args):
260268
azure_monitor_metric=AzureMonitorMetric("test_stress_queue_long_renew_send_and_receive"),
261269
logging_level=LOGGING_LEVEL,
262270
transport_type=args.transport,
271+
rotating_logs=args.rotating_logs,
263272
)
264273

265274
result = await stress_test.run_async()
@@ -303,6 +312,7 @@ async def test_stress_queue_long_renew_session_send_and_receive(args):
303312
azure_monitor_metric=AzureMonitorMetric("test_stress_queue_long_renew_session_send_and_receive"),
304313
logging_level=LOGGING_LEVEL,
305314
transport_type=args.transport,
315+
rotating_logs=args.rotating_logs,
306316
)
307317

308318
result = await stress_test.run_async()
@@ -334,6 +344,7 @@ async def test_stress_queue_peek_messages(args):
334344
azure_monitor_metric=AzureMonitorMetric("test_stress_queue_peek_messages"),
335345
logging_level=LOGGING_LEVEL,
336346
transport_type=args.transport,
347+
rotating_logs=args.rotating_logs,
337348
)
338349

339350
result = await stress_test.run_async()
@@ -374,6 +385,7 @@ async def test_stress_queue_close_and_reopen(args):
374385
azure_monitor_metric=AzureMonitorMetric("test_stress_queue_close_and_reopen"),
375386
logging_level=LOGGING_LEVEL,
376387
transport_type=args.transport,
388+
rotating_logs=args.rotating_logs,
377389
)
378390

379391
result = await stress_test.run_async()
@@ -429,6 +441,7 @@ async def test_stress_queue_check_for_dropped_messages(args):
429441
azure_monitor_metric=AzureMonitorMetric("test_stress_queue_check_for_dropped_messages"),
430442
logging_level=LOGGING_LEVEL,
431443
transport_type=args.transport,
444+
rotating_logs=args.rotating_logs,
432445
)
433446

434447
result = await stress_test.run_async()
@@ -512,7 +525,12 @@ async def run(args):
512525
type=str,
513526
default="Error",
514527
)
528+
# rotate logs by default, if you want to disable it, use --no-rotating-logs flag
529+
parser.add_argument("--no-rotating-logs", action="store_true")
530+
515531
args, _ = parser.parse_known_args()
532+
# store rotating_logs in args for later use
533+
args.rotating_logs = not args.no_rotating_logs
516534

517535
if args.transport:
518536
TRANSPORT_TYPE = TransportType.AmqpOverWebsocket

‎sdk/servicebus/azure-servicebus/stress/templates/testjob.yaml‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -63,12 +63,13 @@ spec:
6363
command: ['bash', '-c', 'mkdir -p $DEBUG_SHARE && python3 test_stress_queues.py --method send_receive_batch --duration 300000 --logging-enable --debug_level Debug ']
6464
{{- end -}}
6565

66+
# TODO: Running with both memray and RotatingFileHandler is resulting in a memory leak. Turn off rotating logs as a workaround to avoid the memory leak when running memray.
6667
{{ if eq .Stress.testTarget "amemray" }}
67-
command: ['bash', '-c', 'mkdir -p $DEBUG_SHARE && memray run --output $DEBUG_SHARE/sb_async_memray_output.bin test_stress_queues_async.py --method send_pull_receive --duration 300000 --logging-enable --debug_level Debug']
68+
command: ['bash', '-c', 'mkdir -p $DEBUG_SHARE && memray run --output $DEBUG_SHARE/sb_async_memray_output.bin test_stress_queues_async.py --method send_pull_receive --duration 300000 --logging-enable --debug_level Debug --no-rotating-logs']
6869
{{- end -}}
6970

7071
{{ if eq .Stress.testTarget "memray" }}
71-
command: ['bash', '-c', 'mkdir -p $DEBUG_SHARE && memray run --output $DEBUG_SHARE/sb_memray_output.bin test_stress_queues.py --method send_pull_receive --duration 300000 --logging-enable --debug_level Debug']
72+
command: ['bash', '-c', 'mkdir -p $DEBUG_SHARE && memray run --output $DEBUG_SHARE/sb_memray_output.bin test_stress_queues.py --method send_pull_receive --duration 300000 --logging-enable --debug_level Debug --no-rotating-logs']
7273
{{- end -}}
7374

7475
{{- include "stress-test-addons.container-env" . | nindent 6 }}

0 commit comments

Comments
 (0)