[EventHub] update Timed to RotatingFileHandler (Azure#39180)

* [EventHub] update timed rotating logger to rotating logger

* update stress w/ git branch comments sb

* sb update directions

Author: swathipil

sdk/eventhub/azure-eventhub/stress/Dockerfile

@@ -1,5 +1,7 @@
 FROM mcr.microsoft.com/cbl-mariner/base/python:3
 
+# To install a package from a git branch, add the line below to the end of the following installs:
+# && yum install -y git
 RUN yum update -y && yum install -y ca-certificates
 
 WORKDIR /app

sdk/eventhub/azure-eventhub/stress/scripts/azure_eventhub_consumer_stress_async.py

@@ -116,9 +116,12 @@ def parse_starting_position(args):
     type=str,
     default="Error",
 )
+# rotate logs by default, if you want to disable it, use --no-rotating-logs flag
+parser.add_argument("--no-rotating-logs", action="store_true")
 
 
 args = parser.parse_args()
+rotating_logs = not args.no_rotating_logs
 starting_position = parse_starting_position(args)
 print_console = args.print_console or (os.environ.get("PRINT_CONSOLE") == "1")
 debug_level = getattr(logging, args.debug_level.upper(), logging.ERROR)
@@ -131,7 +134,7 @@ def parse_starting_position(args):
     log_filename += ".log"
 logdir = os.environ.get("DEBUG_SHARE")
 logfilepath = f"{logdir}/{log_filename}"
-LOGGER = get_logger(logfilepath, "stress_receive_async", level=debug_level, print_console=args.print_console)
+LOGGER = get_logger(logfilepath, "stress_receive_async", level=debug_level, print_console=args.print_console, rotating_logs=rotating_logs)
 LOG_PER_COUNT = args.output_interval
 
 start_time = time.perf_counter()

sdk/eventhub/azure-eventhub/stress/scripts/azure_eventhub_consumer_stress_sync.py

@@ -112,9 +112,13 @@ def parse_starting_position(args):
     type=str,
     default="Error",
 )
+# rotate logs by default, if you want to disable it, use --no-rotating-logs flag
+parser.add_argument("--no-rotating-logs", action="store_true")
 
 
 args = parser.parse_args()
+
+rotating_logs = not args.no_rotating_logs
 starting_position = parse_starting_position(args)
 print_console = args.print_console or (os.environ.get("PRINT_CONSOLE") == "1")
 debug_level = getattr(logging, args.debug_level.upper(), logging.ERROR)
@@ -127,7 +131,7 @@ def parse_starting_position(args):
     log_filename += ".log"
 logdir = os.environ.get("DEBUG_SHARE")
 logfilepath = f"{logdir}/{log_filename}"
-LOGGER = get_logger(logfilepath, "stress_receive_sync", level=debug_level, print_console=print_console)
+LOGGER = get_logger(logfilepath, "stress_receive_sync", level=debug_level, print_console=print_console, rotating_logs=rotating_logs)
 LOG_PER_COUNT = args.output_interval
 
 start_time = time.perf_counter()

sdk/eventhub/azure-eventhub/stress/scripts/azure_eventhub_producer_stress.py

@@ -179,8 +179,12 @@ def __init__(self, argument_parser):
             type=str,
             default="Error",
         )
+        # rotate logs by default, if you want to disable it, use --no-rotating-logs flag
+        parser.add_argument("--no-rotating-logs", action="store_true")
         self.args, _ = parser.parse_known_args()
 
+        # store rotating_logs in args for later use
+        self.rotating_logs = not self.args.no_rotating_logs
         if self.args.send_partition_key and self.args.send_partition_id:
             raise ValueError("Cannot set send_partition_key and send_partition_id at the same time.")
 
@@ -298,6 +302,7 @@ def get_partition_ids(self_inner):
                 method_name,
                 level=self.debug_level,
                 print_console=self.args.print_console,
+                rotating_logs=self.rotating_logs,
             )
             test_method = globals()[method_name]
             self.running = True
@@ -401,7 +406,7 @@ async def get_partition_ids(self_inner):
             logdir = os.environ.get("DEBUG_SHARE")
             logfilepath = f"{logdir}/{log_filename}"
 
-            logger = get_logger(logfilepath, method_name, level=self.debug_level, print_console=self.args.print_console)
+            logger = get_logger(logfilepath, method_name, level=self.debug_level, print_console=self.args.print_console, rotating_logs=self.rotating_logs)
             test_method = globals()[method_name]
             self.running = True
 

sdk/eventhub/azure-eventhub/stress/scripts/dev_requirement.txt

@@ -1,8 +1,10 @@
 psutil
+# To install from a specific git branch, replace the package with a req in the following format:
+# git+<gh-repo-url>@<branch>#subdirectory=<path/to/sdk>&egg=<package-name>
+# ex. git+https://github.com/kashifkhan/azure-sdk-for-python.git@fix_ws_none_error#subdirectory=sdk/eventhub/azure-eventhub&egg=azure-eventhub
 azure-eventhub
 azure-eventhub-checkpointstoreblob
 azure-eventhub-checkpointstoreblob-aio
-azure-servicebus
 azure-storage-blob
 azure-identity
 opencensus-ext-azure

sdk/eventhub/azure-eventhub/stress/scripts/logger.py

@@ -6,12 +6,12 @@
 import os
 import sys
 import logging
-from logging.handlers import TimedRotatingFileHandler
+from logging.handlers import RotatingFileHandler
 
 from opencensus.ext.azure.log_exporter import AzureLogHandler
 
 
-def get_base_logger(log_filename, logger_name, level=logging.ERROR, print_console=False, log_format=None):
+def get_base_logger(log_filename, logger_name, level=logging.ERROR, print_console=False, log_format=None, rotating_logs=True):
     logger = logging.getLogger(logger_name)
     logger.setLevel(level)
     formatter = log_format or logging.Formatter(
@@ -24,13 +24,17 @@ def get_base_logger(log_filename, logger_name, level=logging.ERROR, print_consol
             console_handler.setFormatter(formatter)
             logger.addHandler(console_handler)
     else:
-        # rotated hourly if small file, o/w rotated bi-hourly
-        if level == logging.DEBUG or level == logging.INFO:
-            time = 30
+        if rotating_logs:
+            if not logger.handlers:
+                # 5 MB max file size, 350 files max
+                mb = 50
+                bytes_in_mb = 1_048_576
+                bytes = mb * bytes_in_mb
+                file_handler = RotatingFileHandler(log_filename, maxBytes=bytes, backupCount=300)
+                file_handler.setFormatter(formatter)
+                logger.addHandler(file_handler)
         else:
-            time = 60
-        file_handler = TimedRotatingFileHandler(log_filename, when="M", interval=time, utc=True)
-        if not logger.handlers:
+            file_handler = logging.FileHandler(log_filename)
             file_handler.setFormatter(formatter)
             logger.addHandler(file_handler)
 
@@ -43,28 +47,35 @@ def get_logger(
     level=logging.ERROR,
     print_console=False,
     log_format=None,
+    rotating_logs=True,
 ):
     stress_logger = logging.getLogger(logger_name)
     stress_logger.setLevel(level)
     eventhub_logger = logging.getLogger("azure.eventhub")
     eventhub_logger.setLevel(level)
-    uamqp_logger = logging.getLogger("uamqp")
-    uamqp_logger.setLevel(level)
 
     formatter = log_format or logging.Formatter(
         "%(asctime)s - [%(thread)d.%(threadName)s] - %(name)-12s %(levelname)-8s %(funcName)s(%(lineno)d) %(message)s"
     )
 
-    # rotated hourly if small file, o/w rotated bi-hourly
-    if level == logging.DEBUG or level == logging.INFO:
-        time = 30
+    if rotating_logs:
+        # If any do not have handlers, create a new file handler and add.
+        if not eventhub_logger.handlers or not stress_logger.handlers:
+            # 5 MB max file size, 350 files max
+            mb = 50
+            bytes_in_mb = 1_048_576
+            bytes = mb * bytes_in_mb
+            file_handler = RotatingFileHandler(log_filename, maxBytes=bytes, backupCount=300)
+            file_handler.setFormatter(formatter)
+            if not eventhub_logger.handlers:
+                eventhub_logger.addHandler(file_handler)
+            if not stress_logger.handlers:
+                stress_logger.addHandler(file_handler)
     else:
-        time = 60
-    file_handler = TimedRotatingFileHandler(log_filename, when="M", interval=time, utc=True)
-    file_handler.setFormatter(formatter)
-    eventhub_logger.addHandler(file_handler)
-    uamqp_logger.addHandler(file_handler)
-    stress_logger.addHandler(file_handler)
+        console_handler = logging.FileHandler(log_filename)
+        console_handler.setFormatter(formatter)
+        eventhub_logger.addHandler(console_handler)
+        stress_logger.addHandler(console_handler)
 
     return stress_logger
 

sdk/servicebus/azure-servicebus/stress/Chart.lock

@@ -1,6 +1,6 @@
 dependencies:
 - name: stress-test-addons
   repository: https://stresstestcharts.blob.core.windows.net/helm/
-  version: 0.3.0
-digest: sha256:3e21a7fdf5d6b37e871a6dd9f755888166fbb24802aa517f51d1d9223b47656e
-generated: "2023-10-17T20:04:32.848343767-04:00"
+  version: 0.3.4
+digest: sha256:e1d29df9556aaf06ee08fe1116695e6ce5396826b334b5c9d0f646da6599057f
+generated: "2025-01-07T13:56:37.7848059-08:00"

sdk/servicebus/azure-servicebus/stress/Dockerfile

@@ -1,5 +1,7 @@
 FROM mcr.microsoft.com/cbl-mariner/base/python:3
 
+# To install a package from a git branch, add the line below to the end of the following installs:
+# && yum install -y git
 RUN yum update -y && yum install -y ca-certificates
 
 WORKDIR /app

sdk/servicebus/azure-servicebus/stress/scenarios-matrix.yaml

@@ -26,7 +26,7 @@ matrix:
       testTarget: batchw
     abatchw:
       testTarget: abatchw
-    memray:
-      testTarget: memray
-    amemray:
-      testTarget: amemray
+    #memray:
+    #  testTarget: memray
+    #amemray:
+    #  testTarget: amemray

sdk/servicebus/azure-servicebus/stress/scripts/dev_requirements.txt

@@ -1,6 +1,9 @@
 aiohttp>=3.0
 opencensus-ext-azure
 psutil
+# To install from a specific git branch, replace the package with a req in the following format:
+# git+<gh-repo-url>@<branch>#subdirectory=<path/to/sdk>&egg=<package-name>
+# ex. git+https://github.com/l0lawrence/azure-sdk-for-python.git@sb_retries#subdirectory=sdk/servicebus/azure-servicebus&egg=azure-servicebus
 azure-servicebus
 python-dotenv
 websocket-client