tests: Allow enabling CMG logging

Introduce the '--enable-cmg-logging' option, to allow setting the
logging level for the Gatewat and Scheduler to INFO (default is
WARNING).

Signed-off-by: Phoevos Kalemkeris <[email protected]>

Author: phoevos

tests/conftest.py

@@ -11,6 +11,13 @@ def pytest_addoption(parser: pytest.Parser) -> None:
         help="Skip cleanup for the CMS resources after completing the tests.",
     )
 
+    parser.addoption(
+        "--enable-cmg-logging",
+        action="store_true",
+        default=False,
+        help="Enable logging for the CogStack Model Gateway resources (i.e. Gateway, Scheduler).",
+    )
+
 
 @pytest.fixture(scope="module")
 def cleanup_cms(request: pytest.FixtureRequest) -> bool:
@@ -24,14 +31,13 @@ def setup_logging() -> None:
         if logger_name.startswith("testcontainers"):
             logging.getLogger(logger_name).setLevel(logging.WARNING)
 
-    parent_logger = logging.getLogger("cmg.tests")
+    parent_logger = logging.getLogger("cmg")
     parent_logger.setLevel(logging.DEBUG)
 
-    handler = logging.StreamHandler()
-    handler.setLevel(logging.DEBUG)
-    handler.setFormatter(logging.Formatter("%(levelname)s:%(message)s"))
-    parent_logger.addHandler(handler)
+    if not any(isinstance(handler, logging.StreamHandler) for handler in parent_logger.handlers):
+        handler = logging.StreamHandler()
+        handler.setLevel(logging.DEBUG)
+        handler.setFormatter(logging.Formatter("%(levelname)s:%(name)s:%(message)s"))
+        parent_logger.addHandler(handler)
 
-    # Configure child loggers
-    logging.getLogger("cmg.tests.integration").setLevel(logging.INFO)
-    logging.getLogger("cmg.tests.unit").setLevel(logging.INFO)
+    logging.getLogger("cmg.tests").setLevel(logging.INFO)

tests/integration/test_api.py

@@ -48,7 +48,9 @@ def setup(request: pytest.FixtureRequest, cleanup_cms: bool):
     env = {
         "MLFLOW_TRACKING_URI": f"http://{mlflow_addr}:{mlflow_port}",
     }
-    configure_environment(postgres, rabbitmq, minio, extras=env)
+
+    enable_cmg_logging = request.config.getoption("--enable-cmg-logging")
+    configure_environment(postgres, rabbitmq, minio, enable_logs=enable_cmg_logging, extras=env)
 
     setup_scheduler(request)
 

tests/integration/utils.py

@@ -121,12 +121,17 @@ def configure_environment(
     postgres: PostgresContainer,
     rabbitmq: RabbitMqContainer,
     minio: MinioContainer,
+    enable_logs: bool = False,
     extras: dict = None,
 ):
     log.info("Setting environment variables...")
+    cmg_log_level = logging.INFO if enable_logs else logging.WARNING
     queue_connection_params = rabbitmq.get_connection_params()
     minio_host, minio_port = minio.get_config()["endpoint"].split(":")
     env = {
+        "CMG_COMMON_LOG_LEVEL": logging.getLevelName(logging.WARNING),
+        "CMG_GATEWAY_LOG_LEVEL": logging.getLevelName(cmg_log_level),
+        "CMG_SCHEDULER_LOG_LEVEL": logging.getLevelName(cmg_log_level),
         "CMG_DB_USER": postgres.username,
         "CMG_DB_PASSWORD": postgres.password,
         "CMG_DB_HOST": postgres.get_container_host_ip(),
@@ -157,8 +162,6 @@ def start_scheduler():
         ["poetry", "run", "python3", SCHEDULER_SCRIPT_PATH],
         start_new_session=True,
         env=dict(os.environ),
-        stderr=subprocess.DEVNULL,
-        stdout=subprocess.DEVNULL,
     )