Update loguru and deploy log config (#517)

* Update loguru config

* Update config

* fix log level

* Update deploy log config

Author: wu-clan

backend/backend.dockerfile

@@ -19,11 +19,9 @@ RUN apt-get update \
 
 ENV TZ="Asia/Shanghai"
 
-RUN mkdir -p /var/log/fastapi_server  \
-    && mkdir -p /var/log/supervisor \
-    && mkdir -p /etc/supervisor/conf.d
+RUN mkdir -p /var/log/fastapi_server
 
-COPY deploy/backend/supervisor.conf /etc/supervisor/supervisord.conf
+COPY deploy/backend/supervisord.conf /etc/supervisor/supervisord.conf
 
 COPY deploy/backend/fastapi_server.conf /etc/supervisor/conf.d/
 

backend/celery.dockerfile

@@ -19,11 +19,9 @@ RUN apt-get update \
 
 ENV TZ="Asia/Shanghai"
 
-RUN mkdir -p /var/log/celery  \
-    && mkdir -p /var/log/supervisor \
-    && mkdir -p /etc/supervisor/conf.d
+RUN mkdir -p /var/log/celery
 
-COPY deploy/backend/supervisor.conf /etc/supervisor/supervisord.conf
+COPY deploy/backend/supervisord.conf /etc/supervisor/supervisord.conf
 
 COPY deploy/backend/celery.conf /etc/supervisor/conf.d/
 

backend/common/log.py

@@ -3,8 +3,7 @@
 import inspect
 import logging
 import os
-
-from sys import stderr, stdout
+import sys
 
 from asgi_correlation_id import correlation_id
 from loguru import logger
@@ -37,12 +36,12 @@ def emit(self, record: logging.LogRecord):
 
 def setup_logging():
     """
-    From https://pawamoy.github.io/posts/unify-logging-for-a-gunicorn-uvicorn-app/
+    From https://github.com/benoitc/gunicorn/issues/1572#issuecomment-638391953
     https://github.com/pawamoy/pawamoy.github.io/issues/17
     """
-    # Intercept everything at the root logger
+    # Set the logging handler and level
     logging.root.handlers = [InterceptHandler()]
-    logging.root.setLevel(settings.LOG_ROOT_LEVEL)
+    logging.root.setLevel('NOTSET')
 
     # Remove all log handlers and propagate to root logger
     for name in logging.root.manager.loggerDict.keys():
@@ -55,70 +54,65 @@ def setup_logging():
         # Debug log handlers
         # logging.debug(f'{logging.getLogger(name)}, {logging.getLogger(name).propagate}')
 
-    # Remove every other logger's handlers
-    logger.remove()
-
-    # Define the correlation_id filter function
-    # https://github.com/snok/asgi-correlation-id?tab=readme-ov-file#configure-logging
+    # Define the correlation_id default filter function
     # https://github.com/snok/asgi-correlation-id/issues/7
-    def correlation_id_filter(record) -> bool:
+    def correlation_id_filter(record):
         cid = correlation_id.get(settings.LOG_CID_DEFAULT_VALUE)
         record['correlation_id'] = cid[: settings.LOG_CID_UUID_LENGTH]
-        return True
+        return record
+
+    # Remove default loguru logger
+    logger.remove()
 
-    # Configure loguru logger before starts logging
+    # Set the loguru default handlers
     logger.configure(
         handlers=[
             {
-                'sink': stdout,
-                'level': settings.LOG_STDOUT_LEVEL,
-                'filter': lambda record: correlation_id_filter(record) and record['level'].no <= 25,
-                'format': settings.LOG_STD_FORMAT,
-            },
-            {
-                'sink': stderr,
-                'level': settings.LOG_STDERR_LEVEL,
-                'filter': lambda record: correlation_id_filter(record) and record['level'].no >= 30,
+                'sink': sys.stdout,
+                'filter': lambda record: correlation_id_filter(record),
                 'format': settings.LOG_STD_FORMAT,
-            },
+            }
         ]
     )
 
 
-def set_customize_logfile():
+def set_custom_logfile():
     log_path = path_conf.LOG_DIR
     if not os.path.exists(log_path):
         os.mkdir(log_path)
 
     # log files
-    log_stdout_file = os.path.join(log_path, settings.LOG_STDOUT_FILENAME)
-    log_stderr_file = os.path.join(log_path, settings.LOG_STDERR_FILENAME)
+    log_access_file = os.path.join(log_path, settings.LOG_ACCESS_FILENAME)
+    log_error_file = os.path.join(log_path, settings.LOG_ERROR_FILENAME)
 
-    # loguru logger: https://loguru.readthedocs.io/en/stable/api/logger.html#loguru._logger.Logger.add
+    # set loguru logger default config
+    # https://loguru.readthedocs.io/en/stable/api/logger.html#loguru._logger.Logger.add
     log_config = {
-        'rotation': '10 MB',
-        'retention': '15 days',
-        'compression': 'tar.gz',
-        'enqueue': True,
         'format': settings.LOG_FILE_FORMAT,
+        'enqueue': True,
+        'rotation': '5 MB',
+        'retention': '7 days',
+        'compression': 'tar.gz',
     }
 
     # stdout file
     logger.add(
-        str(log_stdout_file),
-        level=settings.LOG_STDOUT_LEVEL,
-        **log_config,
+        str(log_access_file),
+        level=settings.LOG_ACCESS_LEVEL,
+        filter=lambda record: record['level'].no <= 25,
         backtrace=False,
         diagnose=False,
+        **log_config,
     )
 
     # stderr file
     logger.add(
-        str(log_stderr_file),
-        level=settings.LOG_STDERR_LEVEL,
-        **log_config,
+        str(log_error_file),
+        level=settings.LOG_ERROR_LEVEL,
+        filter=lambda record: record['level'].no >= 30,
         backtrace=True,
         diagnose=True,
+        **log_config,
     )
 
 

backend/core/conf.py

@@ -94,7 +94,6 @@ class Settings(BaseSettings):
     COOKIE_REFRESH_TOKEN_EXPIRE_SECONDS: int = TOKEN_REFRESH_EXPIRE_SECONDS
 
     # Log
-    LOG_ROOT_LEVEL: str = 'NOTSET'
     LOG_STD_FORMAT: str = (
         '<green>{time:YYYY-MM-DD HH:mm:ss.SSS}</> | <lvl>{level: <8}</> | '
         '<cyan> {correlation_id} </> | <lvl>{message}</>'
@@ -105,10 +104,10 @@ class Settings(BaseSettings):
     )
     LOG_CID_DEFAULT_VALUE: str = '-'
     LOG_CID_UUID_LENGTH: int = 32  # must <= 32
-    LOG_STDOUT_LEVEL: str = 'INFO'
-    LOG_STDERR_LEVEL: str = 'ERROR'
-    LOG_STDOUT_FILENAME: str = 'fba_access.log'
-    LOG_STDERR_FILENAME: str = 'fba_error.log'
+    LOG_ACCESS_LEVEL: str = 'INFO'
+    LOG_ERROR_LEVEL: str = 'ERROR'
+    LOG_ACCESS_FILENAME: str = 'fba_access.log'
+    LOG_ERROR_FILENAME: str = 'fba_error.log'
 
     # Middleware
     MIDDLEWARE_CORS: bool = True

backend/core/registrar.py

@@ -14,7 +14,7 @@
 from starlette.staticfiles import StaticFiles
 
 from backend.common.exception.exception_handler import register_exception
-from backend.common.log import set_customize_logfile, setup_logging
+from backend.common.log import set_custom_logfile, setup_logging
 from backend.core.conf import settings
 from backend.core.path_conf import STATIC_DIR, UPLOAD_DIR
 from backend.database.db import create_table
@@ -99,7 +99,7 @@ def register_logger() -> None:
     :return:
     """
     setup_logging()
-    set_customize_logfile()
+    set_custom_logfile()
 
 
 def register_static_file(app: FastAPI):

deploy/backend/celery.conf

@@ -7,6 +7,8 @@ autorestart=true
 startretries=5
 redirect_stderr=true
 stdout_logfile=/var/log/celery/fba_celery_worker.log
+stdout_logfile_maxbytes=5MB
+stdout_logfile_backups=5
 
 [program:celery_beat]
 directory=/fba/backend
@@ -17,6 +19,8 @@ autorestart=true
 startretries=5
 redirect_stderr=true
 stdout_logfile=/var/log/celery/fba_celery_beat.log
+stdout_logfile_maxbytes=5MB
+stdout_logfile_backups=5
 
 [program:celery_flower]
 directory=/fba/backend
@@ -27,3 +31,5 @@ autorestart=true
 startretries=5
 redirect_stderr=true
 stdout_logfile=/var/log/celery/fba_celery_flower.log
+stdout_logfile_maxbytes=5MB
+stdout_logfile_backups=5

deploy/backend/fastapi_server.conf

@@ -7,3 +7,5 @@ autorestart=true
 startretries=5
 redirect_stderr=true
 stdout_logfile=/var/log/fastapi_server/fba_server.log
+stdout_logfile_maxbytes=5MB
+stdout_logfile_backups=5

deploy/backend/gunicorn.conf.py

@@ -26,11 +26,11 @@
 worker_connections = 2000
 
 # 设置进程文件目录
-pidfile = '/fba/gunicorn.pid'
+pidfile = '/var/run/gunicorn.pid'
 
-# 设置访问日志和错误信息日志路径
-accesslog = '/var/log/fastapi_server/gunicorn_access.log'
-errorlog = '/var/log/fastapi_server/gunicorn_error.log'
+# 设置访问日志和错误信息日志
+accesslog = '-'
+errorlog = '-'
 
 # 设置这个值为true 才会把打印信息记录到错误日志里
 capture_output = True