Skip to content

Commit 891ea21

Browse files
adinhodovicadinhodovic
authored
Merge pull request #238 from honeylogic-io/update-logging
feat: Use structlog
2 parents 7b7a87f + ddb5dc3 commit 891ea21

File tree

6 files changed

+287
-46
lines changed

6 files changed

+287
-46
lines changed

‎config/celery_app.py‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
11
import os
2+
from logging.config import dictConfig
23
from pathlib import Path
34

45
from celery import Celery, bootsteps
56
from celery.schedules import crontab
6-
from celery.signals import worker_ready, worker_shutdown
7+
from celery.signals import setup_logging, worker_ready, worker_shutdown
8+
from django.conf import settings
79

810
# set the default Django settings module for the 'celery' program.
911
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.production")
@@ -17,6 +19,11 @@
1719
app.config_from_object("django.conf:settings", namespace="CELERY")
1820

1921

22+
@setup_logging.connect
23+
def config_loggers(*args, **kwargs):
24+
dictConfig(settings.LOGGING)
25+
26+
2027
HEARTBEAT_FILE = Path("/tmp/worker_heartbeat")
2128
READINESS_FILE = Path("/tmp/worker_ready")
2229

‎config/settings/base.py‎

Lines changed: 109 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
from pathlib import Path
66

77
import environ
8+
import structlog
89
from django.urls import reverse_lazy
910

1011
ROOT_DIR = Path(__file__).resolve(strict=True).parent.parent.parent
@@ -92,6 +93,7 @@
9293
"django_custom_error_views",
9394
"django_admin_shellx",
9495
"django_htmx",
96+
"django_structlog",
9597
"health_check",
9698
"meta",
9799
"modelcluster",
@@ -171,6 +173,7 @@
171173
# https://docs.djangoproject.com/en/dev/ref/settings/#middleware
172174
MIDDLEWARE = [
173175
"django_prometheus.middleware.PrometheusBeforeMiddleware",
176+
"django_structlog.middlewares.RequestMiddleware",
174177
"django.middleware.security.SecurityMiddleware",
175178
"whitenoise.middleware.WhiteNoiseMiddleware",
176179
"django.contrib.sessions.middleware.SessionMiddleware",
@@ -283,23 +286,121 @@
283286
# https://docs.djangoproject.com/en/dev/ref/settings/#logging
284287
# See https://docs.djangoproject.com/en/dev/topics/logging for
285288
# more details on how to customize your logging configuration.
289+
DJANGO_ROOT_LOG_LEVEL = env("DJANGO_ROOT_LOG_LEVEL", default="WARNING")
290+
DJANGO_LOG_LEVEL = env("DJANGO_LOG_LEVEL", default="INFO")
291+
DJANGO_REQUEST_LOG_LEVEL = env("DJANGO_REQUEST_LOG_LEVEL", default="INFO")
292+
DJANGO_CELERY_LOG_LEVEL = env("DJANGO_CELERY_LOG_LEVEL", default="INFO")
293+
DJANGO_DATABASE_LOG_LEVEL = env("DJANGO_DATABASE_LOG_LEVEL", default="CRITICAL")
294+
DJANGO_STRUCTLOG_CELERY_ENABLED = True
295+
296+
297+
shared_structlog_processors = [
298+
structlog.contextvars.merge_contextvars,
299+
structlog.stdlib.add_logger_name,
300+
structlog.stdlib.add_log_level,
301+
# Perform %-style formatting.
302+
structlog.stdlib.PositionalArgumentsFormatter(),
303+
# Add a timestamp in ISO 8601 format.
304+
structlog.processors.TimeStamper(fmt="iso"),
305+
structlog.processors.StackInfoRenderer(),
306+
# If some value is in bytes, decode it to a unicode str.
307+
structlog.processors.UnicodeDecoder(),
308+
# Add callsite parameters.
309+
structlog.processors.CallsiteParameterAdder(
310+
{
311+
structlog.processors.CallsiteParameter.FILENAME,
312+
structlog.processors.CallsiteParameter.FUNC_NAME,
313+
structlog.processors.CallsiteParameter.LINENO,
314+
}
315+
),
316+
]
317+
318+
# Logging filtering is handled by the logging library itself
319+
base_structlog_processors = shared_structlog_processors + [
320+
structlog.stdlib.filter_by_level,
321+
]
322+
323+
base_structlog_formatter = [structlog.stdlib.ProcessorFormatter.wrap_for_formatter]
324+
325+
structlog.configure(
326+
processors=base_structlog_processors + base_structlog_formatter, # type: ignore
327+
logger_factory=structlog.stdlib.LoggerFactory(),
328+
wrapper_class=structlog.stdlib.BoundLogger,
329+
cache_logger_on_first_use=True,
330+
)
331+
286332
LOGGING = {
287333
"version": 1,
288334
"disable_existing_loggers": False,
289335
"formatters": {
290-
"verbose": {
291-
"format": "%(levelname)s %(asctime)s %(module)s "
292-
"%(process)d %(thread)d %(message)s"
293-
}
336+
"colored_console": {
337+
"()": structlog.stdlib.ProcessorFormatter,
338+
"processor": structlog.dev.ConsoleRenderer(colors=True),
339+
"foreign_pre_chain": shared_structlog_processors,
340+
},
341+
"json_formatter": {
342+
"()": structlog.stdlib.ProcessorFormatter,
343+
"processor": structlog.processors.JSONRenderer(),
344+
"foreign_pre_chain": shared_structlog_processors,
345+
},
294346
},
295347
"handlers": {
296348
"console": {
297-
"level": "DEBUG",
298349
"class": "logging.StreamHandler",
299-
"formatter": "verbose",
300-
}
350+
"formatter": "colored_console",
351+
},
352+
"json": {
353+
"class": "logging.StreamHandler",
354+
"formatter": "json_formatter",
355+
},
356+
"null": {
357+
"class": "logging.NullHandler",
358+
},
359+
},
360+
"root": {
361+
"handlers": ["console"],
362+
"level": DJANGO_ROOT_LOG_LEVEL,
363+
},
364+
"loggers": {
365+
"django_structlog": {
366+
"level": DJANGO_LOG_LEVEL,
367+
},
368+
# Django Structlog request middlewares
369+
"django_structlog.middlewares": {
370+
"level": DJANGO_REQUEST_LOG_LEVEL,
371+
},
372+
# Django Structlog Celery receivers
373+
"django_structlog.celery": {
374+
"level": DJANGO_CELERY_LOG_LEVEL,
375+
},
376+
"your_app": {
377+
"level": DJANGO_LOG_LEVEL,
378+
},
379+
# DB logs
380+
"django.db.backends": {
381+
"level": DJANGO_DATABASE_LOG_LEVEL,
382+
},
383+
# Use structlog middleware
384+
"django.server": {
385+
"handlers": ["null"],
386+
"propagate": False,
387+
},
388+
# Use structlog middleware
389+
"django.request": {
390+
"handlers": ["null"],
391+
"propagate": False,
392+
},
393+
# Use structlog middleware
394+
"django.channels.server": {
395+
"handlers": ["null"],
396+
"propagate": False,
397+
},
398+
# Use structlog middleware
399+
"werkzeug": {
400+
"handlers": ["null"],
401+
"propagate": False,
402+
},
301403
},
302-
"root": {"level": "INFO", "handlers": ["console"]},
303404
}
304405

305406
# Celery

‎config/settings/local.py‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
11
# pylint: disable=wildcard-import,unused-wildcard-import
2+
import logging
3+
24
from .base import *
35
from .base import env
46

@@ -65,6 +67,29 @@
6567
hostname, _, ips = socket.gethostbyname_ex(socket.gethostname())
6668
INTERNAL_IPS += [".".join(ip.split(".")[:-1] + ["1"]) for ip in ips]
6769

70+
# LOGGING
71+
# ------------------------------------------------------------------------------
72+
# https://docs.djangoproject.com/en/dev/ref/settings/#logging
73+
# See https://docs.djangoproject.com/en/dev/topics/logging for
74+
# more details on how to customize your logging configuration.
75+
# pylint: disable=duplicate-code
76+
77+
DEV_FILTERED_EVENTS = ["request_started"]
78+
79+
80+
class DevelopmentFilter(logging.Filter): # pylint: disable=too-few-public-methods
81+
"""Filter out events in development so they don't clutter the console"""
82+
83+
def filter(self, record):
84+
if DEBUG and isinstance(record.msg, dict):
85+
event = record.msg.get("event")
86+
if event in DEV_FILTERED_EVENTS:
87+
return False
88+
return True
89+
90+
91+
LOGGING["handlers"]["console"]["filters"] = [DevelopmentFilter()] # type: ignore
92+
6893
# Celery
6994
# ------------------------------------------------------------------------------
7095

‎config/settings/production.py‎

Lines changed: 23 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -140,42 +140,29 @@
140140
# A sample logging configuration. The only tangible logging
141141
# performed by this configuration is to send an email to
142142
# the site admins on every HTTP 500 error when DEBUG=False.
143-
LOGGING = {
144-
"version": 1,
145-
"disable_existing_loggers": False,
146-
"filters": {"require_debug_false": {"()": "django.utils.log.RequireDebugFalse"}},
147-
"formatters": {
148-
"verbose": {
149-
"format": "%(levelname)s %(asctime)s %(module)s "
150-
"%(process)d %(thread)d %(message)s"
151-
}
152-
},
153-
"handlers": {
154-
"mail_admins": {
155-
"level": "ERROR",
156-
"filters": ["require_debug_false"],
157-
"class": "django.utils.log.AdminEmailHandler",
158-
},
159-
"console": {
160-
"level": "DEBUG",
161-
"class": "logging.StreamHandler",
162-
"formatter": "verbose",
163-
},
164-
},
165-
"root": {"level": "INFO", "handlers": ["console"]},
166-
"loggers": {
167-
"django.request": {
168-
"handlers": ["mail_admins"],
169-
"level": "ERROR",
170-
"propagate": True,
171-
},
172-
"django.security.DisallowedHost": {
173-
"level": "ERROR",
174-
"handlers": ["console", "mail_admins"],
175-
"propagate": True,
176-
},
177-
},
178-
}
143+
DJANGO_REQUEST_LOG_LEVEL = env("DJANGO_REQUEST_LOG_LEVEL", default="WARNING")
144+
145+
production_structlog_processors = [
146+
structlog.processors.dict_tracebacks,
147+
]
148+
149+
production_shared_structlog_processors = (
150+
shared_structlog_processors + production_structlog_processors
151+
)
152+
153+
structlog.configure(
154+
processors=base_structlog_processors
155+
+ production_structlog_processors
156+
+ base_structlog_formatter, # type: ignore
157+
logger_factory=structlog.stdlib.LoggerFactory(),
158+
cache_logger_on_first_use=True,
159+
)
160+
161+
LOGGING["root"]["handlers"] = ["json"] # type: ignore
162+
LOGGING["loggers"]["django_structlog.middlewares"]["level"] = DJANGO_REQUEST_LOG_LEVEL # type: ignore
163+
164+
# Add dict tracebacks to the JSON formatter
165+
LOGGING["formatters"]["json_formatter"]["foreign_pre_chain"] = production_shared_structlog_processors # type: ignore
179166

180167
# Your stuff...
181168
# ------------------------------------------------------------------------------

0 commit comments

Comments
 (0)