Use gunicorn log config from logstick

.secrets.baseline

@@ -242,10 +242,10 @@
         "filename": "tests/integration/test_schema_migration.py",
         "hashed_secret": "5666c088b494f26cd8f63ace013992f5fc391ce0",
         "is_verified": false,
-        "line_number": 80,
+        "line_number": 81,
         "is_secret": false
       }
     ]
   },
-  "generated_at": "2025-03-13T17:55:47Z"
+  "generated_at": "2025-04-15T03:06:18Z"
 }

Dockerfile

@@ -1,4 +1,3 @@
-# syntax=docker/dockerfile:1
 ARG BASE_VERSION=3.2.2
 ARG REGISTRY=docker.osdc.io
 ARG SERVICE_NAME=indexd
@@ -17,9 +16,9 @@ WORKDIR /${SERVICE_NAME}
 
 COPY . .
 RUN pip install --upgrade setuptools pip \
-    && pip install versionista>=1.1.0 --extra-index-url https://nexus.osdc.io/repository/pypi-gdc-releases/simple \
+    && pip install versionista>=1.1.0 --extra-index-url "$PIP_INDEX_URL" \
     && python3 -m setuptools_scm \
-    && pip install --no-deps -r requirements.txt .
+    && pip install -c requirements.txt .[server]
 
 FROM ${REGISTRY}/ncigdc/${PYTHON_VERSION}:${BASE_VERSION}
 ARG NAME

gunicorn.conf.py

@@ -1,12 +1,8 @@
 from __future__ import annotations
 
-import datetime
-import logging
 import os
-import sys
 
-import json_log_formatter
-from ddtrace import tracer
+from logstick.recipes import GUNICORN_LOG_CONFIG
 
 # Based on the example from https://github.com/benoitc/gunicorn/blob/master/examples/example_config.py
 #
@@ -157,117 +153,5 @@
 loglevel = "info"
 accesslog = "-"
 
-
-# Testing example from https://til.codeinthehole.com/posts/how-to-get-gunicorn-to-log-as-json/
-class JsonRequestFormatter(json_log_formatter.JSONFormatter):
-    def json_record(
-        self,
-        message: str,
-        extra: dict[str, str | int | float],
-        record: logging.LogRecord,
-    ) -> dict[str, str | int | float]:
-        # Convert the log record to a JSON object.
-        # See https://docs.gunicorn.org/en/stable/settings.html#access-log-format
-
-        response_time = datetime.datetime.strptime(
-            record.args["t"], "[%d/%b/%Y:%H:%M:%S %z]"
-        )
-        url = record.args["U"]
-        if record.args["q"]:
-            url += f"?{record.args['q']}"
-
-        span = tracer.current_span()
-        trace_id, span_id = (
-            (str((1 << 64) - 1 & span.trace_id), span.span_id) if span else (None, None)
-        )
-
-        return dict(
-            ts=response_time.isoformat(),
-            path=url,
-            query=record.args["q"],
-            http=dict(
-                status_code=str(record.args["s"]),
-                method=record.args["m"],
-                response_body_bytes=record.args["b"],
-                user_agent=record.args["a"],
-                referer=record.args["f"],
-                x_forwarded_for=record.args["{x-forwarded-for}i"],
-            ),
-            remote_addr=record.args["h"],
-            remote_user=record.args["u"],
-            protocol=record.args["H"],
-            duration_in_ms=record.args["M"],
-            traceparent=record.args["{traceparent}i"],
-            tracestate=record.args["{tracestate}i"],
-            dd=dict(
-                trace_id=str(trace_id or 0),
-                span_id=str(span_id or 0),
-            ),
-        )
-
-
-class JsonErrorFormatter(json_log_formatter.JSONFormatter):
-    def json_record(
-        self,
-        message: str,
-        extra: dict[str, str | int | float],
-        record: logging.LogRecord,
-    ) -> dict[str, str | int | float]:
-        payload: dict[str, str | int | float] = super().json_record(
-            message, extra, record
-        )
-        span = tracer.current_span()
-        trace_id, span_id = (
-            (str((1 << 64) - 1 & span.trace_id), span.span_id) if span else (None, None)
-        )
-        payload["dd.trace_id"] = str(trace_id or 0)
-        payload["dd.span_id"] = str(span_id or 0)
-        payload["level"] = record.levelname
-        return payload
-
-
-gunicorn_loglevel = os.getenv("APP_GUNICORN_LOGLEVEL", "INFO")
-generic_loglevel = os.getenv("APP_GENERIC_LOGLEVEL", "INFO")
-# Ensure the two named loggers that Gunicorn uses are configured to use a custom
 # JSON formatter.
-logconfig_dict = {
-    "version": 1,
-    "formatters": {
-        "json_request": {
-            "()": JsonRequestFormatter,
-        },
-        "json_error": {
-            "()": JsonErrorFormatter,
-        },
-    },
-    "handlers": {
-        "json_request": {
-            "class": "logging.StreamHandler",
-            "stream": sys.stdout,
-            "formatter": "json_request",
-        },
-        "json_error": {
-            "class": "logging.StreamHandler",
-            "stream": sys.stdout,
-            "formatter": "json_error",
-        },
-    },
-    "root": {"level": "INFO", "handlers": []},
-    "loggers": {
-        "gunicorn.access": {
-            "level": gunicorn_loglevel,
-            "handlers": ["json_request"],
-            "propagate": False,
-        },
-        "gunicorn.error": {
-            "level": gunicorn_loglevel,
-            "handlers": ["json_error"],
-            "propagate": False,
-        },
-        "": {
-            "level": generic_loglevel,
-            "handlers": ["json_error"],
-            "propagate": False,
-        },
-    },
-}
+logconfig_dict = GUNICORN_LOG_CONFIG

pyproject.toml

@@ -36,13 +36,8 @@ source = ["indexd"]
 minversion = 6.0
 python_files = "test_*.py"
 python_functions = "test_*"
-# Silence warning by opting in to use pytest 6's new default value
-junit_family = "xunit2"
-junit_logging = "system-out"
-console_output_style = "classic"
 log_cli_level = "INFO"
-log_level = "INFO"
-log_cli = "true"
+log_cli = "1"
 log_auto_indent = "true"
 addopts = [
   "-rvlFE --color=true -p no:ddtrace -p no:ddtrace.pytest_bdd"
@@ -60,6 +55,12 @@ testpaths = [
   "tests/unit",
   "tests/integration"
 ]
+env = [
+  "SQLALCHEMY_WARN_20=1",
+  "LOGSTICK_LOG_LEVEL=debug",
+  "LOGSTICK_OUTPUT_FORMAT=raw",
+  "LOGSTICK_ROOT_LOG_LEVEL=info"
+]
 
 [tool.ruff]
 line-length = 88 # black's default

requirements.txt

@@ -12,8 +12,6 @@ attrs==25.3.0
     #   referencing
 blinker==1.9.0
     # via flask
-bytecode==0.16.1
-    # via ddtrace
 certifi==2025.1.31
     # via requests
 charset-normalizer==3.4.1
@@ -24,46 +22,23 @@ colorama==0.4.6
     # via logstick
 colorlog==6.9.0
     # via logstick
-ddtrace==3.4.1
-    # via indexd (pyproject.toml)
 deprecated==1.2.18
-    # via opentelemetry-api
-envier==0.6.1
-    # via ddtrace
+    # via logstick
 flask==3.1.0
     # via indexd (pyproject.toml)
-gunicorn==23.0.0
-    # via indexd (pyproject.toml)
+greenlet==3.2.0
+    # via sqlalchemy
 idna==3.10
     # via requests
-importlib-metadata==8.6.1
-    # via
-    #   indexd (pyproject.toml)
-    #   opentelemetry-api
-importlib-resources==6.1.3
-    # via indexd (pyproject.toml)
 itsdangerous==2.2.0
     # via flask
 jinja2==3.1.6
     # via flask
-json-log-formatter==1.1.1
-    # via indexd (pyproject.toml)
 jsonschema==4.23.0
-    # via
-    #   indexd (pyproject.toml)
-    #   openapi-schema-validator
-    #   openapi-spec-validator
-jsonschema-spec==0.2.4
-    # via openapi-spec-validator
-jsonschema-specifications==2023.7.1
-    # via
-    #   jsonschema
-    #   openapi-schema-validator
-lazy-object-proxy==1.9.0
-    # via openapi-spec-validator
-legacy-cgi==2.6.2
-    # via ddtrace
-logstick==0.1.3
+    # via indexd (pyproject.toml)
+jsonschema-specifications==2024.10.1
+    # via jsonschema
+logstick==0.1.5
     # via indexd (pyproject.toml)
 markdown-it-py==3.0.0
     # via rich
@@ -73,70 +48,41 @@ markupsafe==3.0.2
     #   werkzeug
 mdurl==0.1.2
     # via markdown-it-py
-openapi-schema-validator==0.6.3
-    # via openapi-spec-validator
-openapi-spec-validator==0.7.0
-    # via indexd (pyproject.toml)
-opentelemetry-api==1.31.0
-    # via ddtrace
-packaging==24.2
-    # via gunicorn
-pathable==0.4.4
-    # via jsonschema-spec
-protobuf==6.30.1
-    # via ddtrace
 psycopg2==2.9.10
     # via indexd (pyproject.toml)
 pygments==2.19.1
     # via rich
-pyyaml==6.0.2
-    # via jsonschema-spec
-referencing==0.30.2
+referencing==0.36.2
     # via
     #   jsonschema
-    #   jsonschema-spec
     #   jsonschema-specifications
 requests==2.32.3
-    # via
-    #   indexd (pyproject.toml)
-    #   jsonschema-spec
-rfc3339-validator==0.1.4
-    # via openapi-schema-validator
-rich==13.9.4
+    # via indexd (pyproject.toml)
+rich==14.0.0
     # via logstick
-rpds-py==0.23.1
+rpds-py==0.24.0
     # via
     #   jsonschema
     #   referencing
-setproctitle==1.3.5
-    # via indexd (pyproject.toml)
-six==1.17.0
-    # via rfc3339-validator
-sqlalchemy==1.3.24
+sqlalchemy==2.0.40
     # via
     #   indexd (pyproject.toml)
     #   sqlalchemy-utils
 sqlalchemy-utils==0.41.2
     # via indexd (pyproject.toml)
 structlog==25.2.0
     # via logstick
-typing-extensions==4.12.2
+typing-extensions==4.13.2
     # via
-    #   ddtrace
     #   indexd (pyproject.toml)
-urllib3==2.3.0
+    #   sqlalchemy
+urllib3==2.4.0
     # via requests
 werkzeug==3.1.3
     # via
     #   flask
     #   indexd (pyproject.toml)
 wrapt==1.17.2
-    # via
-    #   ddtrace
-    #   deprecated
-xmltodict==0.14.2
-    # via ddtrace
+    # via deprecated
 zipp==3.21.0
-    # via
-    #   importlib-metadata
-    #   indexd (pyproject.toml)
+    # via indexd (pyproject.toml)

setup.cfg

@@ -27,22 +27,15 @@ python_requires = >=3.13
 include_package_data = True
 install_requires =
     flask>=2.2
-    jsonschema==v4.23.0
-    importlib_resources<6.2  # TODO: >=6.2 fails for py38 but works ok for 39+
-    logstick
-    sqlalchemy<1.4  # TODO: Unpin sqlalchemy. Pinning is only required when psqlgraph is involved.
+    jsonschema>4.17
+    logstick>=0.1.5
+    sqlalchemy>=2.0
     sqlalchemy-utils>=0.32
     psycopg2>=2.7
     requests>=2.32.2
-    ddtrace>=2.9.1
-    importlib-metadata>=1.4
     typing-extensions
     zipp>=3.19.1
     werkzeug>=3.0.6
-    gunicorn>=23.0.0
-    setproctitle>=1.3.4
-    JSON-log-formatter>=1.1
-    openapi_spec_validator==0.7.0
 
 [options.packages.find]
 where = src
@@ -55,6 +48,11 @@ dev =
     pytest-flask
     PyYAML
     openapi-spec-validator
+server =
+    gunicorn>=23.0.0
+    ddtrace>2.9.1
+    setproctitle>=1.3.4
+    JSON-log-formatter>=1.1
 
 [options.entry_points]
 console_scripts =

src/indexd/__init__.py

@@ -2,6 +2,10 @@
 
 import logstick
 
-logstick.configure_logging(namespace=__name__, disable_existing_loggers=False)
+logstick.configure_logging(
+    namespace=__name__,
+    disable_existing_loggers=False,
+    extra_namespaces=["werkzeug"],
+)
 __distribution = distribution(__name__)
 VERSION = __distribution.version