Skip to content

Commit 8c481e3

Browse files
authored
Merge pull request #127 from scaleapi/add-better-logging-for-dd
Add better logging for dd
2 parents e4b39b5 + a52c980 commit 8c481e3

File tree

7 files changed

+125
-19
lines changed

7 files changed

+125
-19
lines changed

pyproject.toml

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,11 +36,14 @@ dependencies = [
3636
"tzlocal>=5.3.1",
3737
"tzdata>=2025.2",
3838
"pytest>=8.4.0",
39+
"json_log_formatter>=1.1.1",
3940
"pytest-asyncio>=1.0.0",
4041
"scale-gp-beta==0.1.0a20",
4142
"ipykernel>=6.29.5",
4243
"openai==1.99.9", # anything higher than 1.99.9 breaks litellm - https://github.com/BerriAI/litellm/issues/13711
4344
"cloudpickle>=3.1.1",
45+
"datadog>=0.52.1",
46+
"ddtrace>=3.13.0"
4447
]
4548
requires-python = ">= 3.12,<4"
4649
classifiers = [
@@ -222,7 +225,7 @@ warn_unused_ignores = false
222225
warn_redundant_casts = false
223226

224227
disallow_any_generics = true
225-
disallow_untyped_defs = true
228+
# disallow_untyped_defs = true
226229
disallow_untyped_calls = true
227230
disallow_subclassing_any = true
228231
disallow_incomplete_defs = true
@@ -238,7 +241,7 @@ cache_fine_grained = true
238241
# ```
239242
# Changing this codegen to make mypy happy would increase complexity
240243
# and would not be worth it.
241-
disable_error_code = "func-returns-value,overload-cannot-match"
244+
disable_error_code = "func-returns-value,overload-cannot-match,no-untyped-def"
242245

243246
# https://github.com/python/mypy/issues/12162
244247
[[tool.mypy.overrides]]

requirements-dev.lock

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,8 @@ attrs==25.3.0
4040
# via aiohttp
4141
# via jsonschema
4242
# via referencing
43+
bytecode==0.17.0
44+
# via ddtrace
4345
cachetools==5.5.2
4446
# via google-auth
4547
certifi==2023.7.22
@@ -61,6 +63,10 @@ colorlog==6.7.0
6163
# via nox
6264
comm==0.2.3
6365
# via ipykernel
66+
datadog==0.52.1
67+
# via agentex-sdk
68+
ddtrace==3.15.0
69+
# via agentex-sdk
6470
debugpy==1.8.16
6571
# via ipykernel
6672
decorator==5.2.1
@@ -73,6 +79,8 @@ distro==1.8.0
7379
# via openai
7480
# via scale-gp
7581
# via scale-gp-beta
82+
envier==0.6.1
83+
# via ddtrace
7684
execnet==2.1.1
7785
# via pytest-xdist
7886
executing==2.2.0
@@ -120,6 +128,7 @@ idna==3.4
120128
# via yarl
121129
importlib-metadata==7.0.0
122130
# via litellm
131+
# via opentelemetry-api
123132
iniconfig==2.0.0
124133
# via pytest
125134
ipykernel==6.30.1
@@ -135,6 +144,8 @@ jinja2==3.1.6
135144
# via litellm
136145
jiter==0.10.0
137146
# via openai
147+
json-log-formatter==1.1.1
148+
# via agentex-sdk
138149
jsonref==1.1.0
139150
# via agentex-sdk
140151
jsonschema==4.25.0
@@ -186,6 +197,8 @@ openai==1.99.9
186197
# via openai-agents
187198
openai-agents==0.2.7
188199
# via agentex-sdk
200+
opentelemetry-api==1.37.0
201+
# via ddtrace
189202
packaging==23.2
190203
# via huggingface-hub
191204
# via ipykernel
@@ -207,6 +220,7 @@ propcache==0.3.1
207220
# via aiohttp
208221
# via yarl
209222
protobuf==5.29.5
223+
# via ddtrace
210224
# via temporalio
211225
psutil==7.0.0
212226
# via ipykernel
@@ -280,6 +294,7 @@ referencing==0.36.2
280294
regex==2025.7.34
281295
# via tiktoken
282296
requests==2.32.4
297+
# via datadog
283298
# via huggingface-hub
284299
# via kubernetes
285300
# via openai-agents
@@ -363,6 +378,7 @@ typing-extensions==4.12.2
363378
# via nexus-rpc
364379
# via openai
365380
# via openai-agents
381+
# via opentelemetry-api
366382
# via pydantic
367383
# via pydantic-core
368384
# via pyright
@@ -393,6 +409,8 @@ wcwidth==0.2.13
393409
# via prompt-toolkit
394410
websocket-client==1.8.0
395411
# via kubernetes
412+
wrapt==1.17.3
413+
# via ddtrace
396414
yarl==1.20.0
397415
# via aiohttp
398416
zipp==3.17.0

requirements.lock

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,8 @@ attrs==25.3.0
3838
# via aiohttp
3939
# via jsonschema
4040
# via referencing
41+
bytecode==0.17.0
42+
# via ddtrace
4143
cachetools==5.5.2
4244
# via google-auth
4345
certifi==2023.7.22
@@ -57,6 +59,10 @@ colorama==0.4.6
5759
# via griffe
5860
comm==0.2.3
5961
# via ipykernel
62+
datadog==0.52.1
63+
# via agentex-sdk
64+
ddtrace==3.15.0
65+
# via agentex-sdk
6066
debugpy==1.8.16
6167
# via ipykernel
6268
decorator==5.2.1
@@ -66,6 +72,8 @@ distro==1.8.0
6672
# via openai
6773
# via scale-gp
6874
# via scale-gp-beta
75+
envier==0.6.1
76+
# via ddtrace
6977
executing==2.2.0
7078
# via stack-data
7179
fastapi==0.115.14
@@ -109,6 +117,7 @@ idna==3.4
109117
# via yarl
110118
importlib-metadata==8.7.0
111119
# via litellm
120+
# via opentelemetry-api
112121
iniconfig==2.1.0
113122
# via pytest
114123
ipykernel==6.30.1
@@ -124,6 +133,8 @@ jinja2==3.1.6
124133
# via litellm
125134
jiter==0.10.0
126135
# via openai
136+
json-log-formatter==1.1.1
137+
# via agentex-sdk
127138
jsonref==1.1.0
128139
# via agentex-sdk
129140
jsonschema==4.25.0
@@ -169,6 +180,8 @@ openai==1.99.9
169180
# via openai-agents
170181
openai-agents==0.2.7
171182
# via agentex-sdk
183+
opentelemetry-api==1.37.0
184+
# via ddtrace
172185
packaging==25.0
173186
# via huggingface-hub
174187
# via ipykernel
@@ -188,6 +201,7 @@ propcache==0.3.1
188201
# via aiohttp
189202
# via yarl
190203
protobuf==5.29.5
204+
# via ddtrace
191205
# via temporalio
192206
psutil==7.0.0
193207
# via ipykernel
@@ -255,6 +269,7 @@ referencing==0.36.2
255269
regex==2025.7.34
256270
# via tiktoken
257271
requests==2.32.4
272+
# via datadog
258273
# via huggingface-hub
259274
# via kubernetes
260275
# via openai-agents
@@ -333,6 +348,7 @@ typing-extensions==4.12.2
333348
# via nexus-rpc
334349
# via openai
335350
# via openai-agents
351+
# via opentelemetry-api
336352
# via pydantic
337353
# via pydantic-core
338354
# via python-on-whales
@@ -360,6 +376,8 @@ wcwidth==0.2.13
360376
# via prompt-toolkit
361377
websocket-client==1.8.0
362378
# via kubernetes
379+
wrapt==1.17.3
380+
# via ddtrace
363381
yarl==1.20.0
364382
# via aiohttp
365383
zipp==3.23.0

src/agentex/lib/environment_variables.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ class EnvVarKeys(str, Enum):
3939

4040

4141
class Environment(str, Enum):
42+
LOCAL = "local"
4243
DEV = "development"
4344
STAGING = "staging"
4445
PROD = "production"

src/agentex/lib/sdk/fastacp/base/base_acp_server.py

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import asyncio
22
import inspect
3+
import uuid
34
from datetime import datetime
45
from collections.abc import AsyncGenerator, Awaitable, Callable
56
from contextlib import asynccontextmanager
@@ -9,6 +10,7 @@
910
from fastapi import FastAPI, Request
1011
from fastapi.responses import StreamingResponse
1112
from pydantic import TypeAdapter, ValidationError
13+
from starlette.middleware.base import BaseHTTPMiddleware
1214

1315
# from agentex.lib.sdk.fastacp.types import BaseACPConfig
1416
from agentex.lib.environment_variables import EnvironmentVariables, refreshed_environment_variables
@@ -24,7 +26,7 @@
2426
from agentex.lib.types.json_rpc import JSONRPCError, JSONRPCRequest, JSONRPCResponse
2527
from agentex.types.task_message_update import StreamTaskMessageFull, TaskMessageUpdate
2628
from agentex.types.task_message_content import TaskMessageContent
27-
from agentex.lib.utils.logging import make_logger
29+
from agentex.lib.utils.logging import ctx_var_request_id, make_logger
2830
from agentex.lib.utils.model_utils import BaseModel
2931
from agentex.lib.utils.registration import register_agent
3032
from agentex.lib.sdk.fastacp.base.constants import (
@@ -38,6 +40,20 @@
3840
task_message_update_adapter = TypeAdapter(TaskMessageUpdate)
3941

4042

43+
class RequestIDMiddleware(BaseHTTPMiddleware):
44+
"""Middleware to extract or generate request IDs and add them to logs and response headers"""
45+
46+
async def dispatch(self, request: Request, call_next):
47+
# Extract request ID from header or generate a new one if there isn't one
48+
request_id = request.headers.get("x-request-id") or uuid.uuid4().hex
49+
logger.info(f"Request ID: {request_id}")
50+
# Store request ID in request state for access in handlers
51+
ctx_var_request_id.set(request_id)
52+
# Process request
53+
response = await call_next(request)
54+
return response
55+
56+
4157
class BaseACPServer(FastAPI):
4258
"""
4359
AsyncAgentACP provides RPC-style hooks for agent events and commands asynchronously.
@@ -56,6 +72,8 @@ def __init__(self):
5672
self.post("/api")(self._handle_jsonrpc)
5773

5874
# Method handlers
75+
# this just adds a request ID to the request and response headers
76+
self.add_middleware(RequestIDMiddleware)
5977
self._handlers: dict[RPCMethod, Callable] = {}
6078

6179
@classmethod

src/agentex/lib/utils/logging.py

Lines changed: 63 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,31 +1,79 @@
11
import logging
2-
2+
import contextvars
33
from rich.console import Console
44
from rich.logging import RichHandler
5+
import json_log_formatter
6+
import os
7+
import ddtrace
8+
from ddtrace import tracer
9+
10+
_is_datadog_configured = bool(os.environ.get("DD_AGENT_HOST"))
11+
12+
ctx_var_request_id = contextvars.ContextVar[str]("request_id")
13+
14+
15+
class CustomJSONFormatter(json_log_formatter.JSONFormatter):
16+
def json_record(self, message: str, extra: dict, record: logging.LogRecord) -> dict:
17+
extra = super().json_record(message, extra, record)
18+
extra["level"] = record.levelname
19+
extra["name"] = record.name
20+
extra["lineno"] = record.lineno
21+
extra["pathname"] = record.pathname
22+
extra["request_id"] = ctx_var_request_id.get(None)
23+
if _is_datadog_configured:
24+
extra["dd.trace_id"] = tracer.get_log_correlation_context().get("dd.trace_id", None) or getattr(
25+
record, "dd.trace_id", 0
26+
)
27+
extra["dd.span_id"] = tracer.get_log_correlation_context().get("dd.span_id", None) or getattr(
28+
record, "dd.span_id", 0
29+
)
30+
# add the env, service, and version configured for the tracer
31+
# If tracing is not set up, then this should pull values from DD_ENV, DD_SERVICE, and DD_VERSION.
32+
service_override = ddtrace.config.service or os.getenv("DD_SERVICE")
33+
if service_override:
34+
extra["dd.service"] = service_override
35+
36+
env_override = ddtrace.config.env or os.getenv("DD_ENV")
37+
if env_override:
38+
extra["dd.env"] = env_override
539

40+
version_override = ddtrace.config.version or os.getenv("DD_VERSION")
41+
if version_override:
42+
extra["dd.version"] = version_override
643

7-
def make_logger(name: str):
44+
return extra
45+
46+
def make_logger(name: str) -> logging.Logger:
847
"""
948
Creates a logger object with a RichHandler to print colored text.
1049
:param name: The name of the module to create the logger for.
1150
:return: A logger object.
1251
"""
1352
# Create a console object to print colored text
14-
console = Console()
15-
16-
# Create a logger object with the name of the current module
1753
logger = logging.getLogger(name)
18-
19-
# Set the global log level to INFO
2054
logger.setLevel(logging.INFO)
2155

22-
# Add the RichHandler to the logger to print colored text
23-
handler = RichHandler(
24-
console=console,
25-
show_level=False,
26-
show_path=False,
27-
show_time=False,
28-
)
29-
logger.addHandler(handler)
56+
environment = os.getenv("ENVIRONMENT")
57+
if environment == "local":
58+
console = Console()
59+
# Add the RichHandler to the logger to print colored text
60+
handler = RichHandler(
61+
console=console,
62+
show_level=False,
63+
show_path=False,
64+
show_time=False,
65+
)
66+
logger.addHandler(handler)
67+
return logger
68+
69+
stream_handler = logging.StreamHandler()
70+
if _is_datadog_configured:
71+
stream_handler.setFormatter(CustomJSONFormatter())
72+
else:
73+
stream_handler.setFormatter(
74+
logging.Formatter("%(asctime)s %(levelname)s [%(name)s] [%(filename)s:%(lineno)d] - %(message)s")
75+
)
3076

77+
logger.addHandler(stream_handler)
78+
# Create a logger object with the name of the current module
3179
return logger

uv.lock

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)