Skip to content

Commit 4886153

Browse files
authored
fix: Autolog Mechanics (#99)
* Remove parameter logging from tasks. Add useful serialization check for task inputs when autologging. Don't autolog None outputs. Make execution metric logging explicit. * Apply seems_useful_to_serialize to task outputs * Update docs to align with code changes
1 parent debd391 commit 4886153

File tree

10 files changed

+175
-177
lines changed

10 files changed

+175
-177
lines changed

docs/intro.mdx

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -237,7 +237,7 @@ async def execute_command(command: str) -> str:
237237
stdout, stderr = await process.communicate()
238238

239239
# Log additional information
240-
dn.log_param("exit_code", process.returncode)
240+
dn.log_output("exit_code", process.returncode)
241241

242242
result = stdout.decode() if process.returncode == 0 else stderr.decode()
243243
return result # Automatically logged as output

docs/sdk/main.mdx

Lines changed: 24 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -1280,12 +1280,7 @@ def log_outputs(
12801280
### log\_param
12811281

12821282
```python
1283-
log_param(
1284-
key: str,
1285-
value: JsonValue,
1286-
*,
1287-
to: ToObject = "task-or-run",
1288-
) -> None
1283+
log_param(key: str, value: JsonValue) -> None
12891284
```
12901285

12911286
Log a single parameter to the current task or run.
@@ -1309,13 +1304,6 @@ with dreadnode.run("my_run"):
13091304
* **`value`**
13101305
(`JsonValue`)
13111306
–The value of the parameter.
1312-
* **`to`**
1313-
(`ToObject`, default:
1314-
`'task-or-run'`
1315-
)
1316-
–The target object to log the parameter to. Can be "task-or-run" or "run".
1317-
Defaults to "task-or-run". If "task-or-run", the parameter will be logged
1318-
to the current task or run, whichever is the nearest ancestor.
13191307

13201308
<Accordion title="Source code in dreadnode/main.py" icon="code">
13211309
```python
@@ -1324,8 +1312,6 @@ def log_param(
13241312
self,
13251313
key: str,
13261314
value: JsonValue,
1327-
*,
1328-
to: ToObject = "task-or-run",
13291315
) -> None:
13301316
"""
13311317
Log a single parameter to the current task or run.
@@ -1343,11 +1329,8 @@ def log_param(
13431329
Args:
13441330
key: The name of the parameter.
13451331
value: The value of the parameter.
1346-
to: The target object to log the parameter to. Can be "task-or-run" or "run".
1347-
Defaults to "task-or-run". If "task-or-run", the parameter will be logged
1348-
to the current task or run, whichever is the nearest ancestor.
13491332
"""
1350-
self.log_params(to=to, **{key: value})
1333+
self.log_params(**{key: value})
13511334
```
13521335

13531336

@@ -1356,9 +1339,7 @@ def log_param(
13561339
### log\_params
13571340

13581341
```python
1359-
log_params(
1360-
to: ToObject = "run", **params: JsonValue
1361-
) -> None
1342+
log_params(**params: JsonValue) -> None
13621343
```
13631344

13641345
Log multiple parameters to the current task or run.
@@ -1379,13 +1360,6 @@ with dreadnode.run("my_run"):
13791360

13801361
**Parameters:**
13811362

1382-
* **`to`**
1383-
(`ToObject`, default:
1384-
`'run'`
1385-
)
1386-
–The target object to log the parameters to. Can be "task-or-run" or "run".
1387-
Defaults to "task-or-run". If "task-or-run", the parameters will be logged
1388-
to the current task or run, whichever is the nearest ancestor.
13891363
* **`**params`**
13901364
(`JsonValue`, default:
13911365
`{}`
@@ -1395,7 +1369,7 @@ with dreadnode.run("my_run"):
13951369
<Accordion title="Source code in dreadnode/main.py" icon="code">
13961370
```python
13971371
@handle_internal_errors()
1398-
def log_params(self, to: ToObject = "run", **params: JsonValue) -> None:
1372+
def log_params(self, **params: JsonValue) -> None:
13991373
"""
14001374
Log multiple parameters to the current task or run.
14011375
@@ -1413,19 +1387,11 @@ def log_params(self, to: ToObject = "run", **params: JsonValue) -> None:
14131387
~~~
14141388
14151389
Args:
1416-
to: The target object to log the parameters to. Can be "task-or-run" or "run".
1417-
Defaults to "task-or-run". If "task-or-run", the parameters will be logged
1418-
to the current task or run, whichever is the nearest ancestor.
14191390
**params: The parameters to log. Each parameter is a key-value pair.
14201391
"""
1421-
task = current_task_span.get()
1422-
run = current_run_span.get()
1423-
1424-
target = (task or run) if to == "task-or-run" else run
1425-
if target is None:
1426-
raise RuntimeError("log_params() must be called within a run")
1427-
1428-
target.log_params(**params)
1392+
if (run := current_run_span.get()) is None:
1393+
raise RuntimeError("Parameters must be logged within a run")
1394+
run.log_params(**params)
14291395
```
14301396

14311397

@@ -1543,7 +1509,7 @@ with dreadnode.run("my_run"):
15431509
(`bool`, default:
15441510
`True`
15451511
)
1546-
–Whether to automatically log task inputs, outputs, and execution metrics if unspecified.
1512+
–Whether to automatically log task inputs, outputs, and execution metrics if otherwise unspecified.
15471513
* **`**attributes`**
15481514
(`Any`, default:
15491515
`{}`
@@ -1592,7 +1558,7 @@ def run(
15921558
project: The project name to associate the run with. If not provided,
15931559
the project passed to `configure()` will be used, or the
15941560
run will be associated with a default project.
1595-
autolog: Whether to automatically log task inputs, outputs, and execution metrics if unspecified.
1561+
autolog: Whether to automatically log task inputs, outputs, and execution metrics if otherwise unspecified.
15961562
**attributes: Additional attributes to attach to the run span.
15971563
15981564
Returns:
@@ -1919,11 +1885,11 @@ task(
19191885
scorers: None = None,
19201886
name: str | None = None,
19211887
label: str | None = None,
1922-
log_params: Sequence[str] | bool = False,
19231888
log_inputs: Sequence[str]
19241889
| bool
19251890
| Inherited = INHERITED,
19261891
log_output: bool | Inherited = INHERITED,
1892+
log_execution_metrics: bool = False,
19271893
tags: Sequence[str] | None = None,
19281894
**attributes: Any,
19291895
) -> TaskDecorator
@@ -1935,11 +1901,11 @@ task(
19351901
scorers: Sequence[Scorer[R] | ScorerCallable[R]],
19361902
name: str | None = None,
19371903
label: str | None = None,
1938-
log_params: Sequence[str] | bool = False,
19391904
log_inputs: Sequence[str]
19401905
| bool
19411906
| Inherited = INHERITED,
19421907
log_output: bool | Inherited = INHERITED,
1908+
log_execution_metrics: bool = False,
19431909
tags: Sequence[str] | None = None,
19441910
**attributes: Any,
19451911
) -> ScoredTaskDecorator[R]
@@ -1952,11 +1918,11 @@ task(
19521918
| None = None,
19531919
name: str | None = None,
19541920
label: str | None = None,
1955-
log_params: Sequence[str] | bool = False,
19561921
log_inputs: Sequence[str]
19571922
| bool
19581923
| Inherited = INHERITED,
19591924
log_output: bool | Inherited = INHERITED,
1925+
log_execution_metrics: bool = False,
19601926
tags: Sequence[str] | None = None,
19611927
**attributes: Any,
19621928
) -> TaskDecorator
@@ -1992,21 +1958,21 @@ await my_task(2)
19921958
`None`
19931959
)
19941960
–The label of the task - useful for filtering in the UI.
1995-
* **`log_params`**
1996-
(`Sequence[str] | bool`, default:
1997-
`False`
1998-
)
1999-
–Whether to log all, or specific, incoming arguments to the function as parameters.
20001961
* **`log_inputs`**
20011962
(`Sequence[str] | bool | Inherited`, default:
20021963
`INHERITED`
20031964
)
2004-
Whether to log all, or specific, incoming arguments to the function as inputs.
1965+
Log all, or specific, incoming arguments to the function as inputs.
20051966
* **`log_output`**
20061967
(`bool | Inherited`, default:
20071968
`INHERITED`
20081969
)
2009-
–Whether to log the result of the function as an output.
1970+
–Log the result of the function as an output.
1971+
* **`log_execution_metrics`**
1972+
(`bool`, default:
1973+
`False`
1974+
)
1975+
–Log execution metrics for the task, such as success rate and run count.
20101976
* **`tags`**
20111977
(`Sequence[str] | None`, default:
20121978
`None`
@@ -2031,9 +1997,9 @@ def task(
20311997
scorers: t.Sequence[Scorer[t.Any] | ScorerCallable[t.Any]] | None = None,
20321998
name: str | None = None,
20331999
label: str | None = None,
2034-
log_params: t.Sequence[str] | bool = False,
20352000
log_inputs: t.Sequence[str] | bool | Inherited = INHERITED,
20362001
log_output: bool | Inherited = INHERITED,
2002+
log_execution_metrics: bool = False,
20372003
tags: t.Sequence[str] | None = None,
20382004
**attributes: t.Any,
20392005
) -> TaskDecorator:
@@ -2054,9 +2020,9 @@ def task(
20542020
of the task and will be passed the task's output.
20552021
name: The name of the task.
20562022
label: The label of the task - useful for filtering in the UI.
2057-
log_params: Whether to log all, or specific, incoming arguments to the function as parameters.
2058-
log_inputs: Whether to log all, or specific, incoming arguments to the function as inputs.
2059-
log_output: Whether to log the result of the function as an output.
2023+
log_inputs: Log all, or specific, incoming arguments to the function as inputs.
2024+
log_output: Log the result of the function as an output.
2025+
log_execution_metrics: Log execution metrics for the task, such as success rate and run count.
20602026
tags: A list of tags to attach to the task span.
20612027
**attributes: A dictionary of attributes to attach to the task span.
20622028
@@ -2109,9 +2075,9 @@ def task(
21092075
for scorer in scorers or []
21102076
],
21112077
tags=list(tags or []),
2112-
log_params=log_params,
21132078
log_inputs=log_inputs,
21142079
log_output=log_output,
2080+
log_execution_metrics=log_execution_metrics,
21152081
label=_label,
21162082
)
21172083

@@ -2128,7 +2094,6 @@ task_span(
21282094
name: str,
21292095
*,
21302096
label: str | None = None,
2131-
params: AnyDict | None = None,
21322097
tags: Sequence[str] | None = None,
21332098
**attributes: Any,
21342099
) -> TaskSpan[t.Any]
@@ -2150,7 +2115,6 @@ async with dreadnode.task_span("my_task") as task:
21502115
Args:
21512116
name: The name of the task.
21522117
label: The label of the task - useful for filtering in the UI.
2153-
params: A dictionary of parameters to attach to the task span.
21542118
tags: A list of tags to attach to the task span.
21552119
\*\*attributes: A dictionary of attributes to attach to the task span.
21562120

@@ -2166,7 +2130,6 @@ def task_span(
21662130
name: str,
21672131
*,
21682132
label: str | None = None,
2169-
params: AnyDict | None = None,
21702133
tags: t.Sequence[str] | None = None,
21712134
**attributes: t.Any,
21722135
) -> TaskSpan[t.Any]:
@@ -2185,7 +2148,6 @@ def task_span(
21852148
Args:
21862149
name: The name of the task.
21872150
label: The label of the task - useful for filtering in the UI.
2188-
params: A dictionary of parameters to attach to the task span.
21892151
tags: A list of tags to attach to the task span.
21902152
**attributes: A dictionary of attributes to attach to the task span.
21912153
@@ -2200,7 +2162,6 @@ def task_span(
22002162
name=name,
22012163
label=label,
22022164
attributes=attributes,
2203-
params=params,
22042165
tags=tags,
22052166
run_id=run.run_id,
22062167
tracer=self._get_tracer(),

docs/sdk/serialization.mdx

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,53 @@ title: dreadnode.serialization
66
::: dreadnode.serialization
77
*/}
88

9+
seems\_useful\_to\_serialize
10+
----------------------------
11+
12+
```python
13+
seems_useful_to_serialize(obj: Any) -> bool
14+
```
15+
16+
Checks if the object is likely useful to serialize by attempting to
17+
serialize it and checking if the resulting schema indicates a known type.
18+
19+
**Parameters:**
20+
21+
* **`obj`**
22+
(`Any`)
23+
–The Python object to check.
24+
25+
**Returns:**
26+
27+
* **`bool`** ( `bool`
28+
) –True if the object is likely useful to serialize, False otherwise.
29+
30+
<Accordion title="Source code in dreadnode/serialization.py" icon="code">
31+
```python
32+
def seems_useful_to_serialize(obj: t.Any) -> bool:
33+
"""
34+
Checks if the object is likely useful to serialize by attempting to
35+
serialize it and checking if the resulting schema indicates a known type.
36+
37+
Args:
38+
obj: The Python object to check.
39+
40+
Returns:
41+
bool: True if the object is likely useful to serialize, False otherwise.
42+
"""
43+
if obj is None:
44+
return False
45+
46+
with contextlib.suppress(Exception):
47+
_, schema = _serialize(obj)
48+
return schema.get("x-python-datatype") != "unknown"
49+
50+
return False
51+
```
52+
53+
54+
</Accordion>
55+
956
serialize
1057
---------
1158

0 commit comments

Comments
 (0)