Skip to content

Commit ec260df

Browse files
monoxgasrdheekondabriangreunke
authored
feat: v1 Merge (ENG-1616) (#19)
* In the middle of updates * Finalizing scoring->metrics and inputs/params for tasks * Reworking object storage * shuffling some things prior to merge * Pending work * Initial serialization code * Plug serialization and schema generation into log_object. Rename kind -> label to avoid clashing with OTEL stuff * Add logic to write the objects to s3 * Fixes for v1 compat API changes * Fixes for pydantic schema generation failures in serialization * Fixing some serialization behavior related to primitives and Nones. Reduced some nested json serialization. * Lots of docstring writing * Add audio and video serialize handlers (#3) * Adjust Task prop extraction for decorator nesting * README updates * hotfix: Trying to correct endpoint_url issues * Fixing @task decorator typing * Object linking for tasks. * Remove dead ext folder * Add Log Artifact for Files and Directories (#18) * Implement log artifact handles dierctory and individual file * Add example notebook; working log artifact with directory and file; cleanup * Implement smart artifact merging, removed unnecessary info from otel, leveraged s3fs put for batch uploading, adress pr feedback * Moved some artifact components around. Fixed typing/linting errors. Cleaned up logging, warning, and internal error handling * Additional logging updates for the api client * Version to v1.0.0-rc.0 * chore: linting - updated ruff version - marked noqa for existing errors for future work - fixed hooks linting errors - updated config for test and hook linting excludes --------- Co-authored-by: Raja Sekhar Rao Dheekonda <raja@dreadnode.io> Co-authored-by: Raja Sekhar Rao Dheekonda <43563047+rdheekonda@users.noreply.github.com> Co-authored-by: Brian Greunke <briangreunke@pm.me> Co-authored-by: Brian Greunke <44581702+briangreunke@users.noreply.github.com>
1 parent 7d2ae09 commit ec260df

36 files changed

+7817
-1516
lines changed

‎.github/CONTRIBUTING.md‎

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@ We actively welcome your pull requests.
1212
3. If you've changed APIs, update the documentation.
1313
4. Ensure the test suite passes.
1414
5. Make sure your code lints.
15-
6. If you haven't already, complete the Contributor License Agreement ("CLA").
1615

1716
### PR Description Format
1817

@@ -54,13 +53,6 @@ Example:
5453
- Deprecated setup scripts
5554
```
5655

57-
## Contributor License Agreement ("CLA")
58-
59-
In order to accept your pull request, we need you to submit a CLA. You only need
60-
to do this once to work on any of Facebook's open source projects.
61-
62-
Complete your CLA here: <https://code.facebook.com/cla>
63-
6456
## Issues
6557

6658
We use GitHub issues to track public bugs. Please ensure your description is

‎.github/labeler.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,7 @@ type/docs:
6767
- changed-files:
6868
- any-glob-to-any-file: "docs/**/*"
6969
- any-glob-to-any-file: "*.md"
70+
- any-glob-to-any-file: "*.mdx"
7071

7172
# Core Files Labels
7273
type/core:

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
# Testing code
22
notebooks/
3+
examples/.logfire
34

45
# Logfire temp
56
.logfire/

‎.hooks/check_pinned_hash_dependencies.py‎

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,9 @@ def __init__(self) -> None:
1010
self.pinned_pattern = re.compile(r"uses:\s+([^@\s]+)@([a-f0-9]{40})")
1111

1212
# Pattern for actions with version tags (unpinned)
13-
self.unpinned_pattern = re.compile(r"uses:\s+([^@\s]+)@(v\d+(?:\.\d+)*(?:-[a-zA-Z0-9]+(?:\.\d+)*)?)")
13+
self.unpinned_pattern = re.compile(
14+
r"uses:\s+([^@\s]+)@(v\d+(?:\.\d+)*(?:-[a-zA-Z0-9]+(?:\.\d+)*)?)",
15+
)
1416

1517
# Pattern for all uses statements
1618
self.all_uses_pattern = re.compile(r"uses:\s+([^@\s]+)@([^\s\n]+)")
@@ -30,16 +32,18 @@ def format_terminal_link(self, file_path: str, line_number: int) -> str:
3032
def get_line_numbers(self, content: str, pattern: re.Pattern[str]) -> list[tuple[str, int]]:
3133
"""Find matches with their line numbers."""
3234
matches = []
33-
for i, line in enumerate(content.splitlines(), 1):
34-
for match in pattern.finditer(line):
35-
matches.append((match.group(0), i))
35+
matches.extend(
36+
(match.group(0), i)
37+
for i, line in enumerate(content.splitlines(), 1)
38+
for match in pattern.finditer(line)
39+
)
3640
return matches
3741

3842
def check_file(self, file_path: str) -> bool:
3943
"""Check a single file for unpinned dependencies."""
4044
try:
4145
content = Path(file_path).read_text()
42-
except Exception as e:
46+
except (FileNotFoundError, PermissionError, IsADirectoryError, OSError) as e:
4347
print(f"\033[91mError reading file {file_path}: {e}\033[0m")
4448
return False
4549

@@ -84,7 +88,9 @@ def check_file(self, file_path: str) -> bool:
8488
has_errors = True
8589
print("\033[91m[!] Completely unpinned (no SHA or version):\033[0m")
8690
for match, line_num in unpinned_without_hash:
87-
print(f" |- {match} \033[90m({self.format_terminal_link(file_path, line_num)})\033[0m")
91+
print(
92+
f" |- {match} \033[90m({self.format_terminal_link(file_path, line_num)})\033[0m",
93+
)
8894

8995
# Print summary
9096
total_actions = len(pinned_matches) + len(unpinned_matches) + len(unpinned_without_hash)

‎.hooks/generate_pr_description.py‎

Lines changed: 19 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -9,14 +9,17 @@
99
import asyncio
1010
import os
1111
import typing as t
12+
from pathlib import Path
1213

1314
import rigging as rg
1415
import typer
1516

16-
TRUNCATION_WARNING = "\n---\n**Note**: Due to the large size of this diff, some content has been truncated."
17+
TRUNCATION_WARNING = (
18+
"\n---\n**Note**: Due to the large size of this diff, some content has been truncated."
19+
)
1720

1821

19-
@rg.prompt # type: ignore
22+
@rg.prompt
2023
def generate_pr_description(diff: str) -> t.Annotated[str, rg.Ctx("markdown")]: # type: ignore[empty-body]
2124
"""
2225
Analyze the provided git diff and create a PR description in markdown format.
@@ -40,13 +43,19 @@ async def _run_git_command(args: list[str]) -> str:
4043
"""
4144
# Validate git exists in PATH
4245
git_path = "git" # Could use shutil.which("git") for more security
43-
if not any(os.path.isfile(os.path.join(path, "git")) for path in os.environ["PATH"].split(os.pathsep)):
46+
if not any(
47+
Path(path).joinpath("git").is_file() for path in os.environ["PATH"].split(os.pathsep)
48+
):
4449
raise ValueError("Git executable not found in PATH")
4550

4651
# Validate input parameters
4752
if not all(isinstance(arg, str) for arg in args):
4853
raise ValueError("All command arguments must be strings")
4954

55+
def check_return_code(return_code: int):
56+
if return_code != 0:
57+
raise RuntimeError(f"Git command failed: {stderr.decode()}")
58+
5059
# Use os.execv for more secure command execution
5160
try:
5261
# nosec B603 - Input is validated
@@ -58,12 +67,15 @@ async def _run_git_command(args: list[str]) -> str:
5867
)
5968
stdout, stderr = await proc.communicate()
6069

61-
if proc.returncode != 0:
62-
raise RuntimeError(f"Git command failed: {stderr.decode()}")
70+
check_return_code(proc.returncode)
6371

6472
return stdout.decode().strip()
65-
except Exception as e:
66-
raise RuntimeError(f"Failed to execute git command: {e}") from e
73+
except FileNotFoundError as e:
74+
raise RuntimeError("Git executable not found or invalid command") from e
75+
except asyncio.SubprocessError as e:
76+
raise RuntimeError("Error occurred while running the subprocess") from e
77+
except UnicodeDecodeError as e:
78+
raise RuntimeError("Failed to decode the output of the git command") from e
6779

6880

6981
async def get_diff(base_ref: str, source_ref: str, *, exclude: list[str] | None = None) -> str:

‎.pre-commit-config.yaml‎

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -87,15 +87,3 @@ repos:
8787
entry: .hooks/prettier.sh
8888
language: script
8989
types: [json, yaml]
90-
91-
- id: generate-readme
92-
name: Generate README
93-
description: Updates auto-generated sections in README.md
94-
entry: python scripts/generate_readme.py
95-
language: python
96-
files: ^(pyproject\.toml|templates/README\.md\.j2)$
97-
pass_filenames: false
98-
require_serial: true
99-
additional_dependencies:
100-
- jinja2
101-
- tomli>=2.0.1

‎README.md‎

Lines changed: 96 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,99 @@
1-
# sdk
1+
<p align="center">
2+
<img
3+
src="https://d1lppblt9t2x15.cloudfront.net/logos/5714928f3cdc09503751580cffbe8d02.png"
4+
alt="Logo"
5+
align="center"
6+
width="144px"
7+
height="144px"
8+
/>
9+
</p>
210

3-
<!-- BEGIN_AUTO_BADGES -->
4-
<div align="center">
11+
<h3 align="center">
12+
Dreadnode Strikes SDK
13+
</h3>
514

6-
[![Pre-Commit](https://github.com/dreadnode/python-template/actions/workflows/pre-commit.yaml/badge.svg)](https://github.com/dreadnode/python-template/actions/workflows/pre-commit.yaml)
7-
[![Renovate](https://github.com/dreadnode/python-template/actions/workflows/renovate.yaml/badge.svg)](https://github.com/dreadnode/python-template/actions/workflows/renovate.yaml)
15+
<h4 align="center">
16+
<img alt="PyPI - Python Version" src="https://img.shields.io/pypi/pyversions/dreadnode">
17+
<img alt="PyPI - Version" src="https://img.shields.io/pypi/v/dreadnode">
18+
<img alt="GitHub License" src="https://img.shields.io/github/license/dreadnode/sdk">
19+
<img alt="Tests" src="https://img.shields.io/github/actions/workflow/status/dreadnode/sdk/tests.yaml">
20+
<img alt="Pre-Commit" src="https://img.shields.io/github/actions/workflow/status/dreadnode/sdk/pre-commit.yaml">
21+
<img alt="Renovate" src="https://img.shields.io/github/actions/workflow/status/dreadnode/sdk/renovate.yaml">
22+
</h4>
823

9-
</div>
10-
<!-- END_AUTO_BADGES -->
11-
Dreadnode SDK
24+
</br>
25+
26+
Strikes is an platform for building, experimenting with, and evaluating AI security agent code.
27+
28+
- **Experiment + Tasking + Observability** in a single place that's lightweight and scales.
29+
- **Track your data** with parameters, inputs, and outputs all connected to your tasks.
30+
- **Measure everything** with metrics throughout your code and anywhere you need them.
31+
- **Scale your code** from a single run to thousands.
32+
33+
```python
34+
import dreadnode as dn
35+
import rigging as rg
36+
37+
from .tools import reversing_tools
38+
39+
dn.configure()
40+
41+
@dataclass
42+
class Finding:
43+
name: str
44+
severity: str
45+
description: str
46+
exploit_code: str
47+
48+
@dn.scorer(name="Score Finding")
49+
async def score_finding(finding: Finding) -> float:
50+
if finding.severity == "critical":
51+
return 1.0
52+
elif finding.severity == "high":
53+
return 0.8
54+
else:
55+
return 0.2
56+
57+
@dn.task(scorers=[score_finding])
58+
@rg.prompt(tools=[reversing_tools])
59+
async def analyze_binary(binary: str) -> list[Finding]:
60+
"""
61+
Analyze the binary for vulnerabilities.
62+
"""
63+
...
64+
65+
with dn.run(tags=["reverse-engineering"]):
66+
binary = "c2/downloads/service.exe"
67+
68+
dn.log_params(
69+
model="gpt-4",
70+
temperature=0.5,
71+
binary=binary
72+
)
73+
74+
findings = await analyze_binary(binary)
75+
76+
dn.log_metric("findings", len(findings))
77+
```
78+
79+
## Installation
80+
81+
We publish every version to PyPi:
82+
```bash
83+
pip install -U dreadnode
84+
```
85+
86+
If you want to build from source:
87+
```bash
88+
poetry install
89+
```
90+
91+
See our **[installation guide](https://docs.dreadnode.io/strikes/install)** for more options.
92+
93+
## Getting Started
94+
95+
Read through our **[introduction guide](https://docs.dreadnode.io/strikes/intro)** in the docs.
96+
97+
## Examples
98+
99+
Check out **[dreadnode/example-agents](https://github.com/dreadnode/example-agents)** to find your favorite use case.

‎dreadnode/__init__.py‎

Lines changed: 28 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
1-
from .main import DEFAULT_INSTANCE
2-
from .score import Scorer
3-
from .task import Task
4-
from .tracing import RunSpan, Score, Span, TaskSpan
5-
from .version import VERSION
1+
from dreadnode.main import DEFAULT_INSTANCE, Dreadnode
2+
from dreadnode.metric import Metric, MetricDict, Scorer
3+
from dreadnode.object import Object
4+
from dreadnode.task import Task
5+
from dreadnode.tracing.span import RunSpan, Span, TaskSpan
6+
from dreadnode.version import VERSION
67

78
configure = DEFAULT_INSTANCE.configure
89
shutdown = DEFAULT_INSTANCE.shutdown
@@ -12,29 +13,39 @@
1213
task = DEFAULT_INSTANCE.task
1314
task_span = DEFAULT_INSTANCE.task_span
1415
run = DEFAULT_INSTANCE.run
16+
scorer = DEFAULT_INSTANCE.scorer
17+
task_span = DEFAULT_INSTANCE.task_span
1518
push_update = DEFAULT_INSTANCE.push_update
1619

1720
log_metric = DEFAULT_INSTANCE.log_metric
1821
log_param = DEFAULT_INSTANCE.log_param
1922
log_params = DEFAULT_INSTANCE.log_params
20-
log_score = DEFAULT_INSTANCE.log_score
23+
log_input = DEFAULT_INSTANCE.log_input
24+
log_inputs = DEFAULT_INSTANCE.log_inputs
25+
log_output = DEFAULT_INSTANCE.log_output
26+
link_objects = DEFAULT_INSTANCE.link_objects
27+
log_artifact = DEFAULT_INSTANCE.log_artifact
2128

2229
__version__ = VERSION
2330

2431
__all__ = [
25-
"configure",
26-
"shutdown",
27-
"span",
28-
"task",
29-
"run",
30-
"log_metric",
31-
"log_param",
32+
"Dreadnode",
33+
"Metric",
34+
"MetricDict",
35+
"Object",
3236
"Run",
33-
"Task",
34-
"Scorer",
37+
"RunSpan",
3538
"Score",
36-
"TaskSpan",
39+
"Scorer",
3740
"Span",
38-
"RunSpan",
41+
"Task",
42+
"TaskSpan",
3943
"__version__",
44+
"configure",
45+
"log_metric",
46+
"log_param",
47+
"run",
48+
"shutdown",
49+
"span",
50+
"task",
4051
]

0 commit comments

Comments
 (0)