Skip to content

Commit 87ee552

Browse files
mishushakovmishushakov
committed
added performance tests
1 parent f0011a5 commit 87ee552

File tree

5 files changed

+265
-33
lines changed

5 files changed

+265
-33
lines changed

‎.github/workflows/performance_tests.yml‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
name: Performance tests
2+
3+
on:
4+
workflow_call:
5+
secrets:
6+
E2B_API_KEY:
7+
required: true
8+
inputs:
9+
E2B_DOMAIN:
10+
required: false
11+
type: string
12+
E2B_TESTS_TEMPLATE:
13+
required: false
14+
type: string
15+
16+
permissions:
17+
contents: read
18+
19+
jobs:
20+
publish:
21+
defaults:
22+
run:
23+
working-directory: ./python
24+
name: Performance tests
25+
runs-on: ubuntu-22.04
26+
steps:
27+
- name: Checkout repository
28+
uses: actions/checkout@v4
29+
30+
- name: Set up Python
31+
uses: actions/setup-python@v4
32+
with:
33+
python-version: '3.12'
34+
35+
- name: Install and configure Poetry
36+
uses: snok/install-poetry@v1
37+
with:
38+
version: 1.5.1
39+
virtualenvs-create: true
40+
virtualenvs-in-project: true
41+
installer-parallel: true
42+
43+
- name: Install dependencies
44+
run: poetry install
45+
46+
- name: Run performance tests
47+
run: poetry run python tests/performance.py
48+
env:
49+
E2B_API_KEY: ${{ secrets.E2B_API_KEY }}
50+
E2B_DOMAIN: ${{ inputs.E2B_DOMAIN }}
51+
E2B_TESTS_TEMPLATE: ${{ inputs.E2B_TESTS_TEMPLATE }}

‎.github/workflows/pull_request.yml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,14 @@ jobs:
3636
with:
3737
E2B_DOMAIN: ${{ vars.E2B_DOMAIN }}
3838
E2B_TESTS_TEMPLATE: ${{ needs.build-template.outputs.template_id }}
39+
performance-tests:
40+
uses: ./.github/workflows/performance_tests.yml
41+
needs: build-template
42+
secrets:
43+
E2B_API_KEY: ${{ secrets.E2B_API_KEY }}
44+
with:
45+
E2B_DOMAIN: ${{ vars.E2B_DOMAIN }}
46+
E2B_TESTS_TEMPLATE: ${{ needs.build-template.outputs.template_id }}
3947
cleanup-build-template:
4048
uses: ./.github/workflows/cleanup_build_template.yml
4149
needs: [build-template, js-sdk, python-sdk]

‎python/pyproject.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ pytest-asyncio = "^0.23.7"
2424
pytest-xdist = "^3.6.1"
2525
black = "23.12.1"
2626
pydoc-markdown = "^4.8.2"
27+
matplotlib = "^3.8.0"
2728

2829
[build-system]
2930
requires = ["poetry-core"]

‎python/tests/benchmarking.py‎

Lines changed: 0 additions & 33 deletions
This file was deleted.

‎python/tests/performance.py‎

Lines changed: 205 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,205 @@
1+
from e2b_code_interpreter import Sandbox
2+
import time
3+
import os
4+
import statistics
5+
import matplotlib.pyplot as plt
6+
7+
iterations_count = int(os.getenv("E2B_TESTS_PERF_ITERATIONS_COUNT", 20))
8+
template = os.getenv("E2B_TESTS_TEMPLATE", "code-interpreter-v1")
9+
10+
# Lists to store metrics for each iteration
11+
sandbox_creation_times = []
12+
health_check_times = []
13+
first_code_run_times = []
14+
second_code_run_times = []
15+
16+
for i in range(iterations_count):
17+
print(f"\n--- Iteration {i + 1}/{iterations_count} ---")
18+
19+
start_time = time.time()
20+
sbx = Sandbox(template)
21+
end_time = time.time()
22+
sandbox_creation_time = (end_time - start_time) * 1000
23+
sandbox_creation_times.append(sandbox_creation_time)
24+
print(f"Sandbox creation time: {sandbox_creation_time:.2f} milliseconds")
25+
26+
start_time = time.time()
27+
sbx.commands.run("curl http://0.0.0.0:49999/health")
28+
end_time = time.time()
29+
health_check_time = (end_time - start_time) * 1000
30+
health_check_times.append(health_check_time)
31+
print(f"Health check time: {health_check_time:.2f} milliseconds")
32+
33+
start_time = time.time()
34+
sbx.run_code("print('Hello, world!')")
35+
end_time = time.time()
36+
first_code_run_time = (end_time - start_time) * 1000
37+
first_code_run_times.append(first_code_run_time)
38+
print(f"First code run time: {first_code_run_time:.2f} milliseconds")
39+
40+
start_time = time.time()
41+
sbx.run_code("print('Hello, world!')")
42+
end_time = time.time()
43+
second_code_run_time = (end_time - start_time) * 1000
44+
second_code_run_times.append(second_code_run_time)
45+
print(f"Second code run time: {second_code_run_time:.2f} milliseconds")
46+
47+
sbx.kill()
48+
49+
50+
# Calculate and print summary statistics
51+
def print_metric_summary(metric_name, times):
52+
if not times:
53+
return
54+
55+
low = min(times)
56+
high = max(times)
57+
mean = statistics.mean(times)
58+
median = statistics.median(times)
59+
60+
print(f"\n{metric_name} Summary:")
61+
print(f" Low: {low:.2f} ms")
62+
print(f" High: {high:.2f} ms")
63+
print(f" Mean: {mean:.2f} ms")
64+
print(f" Median: {median:.2f} ms")
65+
66+
67+
print("\n" + "=" * 50)
68+
print("PERFORMANCE SUMMARY")
69+
print("=" * 50)
70+
71+
print_metric_summary("Sandbox Creation Time", sandbox_creation_times)
72+
print_metric_summary("Health Check Time", health_check_times)
73+
print_metric_summary("First Code Run Time", first_code_run_times)
74+
print_metric_summary("Second Code Run Time", second_code_run_times)
75+
76+
77+
def create_performance_plot(
78+
template,
79+
iterations_count,
80+
sandbox_creation_times,
81+
health_check_times,
82+
first_code_run_times,
83+
second_code_run_times,
84+
):
85+
"""Create and save a performance visualization plot."""
86+
print(f"\nGenerating performance plot...")
87+
fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(12, 10))
88+
89+
# Plot 1: All metrics over iterations
90+
iterations = list(range(1, iterations_count + 1))
91+
ax1.plot(
92+
iterations,
93+
sandbox_creation_times,
94+
"b-o",
95+
label="Sandbox Creation",
96+
linewidth=1.5,
97+
markersize=6,
98+
markerfacecolor="blue",
99+
markeredgecolor="darkblue",
100+
markeredgewidth=1,
101+
)
102+
ax1.plot(
103+
iterations,
104+
health_check_times,
105+
"g-s",
106+
label="Health Check",
107+
linewidth=1.5,
108+
markersize=6,
109+
markerfacecolor="green",
110+
markeredgecolor="darkgreen",
111+
markeredgewidth=1,
112+
)
113+
ax1.plot(
114+
iterations,
115+
first_code_run_times,
116+
"r-^",
117+
label="First Code Run",
118+
linewidth=1.5,
119+
markersize=6,
120+
markerfacecolor="red",
121+
markeredgecolor="darkred",
122+
markeredgewidth=1,
123+
)
124+
ax1.plot(
125+
iterations,
126+
second_code_run_times,
127+
"m-d",
128+
label="Second Code Run",
129+
linewidth=1.5,
130+
markersize=6,
131+
markerfacecolor="magenta",
132+
markeredgecolor="darkmagenta",
133+
markeredgewidth=1,
134+
)
135+
136+
ax1.set_xlabel("Iteration")
137+
ax1.set_ylabel("Time (ms)")
138+
ax1.set_title(
139+
f"Performance Metrics Over {iterations_count} Iterations - {template}"
140+
)
141+
ax1.legend()
142+
ax1.grid(True, alpha=0.3)
143+
144+
# Set x-axis to show each iteration step
145+
ax1.set_xticks(iterations)
146+
ax1.set_xlim(0.5, iterations_count + 0.5)
147+
148+
# Plot 2: Box plot for distribution
149+
all_metrics = [
150+
sandbox_creation_times,
151+
health_check_times,
152+
first_code_run_times,
153+
second_code_run_times,
154+
]
155+
metric_names = [
156+
"Sandbox\nCreation",
157+
"Health\nCheck",
158+
"First Code\nRun",
159+
"Second Code\nRun",
160+
]
161+
162+
box_plot = ax2.boxplot(all_metrics, labels=metric_names, patch_artist=True)
163+
colors = ["lightblue", "lightgreen", "lightcoral", "plum"]
164+
for patch, color in zip(box_plot["boxes"], colors):
165+
patch.set_facecolor(color)
166+
167+
ax2.set_ylabel("Time (ms)")
168+
ax2.set_title(f"Performance Distribution - {template}")
169+
ax2.grid(True, alpha=0.3)
170+
171+
plt.tight_layout()
172+
173+
# Show summary statistics in the plot
174+
stats_text = f"""Summary Statistics:
175+
Sandbox Creation: {statistics.mean(sandbox_creation_times):.1f}ms avg
176+
Health Check: {statistics.mean(health_check_times):.1f}ms avg
177+
First Code Run: {statistics.mean(first_code_run_times):.1f}ms avg
178+
Second Code Run: {statistics.mean(second_code_run_times):.1f}ms avg"""
179+
180+
fig.text(
181+
0.02,
182+
0.02,
183+
stats_text,
184+
fontsize=8,
185+
verticalalignment="bottom",
186+
bbox=dict(boxstyle="round", facecolor="wheat", alpha=0.8),
187+
)
188+
189+
# Save the plot
190+
plot_filename = f"performance_plot.png"
191+
plt.savefig(plot_filename, dpi=300, bbox_inches="tight")
192+
print(f"Performance plot saved as: {plot_filename}")
193+
194+
return plot_filename
195+
196+
197+
# Create performance plot
198+
create_performance_plot(
199+
template,
200+
iterations_count,
201+
sandbox_creation_times,
202+
health_check_times,
203+
first_code_run_times,
204+
second_code_run_times,
205+
)

0 commit comments

Comments
 (0)