Skip to content

Commit f64db97

Browse files
zimbatmzimbatm
committed
add benchmark notes
fixes #93
1 parent 5419dc1 commit f64db97

File tree

6 files changed

+257
-0
lines changed

6 files changed

+257
-0
lines changed

‎benchmark/README.md‎

Lines changed: 238 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,238 @@
1+
# Benchmarking devshell evaluation
2+
3+
devshell is built on top of nix, and the nixpkgs module system, which can take
4+
quite a while to evaluate.
5+
6+
## Hyperfine
7+
8+
`$ hyperfine -w 3 'nix-instantiate ../shell.nix' 'nix-instantiate ./devshell-nix.nix' 'nix-instantiate ./devshell-toml.nix' 'nix-instantiate ./nixpkgs-mkshell.nix'`
9+
```
10+
Benchmark #1: nix-instantiate ../shell.nix
11+
Time (mean ± σ): 1.082 s ± 0.011 s [User: 732.6 ms, System: 154.7 ms]
12+
Range (min … max): 1.065 s … 1.099 s 10 runs
13+
14+
Benchmark #2: nix-instantiate ./devshell-nix.nix
15+
Time (mean ± σ): 412.1 ms ± 3.3 ms [User: 300.0 ms, System: 63.8 ms]
16+
Range (min … max): 406.8 ms … 417.4 ms 10 runs
17+
18+
Benchmark #3: nix-instantiate ./devshell-toml.nix
19+
Time (mean ± σ): 411.6 ms ± 6.6 ms [User: 299.7 ms, System: 64.7 ms]
20+
Range (min … max): 403.6 ms … 420.5 ms 10 runs
21+
22+
Benchmark #4: nix-instantiate ./nixpkgs-mkshell.nix
23+
Time (mean ± σ): 359.7 ms ± 9.1 ms [User: 269.2 ms, System: 52.6 ms]
24+
Range (min … max): 349.8 ms … 379.9 ms 10 runs
25+
26+
Summary
27+
'nix-instantiate ./nixpkgs-mkshell.nix' ran
28+
1.14 ± 0.03 times faster than 'nix-instantiate ./devshell-toml.nix'
29+
1.15 ± 0.03 times faster than 'nix-instantiate ./devshell-nix.nix'
30+
3.01 ± 0.08 times faster than 'nix-instantiate ../shell.nix'
31+
```
32+
33+
## Nix stats
34+
35+
### repo shell
36+
37+
`$ NIX_SHOW_STATS=1 nix-instantiate ./../shell.nix 2>&1`
38+
```
39+
warning: you did not specify '--add-root'; the result might be removed by the garbage collector
40+
/nix/store/xbadj2p1yva55gcm6n6ml028wdgbap5f-devshell.drv
41+
{
42+
"cpuTime": 0.742534,
43+
"envs": {
44+
"number": 189808,
45+
"elements": 321331,
46+
"bytes": 5607576
47+
},
48+
"list": {
49+
"elements": 119124,
50+
"bytes": 952992,
51+
"concats": 13943
52+
},
53+
"values": {
54+
"number": 607122,
55+
"bytes": 14570928
56+
},
57+
"symbols": {
58+
"number": 37696,
59+
"bytes": 917618
60+
},
61+
"sets": {
62+
"number": 60629,
63+
"bytes": 42826168,
64+
"elements": 1764214
65+
},
66+
"sizes": {
67+
"Env": 16,
68+
"Value": 24,
69+
"Bindings": 8,
70+
"Attr": 24
71+
},
72+
"nrOpUpdates": 29262,
73+
"nrOpUpdateValuesCopied": 1463167,
74+
"nrThunks": 446497,
75+
"nrAvoided": 320262,
76+
"nrLookups": 175553,
77+
"nrPrimOpCalls": 139902,
78+
"nrFunctionCalls": 167314,
79+
"gc": {
80+
"heapSize": 402915328,
81+
"totalBytes": 75203696
82+
}
83+
}
84+
```
85+
86+
87+
### devshell-nix
88+
89+
`$ NIX_SHOW_STATS=1 nix-instantiate ./devshell-nix.nix 2>&1`
90+
```
91+
warning: you did not specify '--add-root'; the result might be removed by the garbage collector
92+
/nix/store/r304514lvygyzh8l75cgjhkhfqs15d1m-devshell.drv
93+
{
94+
"cpuTime": 0.364713,
95+
"envs": {
96+
"number": 74392,
97+
"elements": 105623,
98+
"bytes": 2035256
99+
},
100+
"list": {
101+
"elements": 40998,
102+
"bytes": 327984,
103+
"concats": 2342
104+
},
105+
"values": {
106+
"number": 243435,
107+
"bytes": 5842440
108+
},
109+
"symbols": {
110+
"number": 29700,
111+
"bytes": 662792
112+
},
113+
"sets": {
114+
"number": 18563,
115+
"bytes": 23664496,
116+
"elements": 979833
117+
},
118+
"sizes": {
119+
"Env": 16,
120+
"Value": 24,
121+
"Bindings": 8,
122+
"Attr": 24
123+
},
124+
"nrOpUpdates": 5563,
125+
"nrOpUpdateValuesCopied": 837612,
126+
"nrThunks": 186590,
127+
"nrAvoided": 109995,
128+
"nrLookups": 34126,
129+
"nrPrimOpCalls": 54602,
130+
"nrFunctionCalls": 65308,
131+
"gc": {
132+
"heapSize": 402915328,
133+
"totalBytes": 35643456
134+
}
135+
}
136+
```
137+
138+
### devshell-toml
139+
140+
`$ NIX_SHOW_STATS=1 nix-instantiate ./devshell-toml.nix 2>&1`
141+
```
142+
warning: you did not specify '--add-root'; the result might be removed by the garbage collector
143+
/nix/store/r304514lvygyzh8l75cgjhkhfqs15d1m-devshell.drv
144+
{
145+
"cpuTime": 0.291564,
146+
"envs": {
147+
"number": 74405,
148+
"elements": 105638,
149+
"bytes": 2035584
150+
},
151+
"list": {
152+
"elements": 41008,
153+
"bytes": 328064,
154+
"concats": 2342
155+
},
156+
"values": {
157+
"number": 243463,
158+
"bytes": 5843112
159+
},
160+
"symbols": {
161+
"number": 29700,
162+
"bytes": 662793
163+
},
164+
"sets": {
165+
"number": 18572,
166+
"bytes": 23665144,
167+
"elements": 979857
168+
},
169+
"sizes": {
170+
"Env": 16,
171+
"Value": 24,
172+
"Bindings": 8,
173+
"Attr": 24
174+
},
175+
"nrOpUpdates": 5564,
176+
"nrOpUpdateValuesCopied": 837622,
177+
"nrThunks": 186605,
178+
"nrAvoided": 110009,
179+
"nrLookups": 34136,
180+
"nrPrimOpCalls": 54608,
181+
"nrFunctionCalls": 65318,
182+
"gc": {
183+
"heapSize": 402915328,
184+
"totalBytes": 35643456
185+
}
186+
}
187+
```
188+
189+
### nixpkgs-mkshell
190+
191+
`$ NIX_SHOW_STATS=1 nix-instantiate ./nixpkgs-mkshell.nix 2>&1`
192+
```
193+
warning: you did not specify '--add-root'; the result might be removed by the garbage collector
194+
/nix/store/d0is0v6f86dsj2sjrdl0bszq0w0fhpn8-nix-shell.drv
195+
{
196+
"cpuTime": 0.271529,
197+
"envs": {
198+
"number": 57978,
199+
"elements": 78401,
200+
"bytes": 1554856
201+
},
202+
"list": {
203+
"elements": 33856,
204+
"bytes": 270848,
205+
"concats": 1192
206+
},
207+
"values": {
208+
"number": 205224,
209+
"bytes": 4925376
210+
},
211+
"symbols": {
212+
"number": 29169,
213+
"bytes": 641516
214+
},
215+
"sets": {
216+
"number": 13313,
217+
"bytes": 22291168,
218+
"elements": 924361
219+
},
220+
"sizes": {
221+
"Env": 16,
222+
"Value": 24,
223+
"Bindings": 8,
224+
"Attr": 24
225+
},
226+
"nrOpUpdates": 3373,
227+
"nrOpUpdateValuesCopied": 797699,
228+
"nrThunks": 159998,
229+
"nrAvoided": 86376,
230+
"nrLookups": 19189,
231+
"nrPrimOpCalls": 46043,
232+
"nrFunctionCalls": 50785,
233+
"gc": {
234+
"heapSize": 402915328,
235+
"totalBytes": 32125552
236+
}
237+
}
238+
```

‎benchmark/devshell-nix.nix‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
{ system ? builtins.currentSystem }:
2+
let
3+
devshell = import ../. { inherit system; };
4+
in
5+
devshell.mkShell { }

‎benchmark/devshell-toml.nix‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
{ system ? builtins.currentSystem }:
2+
let
3+
devshell = import ../. { inherit system; };
4+
in
5+
devshell.fromTOML ./devshell-toml.toml

‎benchmark/devshell-toml.toml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
# Empty TOML
2+
[devshell]

‎benchmark/nixpkgs-mkshell.nix‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
{ system ? builtins.currentSystem }:
2+
let
3+
pkgs = import (import ../nix/nixpkgs.nix) { inherit system; };
4+
in
5+
pkgs.mkShell { }

‎devshell.toml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,9 @@ packages = [
2020
"diffutils", # used by golangci-lint
2121
"goreleaser",
2222
"mdbook",
23+
"mdsh",
2324
"webfs",
25+
"hyperfine",
2426
]
2527

2628
# Declare commands that are available in the environment.

0 commit comments

Comments
 (0)