Skip to content

Commit d4886a7

Browse files
coderbirjucoderbirju
committed
feat: Add more metrics for benchmarks
Signed-off-by: Arjun Raja Yogidas <[email protected]>
1 parent 55a9ab1 commit d4886a7

File tree

5 files changed

+216
-22
lines changed

5 files changed

+216
-22
lines changed

‎Makefile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -283,7 +283,7 @@ test-benchmark-vm:
283283

284284
.PHONY: test-benchmark-container
285285
test-benchmark-container:
286-
cd benchmark/container && go test -ldflags $(LDFLAGS) -bench=. -benchmem --installed="$(INSTALLED)"
286+
cd benchmark/container && go test -ldflags $(LDFLAGS) -bench=. -benchmem -benchtime=1x -timeout 2h --installed="$(INSTALLED)"
287287

288288
.PHONY: gen-code
289289
# Since different projects may have different versions of tool binaries,

‎benchmark/all/all_test.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,11 +31,11 @@ func BenchmarkAll(b *testing.B) {
3131
}
3232

3333
b.Run("BenchmarkContainerRun", func(b *testing.B) {
34-
suite.BenchmarkContainerRun(b)
34+
suite.BenchmarkContainerRun(b, "finch")
3535
})
3636

3737
b.Run("BenchmarkImageBuild", func(b *testing.B) {
38-
suite.BenchmarkImageBuild(b)
38+
suite.BenchmarkImageBuild(b, "finch")
3939
})
4040

4141
err = suite.StopVM()

‎benchmark/benchmark.go

Lines changed: 28 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ package benchmark
77
import (
88
"flag"
99
"fmt"
10-
"os"
10+
"os/exec"
1111
"path/filepath"
1212
"sync"
1313
"testing"
@@ -45,18 +45,40 @@ func (m *Metrics) Add(other Metrics) {
4545

4646
// GetSubject returns the testing subject based on INSTALLED flag.
4747
func GetSubject() (string, error) {
48-
wd, err := os.Getwd()
49-
if err != nil {
50-
return "", fmt.Errorf("failed to get the current working directory: %w", err)
51-
}
48+
// wd, err := os.Getwd()
49+
// if err != nil {
50+
// return "", fmt.Errorf("failed to get the current working directory: %w", err)
51+
// }
5252

53-
subject := filepath.Join(wd, "../../_output/bin/finch")
53+
subject := filepath.Join("/usr/bin/", "finch")
5454
if *Installed {
5555
subject = InstalledTestSubject
5656
}
5757
return subject, nil
5858
}
5959

60+
func GetDocker() (string, error) {
61+
subject := filepath.Join("/usr/bin/", "docker")
62+
return subject, nil
63+
}
64+
65+
func CleanUpFunc() error {
66+
cmd1 := exec.Command("sudo", "rm", "-rf", "/var/lib/docker")
67+
if err := cmd1.Run(); err != nil {
68+
return fmt.Errorf("docker cleanup failed: %v", err)
69+
}
70+
cmd2 := exec.Command("sudo", "rm", "-rf", "/var/run/docker.pid")
71+
if err := cmd2.Run(); err != nil {
72+
return fmt.Errorf("failed to stop docker process: %v", err)
73+
}
74+
cmd3 := exec.Command("sudo", "systemctl", "restart", "docker")
75+
if err := cmd3.Run(); err != nil {
76+
return fmt.Errorf("docker failed to restart: %v", err)
77+
}
78+
return nil
79+
80+
}
81+
6082
// Wrapper reports the benchmarking metrics of targetFunc to testing.B.
6183
func Wrapper(b *testing.B, targetFunc func(), cleanupFunc func()) {
6284
metricsSum := Metrics{}

‎benchmark/container/container_test.go

Lines changed: 28 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,11 +17,35 @@ func BenchmarkContainer(b *testing.B) {
1717
b.Fatal(err)
1818
}
1919

20-
b.Run("BenchmarkContainerRun", func(b *testing.B) {
21-
suite.BenchmarkContainerRun(b)
20+
b.Run("BenchmarkContainerPull-docker", func(b *testing.B) {
21+
suite.BenchmarkContainerPull(b, "docker")
2222
})
2323

24-
b.Run("BenchmarkImageBuild", func(b *testing.B) {
25-
suite.BenchmarkImageBuild(b)
24+
b.Run("BenchmarkContainerRun-docker", func(b *testing.B) {
25+
suite.BenchmarkContainerRun(b, "docker")
2626
})
27+
b.Run("BenchmarkImageBuild-docker", func(b *testing.B) {
28+
suite.BenchmarkImageBuild(b, "docker")
29+
})
30+
31+
b.Run("BenchmarkImageDelete-docker", func(b *testing.B) {
32+
suite.BenchmarkImageDelete(b, "docker")
33+
})
34+
35+
b.Run("BenchmarkContainerRun-finch", func(b *testing.B) {
36+
suite.BenchmarkContainerRun(b, "finch")
37+
})
38+
39+
b.Run("BenchmarkContainerPull-finch", func(b *testing.B) {
40+
suite.BenchmarkContainerPull(b, "finch")
41+
})
42+
43+
b.Run("BenchmarkImageBuild-finch", func(b *testing.B) {
44+
suite.BenchmarkImageBuild(b, "finch")
45+
})
46+
47+
b.Run("BenchmarkImageDelete-finch", func(b *testing.B) {
48+
suite.BenchmarkImageDelete(b, "finch")
49+
})
50+
2751
}

‎benchmark/suite.go

Lines changed: 157 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -16,14 +16,16 @@ import (
1616

1717
const (
1818
virtualMachineRootCmd = "vm"
19-
alpineImage = "public.ecr.aws/docker/library/alpine:latest"
19+
alpineImage = "public.ecr.aws/soci-workshop-examples/node:latest"
2020
testImageName = "test:tag"
2121
testContainerName = "ctr-test"
2222
)
2323

24+
// public.ecr.aws/soci-workshop-examples/mongo:latest public.ecr.aws/soci-workshop-examples/redis:latest public.ecr.aws/docker/library/alpine:latest
2425
// Suite is a struct that groups benchmark functions and shared state.
2526
type Suite struct {
2627
subject string
28+
docker string
2729
}
2830

2931
// Setup initializes the Suite by getting the subject.
@@ -32,7 +34,13 @@ func (suite *Suite) Setup() error {
3234
if err != nil {
3335
return err
3436
}
37+
38+
docker, err := GetDocker()
39+
if err != nil {
40+
return err
41+
}
3542
suite.subject = subject
43+
suite.docker = docker
3644
return nil
3745
}
3846

@@ -80,18 +88,128 @@ func (suite *Suite) BenchmarkVMStart(b *testing.B) {
8088
}
8189

8290
// BenchmarkContainerRun measures the metrics to run a container.
83-
func (suite *Suite) BenchmarkContainerRun(b *testing.B) {
84-
assert.NoError(b, exec.Command(suite.subject, "pull", alpineImage).Run()) //nolint:gosec // testing only
91+
func (suite *Suite) BenchmarkContainerRun(b *testing.B, binaryName string) {
92+
if binaryName == "finch" {
93+
assert.NoError(b, exec.Command("sudo", suite.subject, "pull", alpineImage).Run()) //nolint:gosec // testing only
94+
Wrapper(b, func() {
95+
assert.NoError(b, exec.Command("sudo", suite.subject, "run", "--name", testContainerName, alpineImage).Run()) //nolint:gosec // testing only
96+
}, func() {
97+
assert.NoError(b, exec.Command("sudo", suite.subject, "rm", "--force", testContainerName).Run()) //nolint:gosec // testing only
98+
})
99+
assert.NoError(b, exec.Command("sudo", suite.subject, "rmi", "--force", alpineImage).Run()) //nolint:gosec // testing only
100+
} else {
101+
assert.NoError(b, CleanUpFunc())
102+
assert.NoError(b, exec.Command("sudo", suite.docker, "pull", alpineImage).Run()) //nolint:gosec // testing only
103+
Wrapper(b, func() {
104+
assert.NoError(b, exec.Command("sudo", suite.docker, "run", "--name", testContainerName, alpineImage).Run()) //nolint:gosec // testing only
105+
}, func() {
106+
assert.NoError(b, exec.Command("sudo", suite.docker, "rm", "--force", testContainerName).Run()) //nolint:gosec // testing only
107+
})
108+
assert.NoError(b, exec.Command("sudo", suite.docker, "rmi", "--force", alpineImage).Run()) //nolint:gosec // testing only
109+
}
110+
111+
}
112+
113+
func (suite *Suite) BenchmarkContainerRunDocker(b *testing.B) {
114+
assert.NoError(b, exec.Command("sudo", suite.docker, "pull", alpineImage).Run()) //nolint:gosec // testing only
85115
Wrapper(b, func() {
86-
assert.NoError(b, exec.Command(suite.subject, "run", "--name", testContainerName, alpineImage).Run()) //nolint:gosec // testing only
116+
assert.NoError(b, exec.Command("sudo", suite.docker, "run", "--name", testContainerName, alpineImage).Run()) //nolint:gosec // testing only
87117
}, func() {
88-
assert.NoError(b, exec.Command(suite.subject, "rm", "--force", testContainerName).Run()) //nolint:gosec // testing only
118+
assert.NoError(b, exec.Command("sudo", suite.docker, "rm", "--force", testContainerName).Run()) //nolint:gosec // testing only
89119
})
90-
assert.NoError(b, exec.Command(suite.subject, "rmi", "--force", alpineImage).Run()) //nolint:gosec // testing only
120+
assert.NoError(b, exec.Command("sudo", suite.docker, "rmi", "--force", alpineImage).Run()) //nolint:gosec // testing only
121+
}
122+
123+
func (suite *Suite) BenchmarkContainerPull(b *testing.B, binaryName string) {
124+
if binaryName == "finch" {
125+
Wrapper(b, func() {
126+
assert.NoError(b, exec.Command("sudo", suite.subject, "pull", alpineImage).Run()) //nolint:gosec // testing only
127+
}, func() {
128+
assert.NoError(b, exec.Command("sudo", suite.subject, "rmi", "--force", alpineImage).Run()) //nolint:gosec // testing only
129+
})
130+
} else {
131+
assert.NoError(b, CleanUpFunc())
132+
Wrapper(b, func() {
133+
assert.NoError(b, exec.Command("sudo", suite.docker, "pull", alpineImage).Run()) //nolint:gosec // testing only
134+
}, func() {
135+
assert.NoError(b, exec.Command("sudo", suite.docker, "rmi", "--force", alpineImage).Run()) //nolint:gosec // testing only
136+
})
137+
}
91138
}
92139

140+
// func (suite *Suite) BenchmarkContainerPullDocker(b *testing.B) {
141+
// assert.NoError(b, CleanUpFunc())
142+
// Wrapper(b, func() {
143+
// assert.NoError(b, exec.Command("sudo", suite.docker, "pull", alpineImage).Run()) //nolint:gosec // testing only
144+
// }, func() {
145+
// assert.NoError(b, exec.Command("sudo", suite.docker, "rmi", "--force", alpineImage).Run()) //nolint:gosec // testing only
146+
// })
147+
// }
148+
149+
// func (suite *Suite) BenchmarkContainerPullFinch(b *testing.B) {
150+
// Wrapper(b, func() {
151+
// assert.NoError(b, exec.Command("sudo", suite.subject, "pull", alpineImage).Run()) //nolint:gosec // testing only
152+
// }, func() {
153+
// assert.NoError(b, exec.Command("sudo", suite.subject, "rmi", "--force", alpineImage).Run()) //nolint:gosec // testing only
154+
// })
155+
// }
156+
93157
// BenchmarkImageBuild measures the metrics to build an image.
94-
func (suite *Suite) BenchmarkImageBuild(b *testing.B) {
158+
func (suite *Suite) BenchmarkImageBuild(b *testing.B, binaryName string) {
159+
homeDir, err := os.UserHomeDir()
160+
assert.NoError(b, err)
161+
tempDir, err := os.MkdirTemp(homeDir, "finch-test")
162+
assert.NoError(b, err)
163+
dockerFilePath := filepath.Join(tempDir, "Dockerfile")
164+
err = os.WriteFile(dockerFilePath, []byte(fmt.Sprintf(`FROM %s
165+
CMD ["echo", "finch-test-dummy-output"]
166+
`, alpineImage)), 0o600)
167+
assert.NoError(b, err)
168+
buildContext := filepath.Dir(dockerFilePath)
169+
defer os.RemoveAll(buildContext) //nolint:errcheck // testing only
170+
if binaryName == "finch" {
171+
assert.NoError(b, exec.Command("sudo", suite.subject, "builder", "prune").Run())
172+
Wrapper(b, func() {
173+
assert.NoError(b, exec.Command("sudo", suite.subject, "build", "--tag", testImageName, buildContext, "--no-cache").Run()) //nolint:gosec // testing only
174+
}, func() {
175+
assert.NoError(b, exec.Command("sudo", suite.subject, "rmi", "--force", testImageName).Run()) //nolint:gosec // testing only
176+
})
177+
} else {
178+
assert.NoError(b, CleanUpFunc())
179+
Wrapper(b, func() {
180+
// assert.NoError(b, exec.Command("sudo","DOCKER_BUILDKIT=0", suite.docker, "buildx", "build", "--tag", testImageName, buildContext, "--no-cache", "--builder", "default" "--pull", "--build-arg", "CACHEBUST=$(date +%s)").Run()) //nolint:gosec // testing only
181+
assert.NoError(b, exec.Command("sudo", "DOCKER_BUILDKIT=0", suite.docker, "buildx", "build", "--tag", testImageName, buildContext, "--no-cache").Run()) //nolint:gosec // testing only
182+
}, func() {
183+
assert.NoError(b, exec.Command("sudo", suite.docker, "rmi", "--force", testImageName).Run()) //nolint:gosec // testing only
184+
})
185+
}
186+
}
187+
188+
func (suite *Suite) BenchmarkImageBuildDocker(b *testing.B) {
189+
homeDir, err := os.UserHomeDir()
190+
assert.NoError(b, err)
191+
tempDir, err := os.MkdirTemp(homeDir, "finch-test")
192+
assert.NoError(b, err)
193+
dockerFilePath := filepath.Join(tempDir, "Dockerfile")
194+
err = os.WriteFile(dockerFilePath, []byte(fmt.Sprintf(`FROM %s
195+
CMD ["echo", "finch-test-dummy-output"]
196+
`, alpineImage)), 0o600)
197+
assert.NoError(b, err)
198+
buildContext := filepath.Dir(dockerFilePath)
199+
defer os.RemoveAll(buildContext) //nolint:errcheck // testing only
200+
assert.NoError(b, exec.Command("sudo", "rm", "-rf", "/var/lib/docker").Run())
201+
assert.NoError(b, exec.Command("sudo", "rm", "-rf", "/var/run/docker.pid").Run())
202+
assert.NoError(b, exec.Command("sudo", "systemctl", "restart", "docker").Run())
203+
// assert.NoError(b, exec.Command("sudo", suite.docker, "system", "prune", "-f", "--all", "--volumes").Run())
204+
Wrapper(b, func() {
205+
// assert.NoError(b, exec.Command("sudo","DOCKER_BUILDKIT=0", suite.docker, "buildx", "build", "--tag", testImageName, buildContext, "--no-cache", "--builder", "default" "--pull", "--build-arg", "CACHEBUST=$(date +%s)").Run()) //nolint:gosec // testing only
206+
assert.NoError(b, exec.Command("sudo", "DOCKER_BUILDKIT=0", suite.docker, "buildx", "build", "--tag", testImageName, buildContext, "--no-cache").Run()) //nolint:gosec // testing only
207+
}, func() {
208+
assert.NoError(b, exec.Command("sudo", suite.docker, "rmi", "--force", testImageName).Run()) //nolint:gosec // testing only
209+
})
210+
}
211+
212+
func (suite *Suite) BenchmarkImageDelete(b *testing.B, binaryImage string) {
95213
homeDir, err := os.UserHomeDir()
96214
assert.NoError(b, err)
97215
tempDir, err := os.MkdirTemp(homeDir, "finch-test")
@@ -103,9 +221,39 @@ func (suite *Suite) BenchmarkImageBuild(b *testing.B) {
103221
assert.NoError(b, err)
104222
buildContext := filepath.Dir(dockerFilePath)
105223
defer os.RemoveAll(buildContext) //nolint:errcheck // testing only
224+
if binaryImage == "finch" {
225+
assert.NoError(b, exec.Command("sudo", suite.subject, "build", "--tag", testImageName, buildContext).Run()) //nolint:gosec // testing only
226+
Wrapper(b, func() {
227+
assert.NoError(b, exec.Command("sudo", suite.subject, "rmi", "--force", testImageName).Run()) //nolint:gosec // testing only
228+
}, func() {
229+
assert.NoError(b, exec.Command("sudo", suite.subject, "rmi", "--help").Run())
230+
})
231+
} else {
232+
assert.NoError(b, exec.Command("sudo", suite.docker, "build", "--tag", testImageName, buildContext).Run()) //nolint:gosec // testing only
233+
Wrapper(b, func() {
234+
assert.NoError(b, exec.Command("sudo", suite.docker, "rmi", "--force", testImageName).Run()) //nolint:gosec // testing only
235+
}, func() {
236+
assert.NoError(b, exec.Command("sudo", suite.docker, "rmi", "--help").Run())
237+
})
238+
}
239+
}
240+
241+
func (suite *Suite) BenchmarkImageDeleteDocker(b *testing.B) {
242+
homeDir, err := os.UserHomeDir()
243+
assert.NoError(b, err)
244+
tempDir, err := os.MkdirTemp(homeDir, "finch-test")
245+
assert.NoError(b, err)
246+
dockerFilePath := filepath.Join(tempDir, "Dockerfile")
247+
err = os.WriteFile(dockerFilePath, []byte(fmt.Sprintf(`FROM %s
248+
CMD ["echo", "finch-test-dummy-output"]
249+
`, alpineImage)), 0o600)
250+
assert.NoError(b, err)
251+
buildContext := filepath.Dir(dockerFilePath)
252+
defer os.RemoveAll(buildContext) //nolint:errcheck // testing only
253+
assert.NoError(b, exec.Command("sudo", suite.docker, "build", "--tag", testImageName, buildContext).Run()) //nolint:gosec // testing only
106254
Wrapper(b, func() {
107-
assert.NoError(b, exec.Command(suite.subject, "build", "--tag", testImageName, buildContext).Run()) //nolint:gosec // testing only
255+
assert.NoError(b, exec.Command("sudo", suite.docker, "rmi", "--force", testImageName).Run()) //nolint:gosec // testing only
108256
}, func() {
109-
assert.NoError(b, exec.Command(suite.subject, "rmi", "--force", testImageName).Run()) //nolint:gosec // testing only
257+
assert.NoError(b, exec.Command("sudo", suite.docker, "rmi", "--help").Run())
110258
})
111259
}

0 commit comments

Comments
 (0)