Skip to content

Commit d9845c6

Browse files
coderbirjucoderbirju
authored
Merge pull request #5 from coderbirju/temp-finch-v-docker-benchmarking
feat: Add more metrics for benchmarks
2 parents 25133cf + 420e6aa commit d9845c6

File tree

6 files changed

+242
-22
lines changed

6 files changed

+242
-22
lines changed

‎.github/workflows/benchmark-comparision.yaml‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
# - When a third-party action is added (i.e., `uses`), please also add it to `download-licenses` in Makefile.
2+
# - When a job is added/removed/renamed, please make corresponding changes in ci-docs.yaml.
3+
name: Benchmark
4+
on:
5+
push:
6+
branches:
7+
- main
8+
paths-ignore:
9+
- '**.md'
10+
- 'contrib/**'
11+
12+
permissions:
13+
# deployments permission to deploy GitHub pages website
14+
deployments: write
15+
# contents permission to update benchmark contents in gh-pages branch
16+
contents: write
17+
18+
jobs:
19+
benchmark:
20+
strategy:
21+
fail-fast: false
22+
matrix:
23+
os:
24+
[ ubuntu-latest ]
25+
runs-on: ${{ matrix.os }}
26+
steps:
27+
- uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
28+
with:
29+
# We need to get all the git tags to make version injection work. See VERSION in Makefile for more detail.
30+
fetch-depth: 0
31+
submodules: true
32+
- name: Clean up previous files
33+
run: |
34+
sudo rm -rf /opt/finch
35+
sudo rm -rf ~/.finch
36+
sudo rm -rf ./_output
37+
if pgrep '^qemu-system'; then
38+
sudo pkill '^qemu-system'
39+
fi
40+
if pgrep '^socket_vmnet'; then
41+
sudo pkill '^socket_vmnet'
42+
fi
43+
- name: Install Dependencies
44+
run: sudo apt-get install -y golang lz4 automake autoconf libtool yq
45+
- name: Build project
46+
run: |
47+
make
48+
- name: Run benchmark
49+
run: make test-benchmark-container | tee benchmark-container.txt
50+
- name: Set OS info as env variable
51+
run: |
52+
echo "OS_VERSION=$(lsb_release -sr)" >> $GITHUB_ENV
53+
echo "ARCH=$(uname -m)" >> $GITHUB_ENV
54+
- name: Store benchmark result
55+
uses: benchmark-action/github-action-benchmark@4de1bed97a47495fc4c5404952da0499e31f5c29 # v1.20.3
56+
with:
57+
name: Finch Benchmark
58+
tool: 'go'
59+
benchmark-data-dir-path: "dev/bench/comparison/${{ env.OS_VERSION }}/${{ env.ARCH }}"
60+
output-file-path: benchmark-container.txt
61+
- name: Push benchmark result
62+
run: git push 'https://github.com/coderbirju/finch.git' gh-pages:gh-pages

‎Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -284,7 +284,7 @@ test-benchmark-vm:
284284

285285
.PHONY: test-benchmark-container
286286
test-benchmark-container:
287-
cd benchmark/container && go test -ldflags $(LDFLAGS) -bench=. -benchmem --installed="$(INSTALLED)"
287+
cd benchmark/container && go test -ldflags $(LDFLAGS) -bench=. -benchmem -benchtime=1x -timeout 2h --installed="$(INSTALLED)"
288288

289289
.PHONY: gen-code
290290
# Since different projects may have different versions of tool binaries,

‎benchmark/all/all_test.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,11 +31,11 @@ func BenchmarkAll(b *testing.B) {
3131
}
3232

3333
b.Run("BenchmarkContainerRun", func(b *testing.B) {
34-
suite.BenchmarkContainerRun(b)
34+
suite.BenchmarkContainerRun(b, "finch")
3535
})
3636

3737
b.Run("BenchmarkImageBuild", func(b *testing.B) {
38-
suite.BenchmarkImageBuild(b)
38+
suite.BenchmarkImageBuild(b, "finch")
3939
})
4040

4141
err = suite.StopVM()

‎benchmark/benchmark.go‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,58 @@ func GetSubject() (string, error) {
5757
return subject, nil
5858
}
5959

60+
func GetDocker() (string, error) {
61+
subject := filepath.Join("/usr/bin/", "docker")
62+
return subject, nil
63+
}
64+
65+
// func CleanUpFunc() error {
66+
// cmd0 := exec.Command("sudo", "systemctl", "stop", "docker")
67+
// if err := cmd0.Run(); err != nil {
68+
// return fmt.Errorf("docker stop failed: %v", err)
69+
// }
70+
// cmd1 := exec.Command("sudo", "rm", "-rf", "/var/lib/docker")
71+
// if err := cmd1.Run(); err != nil {
72+
// return fmt.Errorf("docker cleanup failed: %v", err)
73+
// }
74+
// cmd2 := exec.Command("sudo", "rm", "-rf", "/var/run/docker.pid")
75+
// if err := cmd2.Run(); err != nil {
76+
// return fmt.Errorf("failed to stop docker process: %v", err)
77+
// }
78+
// cmd3 := exec.Command("sudo", "systemctl", "restart", "docker")
79+
// if err := cmd3.Run(); err != nil {
80+
// return fmt.Errorf("docker failed to restart: %v", err)
81+
// }
82+
83+
// cmd4 := exec.Command("sudo", "rm", "-rf", "/var/lib/finch/buildkit/cache.db")
84+
// if err := cmd4.Run(); err != nil {
85+
// return fmt.Errorf("finch buildkit cleanup failed: %v", err)
86+
// }
87+
88+
// cmd5 := exec.Command("sudo", "docker", "builder", "prune", "-a", "-f")
89+
// if err := cmd5.Run(); err != nil {
90+
// return fmt.Errorf("docker builder prune failed: %v", err)
91+
// }
92+
93+
// cmd6 := exec.Command("sudo", "docker", "image", "prune", "-a", "-f")
94+
// if err := cmd6.Run(); err != nil {
95+
// return fmt.Errorf("docker image prune failed: %v", err)
96+
// }
97+
98+
// cmd7 := exec.Command("sudo", "rm", "-rf", "/var/lib/containerd")
99+
// if err := cmd7.Run(); err != nil {
100+
// return fmt.Errorf("containerd cleanup failed: %v", err)
101+
// }
102+
103+
// cmd8 := exec.Command("sudo", "systemctl", "restart", "containerd")
104+
// if err := cmd8.Run(); err != nil {
105+
// return fmt.Errorf("docker failed to restart: %v", err)
106+
// }
107+
108+
// return nil
109+
110+
// }
111+
60112
// Wrapper reports the benchmarking metrics of targetFunc to testing.B.
61113
func Wrapper(b *testing.B, targetFunc func(), cleanupFunc func()) {
62114
metricsSum := Metrics{}

‎benchmark/container/container_test.go‎

Lines changed: 29 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,11 +17,36 @@ func BenchmarkContainer(b *testing.B) {
1717
b.Fatal(err)
1818
}
1919

20-
b.Run("BenchmarkContainerRun", func(b *testing.B) {
21-
suite.BenchmarkContainerRun(b)
20+
b.Run("BenchmarkContainerRun-docker", func(b *testing.B) {
21+
suite.BenchmarkContainerRun(b, "docker")
2222
})
2323

24-
b.Run("BenchmarkImageBuild", func(b *testing.B) {
25-
suite.BenchmarkImageBuild(b)
24+
b.Run("BenchmarkContainerRun-finch", func(b *testing.B) {
25+
suite.BenchmarkContainerRun(b, "finch")
2626
})
27+
28+
b.Run("BenchmarkContainerPull-docker", func(b *testing.B) {
29+
suite.BenchmarkContainerPull(b, "docker")
30+
})
31+
32+
b.Run("BenchmarkContainerPull-finch", func(b *testing.B) {
33+
suite.BenchmarkContainerPull(b, "finch")
34+
})
35+
36+
b.Run("BenchmarkImageBuild-docker", func(b *testing.B) {
37+
suite.BenchmarkImageBuild(b, "docker")
38+
})
39+
40+
b.Run("BenchmarkImageBuild-finch", func(b *testing.B) {
41+
suite.BenchmarkImageBuild(b, "finch")
42+
})
43+
44+
b.Run("BenchmarkImageDelete-docker", func(b *testing.B) {
45+
suite.BenchmarkImageDelete(b, "docker")
46+
})
47+
48+
b.Run("BenchmarkImageDelete-finch", func(b *testing.B) {
49+
suite.BenchmarkImageDelete(b, "finch")
50+
})
51+
2752
}

‎benchmark/suite.go‎

Lines changed: 96 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -16,14 +16,17 @@ import (
1616

1717
const (
1818
virtualMachineRootCmd = "vm"
19-
alpineImage = "public.ecr.aws/docker/library/alpine:latest"
19+
alpineImage = "public.ecr.aws/y0o4y9o3/anaconda-pkg-build:latest"
2020
testImageName = "test:tag"
2121
testContainerName = "ctr-test"
22+
ligthImage = "public.ecr.aws/docker/library/amazonlinux:latest"
2223
)
2324

25+
// public.ecr.aws/soci-workshop-examples/mongo:latest public.ecr.aws/soci-workshop-examples/redis:latest public.ecr.aws/docker/library/alpine:latest
2426
// Suite is a struct that groups benchmark functions and shared state.
2527
type Suite struct {
2628
subject string
29+
docker string
2730
}
2831

2932
// Setup initializes the Suite by getting the subject.
@@ -32,7 +35,13 @@ func (suite *Suite) Setup() error {
3235
if err != nil {
3336
return err
3437
}
38+
39+
docker, err := GetDocker()
40+
if err != nil {
41+
return err
42+
}
3543
suite.subject = subject
44+
suite.docker = docker
3645
return nil
3746
}
3847

@@ -80,18 +89,49 @@ func (suite *Suite) BenchmarkVMStart(b *testing.B) {
8089
}
8190

8291
// BenchmarkContainerRun measures the metrics to run a container.
83-
func (suite *Suite) BenchmarkContainerRun(b *testing.B) {
84-
assert.NoError(b, exec.Command(suite.subject, "pull", alpineImage).Run()) //nolint:gosec // testing only
85-
Wrapper(b, func() {
86-
assert.NoError(b, exec.Command(suite.subject, "run", "--name", testContainerName, alpineImage).Run()) //nolint:gosec // testing only
87-
}, func() {
88-
assert.NoError(b, exec.Command(suite.subject, "rm", "--force", testContainerName).Run()) //nolint:gosec // testing only
89-
})
90-
assert.NoError(b, exec.Command(suite.subject, "rmi", "--force", alpineImage).Run()) //nolint:gosec // testing only
92+
func (suite *Suite) BenchmarkContainerRun(b *testing.B, binaryName string) {
93+
// assert.NoError(b, CleanUpFunc())
94+
if binaryName == "finch" {
95+
assert.NoError(b, exec.Command("sudo", suite.subject, "pull", alpineImage).Run()) //nolint:gosec // testing only
96+
Wrapper(b, func() {
97+
assert.NoError(b, exec.Command("sudo", suite.subject, "run", "--name", testContainerName, alpineImage).Run()) //nolint:gosec // testing only
98+
}, func() {
99+
assert.NoError(b, exec.Command("sudo", suite.subject, "rm", "--force", testContainerName).Run()) //nolint:gosec // testing only
100+
})
101+
assert.NoError(b, exec.Command("sudo", suite.subject, "rmi", "--force", alpineImage).Run()) //nolint:gosec // testing only
102+
} else {
103+
// assert.NoError(b, CleanUpFunc())
104+
assert.NoError(b, exec.Command("sudo", suite.docker, "pull", alpineImage).Run()) //nolint:gosec // testing only
105+
Wrapper(b, func() {
106+
assert.NoError(b, exec.Command("sudo", suite.docker, "run", "--name", testContainerName, alpineImage).Run()) //nolint:gosec // testing only
107+
}, func() {
108+
assert.NoError(b, exec.Command("sudo", suite.docker, "rm", "--force", testContainerName).Run()) //nolint:gosec // testing only
109+
})
110+
assert.NoError(b, exec.Command("sudo", suite.docker, "rmi", "--force", alpineImage).Run()) //nolint:gosec // testing only
111+
}
112+
113+
}
114+
115+
func (suite *Suite) BenchmarkContainerPull(b *testing.B, binaryName string) {
116+
if binaryName == "finch" {
117+
Wrapper(b, func() {
118+
assert.NoError(b, exec.Command("sudo", suite.subject, "pull", alpineImage, "--namespace=finch").Run()) //nolint:gosec // testing only
119+
}, func() {
120+
assert.NoError(b, exec.Command("sudo", suite.subject, "rmi", "--force", alpineImage).Run()) //nolint:gosec // testing only
121+
})
122+
} else {
123+
assert.NoError(b, exec.Command("sudo", suite.docker, "images", "prune", "-a").Run())
124+
assert.NoError(b, exec.Command("sudo", suite.docker, "volume", "prune", "-a").Run())
125+
Wrapper(b, func() {
126+
assert.NoError(b, exec.Command("sudo", suite.docker, "pull", alpineImage).Run()) //nolint:gosec // testing only
127+
}, func() {
128+
assert.NoError(b, exec.Command("sudo", suite.docker, "rmi", "--force", alpineImage).Run()) //nolint:gosec // testing only
129+
})
130+
}
91131
}
92132

93133
// BenchmarkImageBuild measures the metrics to build an image.
94-
func (suite *Suite) BenchmarkImageBuild(b *testing.B) {
134+
func (suite *Suite) BenchmarkImageBuild(b *testing.B, binaryName string) {
95135
homeDir, err := os.UserHomeDir()
96136
assert.NoError(b, err)
97137
tempDir, err := os.MkdirTemp(homeDir, "finch-test")
@@ -103,9 +143,50 @@ func (suite *Suite) BenchmarkImageBuild(b *testing.B) {
103143
assert.NoError(b, err)
104144
buildContext := filepath.Dir(dockerFilePath)
105145
defer os.RemoveAll(buildContext) //nolint:errcheck // testing only
106-
Wrapper(b, func() {
107-
assert.NoError(b, exec.Command(suite.subject, "build", "--tag", testImageName, buildContext).Run()) //nolint:gosec // testing only
108-
}, func() {
109-
assert.NoError(b, exec.Command(suite.subject, "rmi", "--force", testImageName).Run()) //nolint:gosec // testing only
110-
})
146+
// assert.NoError(b, CleanUpFunc())
147+
if binaryName == "finch" {
148+
assert.NoError(b, exec.Command("sudo", suite.subject, "builder", "prune").Run())
149+
Wrapper(b, func() {
150+
assert.NoError(b, exec.Command("sudo", suite.subject, "build", "--tag", testImageName, buildContext, "--namespace=finch").Run()) //nolint:gosec // testing only
151+
}, func() {
152+
assert.NoError(b, exec.Command("sudo", suite.subject, "rmi", "--force", testImageName).Run()) //nolint:gosec // testing only
153+
})
154+
} else {
155+
// assert.NoError(b, CleanUpFunc())
156+
assert.NoError(b, exec.Command("sudo", suite.docker, "builder", "prune").Run())
157+
Wrapper(b, func() {
158+
assert.NoError(b, exec.Command("sudo", suite.docker, "build", "--tag", testImageName, buildContext, "--no-cache").Run()) //nolint:gosec // testing only
159+
}, func() {
160+
assert.NoError(b, exec.Command("sudo", suite.docker, "rmi", "--force", testImageName).Run()) //nolint:gosec // testing only
161+
})
162+
}
163+
}
164+
165+
func (suite *Suite) BenchmarkImageDelete(b *testing.B, binaryImage string) {
166+
homeDir, err := os.UserHomeDir()
167+
assert.NoError(b, err)
168+
tempDir, err := os.MkdirTemp(homeDir, "finch-test")
169+
assert.NoError(b, err)
170+
dockerFilePath := filepath.Join(tempDir, "Dockerfile")
171+
err = os.WriteFile(dockerFilePath, []byte(fmt.Sprintf(`FROM %s
172+
CMD ["echo", "finch-test-dummy-output"]
173+
`, alpineImage)), 0o600)
174+
assert.NoError(b, err)
175+
buildContext := filepath.Dir(dockerFilePath)
176+
defer os.RemoveAll(buildContext) //nolint:errcheck // testing only
177+
if binaryImage == "finch" {
178+
assert.NoError(b, exec.Command("sudo", suite.subject, "build", "--tag", testImageName, buildContext).Run()) //nolint:gosec // testing only
179+
Wrapper(b, func() {
180+
assert.NoError(b, exec.Command("sudo", suite.subject, "rmi", "--force", testImageName).Run()) //nolint:gosec // testing only
181+
}, func() {
182+
assert.NoError(b, exec.Command("sudo", suite.subject, "rmi", "--help").Run())
183+
})
184+
} else {
185+
assert.NoError(b, exec.Command("sudo", suite.docker, "build", "--tag", testImageName, buildContext).Run()) //nolint:gosec // testing only
186+
Wrapper(b, func() {
187+
assert.NoError(b, exec.Command("sudo", suite.docker, "rmi", "--force", testImageName).Run()) //nolint:gosec // testing only
188+
}, func() {
189+
assert.NoError(b, exec.Command("sudo", suite.docker, "rmi", "--help").Run())
190+
})
191+
}
111192
}

0 commit comments

Comments
 (0)