Skip to content

Commit f8c71c7

Browse files
committed
Use docker buildx to build Docker images
Move away from docker-api gem which does not have support for `docker buildx`. Add a wrapper to execute `docker buildx` commands in the shell and use that for Docker operations. Closes: https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/8469 Signed-off-by: Balasankar 'Balu' C <[email protected]>
1 parent a068eb2 commit f8c71c7

File tree

5 files changed

+277
-25
lines changed

5 files changed

+277
-25
lines changed

lib/gitlab/build/image.rb

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,15 @@ def copy_image_to_dockerhub(final_tag)
5151
SkopeoHelper.copy_image(source, target)
5252
end
5353

54+
def copy_image_to_gitlab_registry(final_tag)
55+
source = source_image_address
56+
target = gitlab_registry_image_address(tag: final_tag)
57+
58+
SkopeoHelper.login('gitlab-ci-token', Gitlab::Util.get_env('CI_JOB_TOKEN'), Gitlab::Util.get_env('CI_REGISTRY'))
59+
60+
SkopeoHelper.copy_image(source, target)
61+
end
62+
5463
def source_image_address
5564
raise NotImplementedError
5665
end

lib/gitlab/docker_helper.rb

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
require 'open3'
2+
require_relative 'docker_operations'
3+
4+
# TODO: Deprecate DockerOperations
5+
class DockerHelper < DockerOperations
6+
class << self
7+
def authenticate(username: nil, password: nil, registry: nil)
8+
puts "Logging in to Docker registry"
9+
10+
stdout, stderr, status = Open3.popen3({}, *%W[docker login --username=#{username} --password-stdin #{registry}]) do |stdin, stdout, stderr, wait_thr|
11+
stdin.puts(password)
12+
stdin.close
13+
14+
[stdout.read, stderr.read, wait_thr.value]
15+
end
16+
17+
return if status.success?
18+
19+
puts "Failed to login to Docker registry."
20+
puts "Output is: #{stdout}"
21+
puts stderr
22+
Kernel.exit 1
23+
end
24+
25+
# TODO: When multi-arch images are built by default, modify `platforms`
26+
# array to include `linux/arm64` also
27+
def build(location, image, tag, dockerfile: nil, buildargs: nil, platforms: %w[linux/amd64], push: true)
28+
create_builder
29+
30+
commands = %W[docker buildx build #{location} -t #{image}:#{tag}]
31+
32+
if (env_var_platforms = Gitlab::Util.get_env('DOCKER_BUILD_PLATFORMS'))
33+
platforms.append(env_var_platforms.split(",").map(&:strip))
34+
end
35+
36+
platforms.uniq!
37+
38+
commands += %W[--platform=#{platforms.join(',')}]
39+
40+
# If specified to push, we must push to registry. Even if not, if the
41+
# image being built is multiarch, we must push to registry.
42+
commands += %w[--push] if push || platforms.length > 1
43+
44+
commands += %W[-f #{dockerfile}] if dockerfile
45+
46+
buildargs&.each do |arg|
47+
commands += %W[--build-arg='#{arg}']
48+
end
49+
50+
puts "Running command: #{commands.join(' ')}"
51+
52+
Open3.popen2e(*commands) do |_, stdout_stderr, status|
53+
while line = stdout_stderr.gets
54+
puts line
55+
end
56+
57+
Kernel.exit 1 unless status.value.success?
58+
end
59+
end
60+
61+
def create_builder
62+
cleanup_existing_builder
63+
64+
puts "Creating docker builder instance"
65+
# TODO: For multi-arch builds, use Kubernetes driver for builder instance
66+
_, stdout_stderr, status = Open3.popen2e(*%w[docker buildx create --bootstrap --use --name omnibus-gitlab-builder])
67+
68+
return if status.value.success?
69+
70+
puts "Creating builder instance failed."
71+
puts "Output: #{stdout_stderr.read}"
72+
raise
73+
end
74+
75+
def cleanup_existing_builder
76+
puts "Cleaning any existing builder instances."
77+
_, _, status = Open3.popen2e(*%w[docker buildx ls | grep omnibus-gitlab-builder])
78+
unless status.value.success?
79+
puts "omnibus-gitlab-builder instance not found. Not attempting to clean."
80+
return
81+
end
82+
83+
_, stdout_stderr, status = Open3.popen2e(*%w[docker buildx rm --force omnibus-gitlab-builder])
84+
if status.value.success?
85+
puts "Successfully cleaned omnibus-gitlab-builder instance."
86+
else
87+
puts "Cleaning of omnibus-gitlab-builder instance failed."
88+
puts "Output: #{stdout_stderr.read}"
89+
end
90+
end
91+
end
92+
end

lib/gitlab/tasks/docker_tasks.rake

Lines changed: 12 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ require_relative '../build/gitlab_image'
55
require_relative '../build/info/ci'
66
require_relative '../build/info/docker'
77
require_relative '../docker_operations'
8+
require_relative '../docker_helper'
89
require_relative '../util'
910

1011
namespace :docker do
@@ -14,11 +15,8 @@ namespace :docker do
1415
Gitlab::Util.section('docker:build:image') do
1516
Build::GitlabImage.write_release_file
1617
location = File.absolute_path(File.join(File.dirname(File.expand_path(__FILE__)), "../../../docker"))
17-
DockerOperations.build(
18-
location,
19-
Build::GitlabImage.gitlab_registry_image_address,
20-
'latest'
21-
)
18+
DockerHelper.authenticate(username: "gitlab-ci-token", password: Gitlab::Util.get_env("CI_JOB_TOKEN"), registry: Gitlab::Util.get_env('CI_REGISTRY'))
19+
DockerHelper.build(location, Build::GitlabImage.gitlab_registry_image_address, Build::Info::Docker.tag)
2220
end
2321
end
2422
end
@@ -28,11 +26,11 @@ namespace :docker do
2826
# Only runs on dev.gitlab.org
2927
task :staging do
3028
Gitlab::Util.section('docker:push:staging') do
31-
Build::GitlabImage.tag_and_push_to_gitlab_registry(Build::Info::Docker.tag)
32-
33-
# Also tag with CI_COMMIT_REF_SLUG so that manual testing using Docker
34-
# can use the same image name/tag.
35-
Build::GitlabImage.tag_and_push_to_gitlab_registry(Build::Info::CI.commit_ref_slug)
29+
# As part of build, the image is already tagged and pushed to GitLab
30+
# registry with `Build::Info::Docker.tag` as the tag. Also copy the
31+
# image with `CI_COMMIT_REF_SLUG` as the tag so that manual testing
32+
# using Docker can use the same image name/tag.
33+
Build::GitlabImage.copy_image_to_gitlab_registry(Build::Info::CI.commit_ref_slug)
3634
end
3735
end
3836

@@ -88,11 +86,11 @@ namespace :docker do
8886
desc "Push triggered Docker Image to GitLab Registry"
8987
task :triggered do
9088
Gitlab::Util.section('docker:push:triggered') do
91-
Build::GitlabImage.tag_and_push_to_gitlab_registry(Build::Info::Docker.tag)
92-
93-
# Also tag with CI_COMMIT_REF_SLUG so that manual testing using Docker
89+
# As part of build, the image is already tagged and pushed with
90+
# `Build::Info::Docker.tag` as the tag. Also copy the image with
91+
# `CI_COMMIT_REF_SLUG` as the tag so that manual testing using Docker
9492
# can use the same image name/tag.
95-
Build::GitlabImage.tag_and_push_to_gitlab_registry(Build::Info::CI.commit_ref_slug)
93+
Build::GitlabImage.copy_image_to_gitlab_registry(Build::Info::CI.commit_ref_slug)
9694
end
9795
end
9896
end

spec/lib/gitlab/docker_helper_spec.rb

Lines changed: 146 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,146 @@
1+
require 'spec_helper'
2+
require 'gitlab/docker_helper'
3+
4+
RSpec.describe DockerHelper do
5+
describe '.authenticate' do
6+
before do
7+
stdin_mock = double(puts: true, close: true)
8+
stdout_mock = double(read: true)
9+
stderr_mock = double(read: true)
10+
wait_thr_mock = double(value: double(success?: true))
11+
12+
allow(Open3).to receive(:popen3).with({}, "docker", "login", any_args).and_yield(stdin_mock, stdout_mock, stderr_mock, wait_thr_mock)
13+
end
14+
15+
context 'when a registry is not specified' do
16+
it 'runs the command to login to docker.io' do
17+
expect(Open3).to receive(:popen3).with({}, "docker", "login", "--username=dummy-username", "--password-stdin", "")
18+
19+
described_class.authenticate(username: 'dummy-username', password: 'dummy-password')
20+
end
21+
end
22+
23+
context 'when a registry is specified' do
24+
it 'runs the command to login to specified registry' do
25+
expect(Open3).to receive(:popen3).with({}, "docker", "login", "--username=dummy-username", "--password-stdin", "registry.gitlab.com")
26+
27+
described_class.authenticate(username: 'dummy-username', password: 'dummy-password', registry: 'registry.gitlab.com')
28+
end
29+
end
30+
end
31+
32+
describe '.build' do
33+
shared_examples 'docker build command invocation' do
34+
end
35+
36+
before do
37+
stdout_stderr_mock = double(gets: nil)
38+
status_mock = double(value: double(success?: true))
39+
40+
allow(described_class).to receive(:create_builder).and_return(true)
41+
42+
allow(Open3).to receive(:popen2e).with("docker", "buildx", "build", any_args).and_yield(nil, stdout_stderr_mock, status_mock)
43+
end
44+
45+
context 'when a single platform is specified' do
46+
context 'when push is not explicitly disabled' do
47+
let(:expected_args) { %w[docker buildx build /tmp/foo -t sample:value --platform=linux/amd64 --push] }
48+
49+
it 'calls docker build command with correct arguments' do
50+
expect(Open3).to receive(:popen2e).with(*expected_args)
51+
52+
described_class.build('/tmp/foo', 'sample', 'value')
53+
end
54+
end
55+
56+
context 'when push is explicitly disabled' do
57+
let(:expected_args) { %w[docker buildx build /tmp/foo -t sample:value --platform=linux/amd64] }
58+
59+
it 'calls docker build command with correct arguments' do
60+
expect(Open3).to receive(:popen2e).with(*expected_args)
61+
62+
described_class.build('/tmp/foo', 'sample', 'value', push: false)
63+
end
64+
end
65+
end
66+
67+
context 'when multiple platforms are specified via env vars' do
68+
before do
69+
stub_env_var('DOCKER_BUILD_PLATFORMS', 'linux/arm64')
70+
end
71+
72+
let(:expected_args) { %w[docker buildx build /tmp/foo -t sample:value --platform=linux/amd64,linux/arm64 --push] }
73+
74+
it 'calls docker build command with correct arguments' do
75+
expect(Open3).to receive(:popen2e).with(*expected_args)
76+
77+
described_class.build('/tmp/foo', 'sample', 'value')
78+
end
79+
80+
context 'even if push is explicitly disabled' do
81+
let(:expected_args) { %w[docker buildx build /tmp/foo -t sample:value --platform=linux/amd64,linux/arm64 --push] }
82+
83+
it 'calls docker build command with correct arguments' do
84+
expect(Open3).to receive(:popen2e).with(*expected_args)
85+
86+
described_class.build('/tmp/foo', 'sample', 'value', push: false)
87+
end
88+
end
89+
end
90+
91+
context 'when build_args are specified' do
92+
let(:expected_args) { %w[docker buildx build /tmp/foo -t sample:value --platform=linux/amd64 --push --build-arg='FOO=BAR'] }
93+
94+
it 'calls docker build command with correct arguments' do
95+
expect(Open3).to receive(:popen2e).with(*expected_args)
96+
97+
described_class.build('/tmp/foo', 'sample', 'value', buildargs: ["FOO=BAR"])
98+
end
99+
end
100+
end
101+
102+
describe '.create_builder' do
103+
before do
104+
allow(described_class).to receive(:cleanup_existing_builder).and_return(true)
105+
106+
stdout_stderr_mock = double(gets: nil)
107+
status_mock = double(value: double(success?: true))
108+
allow(Open3).to receive(:popen2e).with("docker", "buildx", "create", any_args).and_return([nil, stdout_stderr_mock, status_mock])
109+
end
110+
111+
it 'calls docker buildx create command with correct arguments' do
112+
expect(Open3).to receive(:popen2e).with(*%w[docker buildx create --bootstrap --use --name omnibus-gitlab-builder])
113+
114+
described_class.create_builder
115+
end
116+
end
117+
118+
describe '.cleanup_existing_builder' do
119+
context 'when no builder instance exist' do
120+
before do
121+
status_mock = double(value: double(success?: false))
122+
allow(Open3).to receive(:popen2e).with("docker", "buildx", "ls", any_args).and_return([nil, nil, status_mock])
123+
end
124+
125+
it 'does not call docker buildx rm' do
126+
expect(Open3).not_to receive(:popen2e).with(*%w[docker buildx rm --force omnibus-gitlab-builder])
127+
128+
described_class.cleanup_existing_builder
129+
end
130+
end
131+
132+
context 'when builder instance exists' do
133+
before do
134+
status_mock = double(value: double(success?: true))
135+
allow(Open3).to receive(:popen2e).with("docker", "buildx", "ls", any_args).and_return([nil, nil, status_mock])
136+
allow(Open3).to receive(:popen2e).with("docker", "buildx", "rm", any_args).and_return([nil, nil, status_mock])
137+
end
138+
139+
it 'calls docker buildx rm command with correct arguments' do
140+
expect(Open3).to receive(:popen2e).with(*%w[docker buildx rm --force omnibus-gitlab-builder])
141+
142+
described_class.cleanup_existing_builder
143+
end
144+
end
145+
end
146+
end

spec/lib/gitlab/tasks/docker_tasks_spec.rb

Lines changed: 18 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -12,14 +12,17 @@
1212
end
1313

1414
it 'calls build command with correct parameters' do
15-
allow(ENV).to receive(:[]).with('CI_REGISTRY_IMAGE').and_return('dev.gitlab.org:5005/gitlab/omnibus-gitlab')
15+
allow(ENV).to receive(:[]).with('CI_REGISTRY_IMAGE').and_return('registry.com/group/repo')
16+
allow(Build::Info::Docker).to receive(:tag).and_return('9.0.0')
1617
allow(Build::Info::Package).to receive(:name).and_return('gitlab-ce')
1718
allow(Build::GitlabImage).to receive(:write_release_file).and_return(true)
1819
allow(File).to receive(:expand_path).and_return('/tmp/omnibus-gitlab/lib/gitlab/tasks/docker_tasks.rake')
19-
allow(DockerOperations).to receive(:build).and_call_original
2020

21-
expect(DockerOperations).to receive(:build).with("/tmp/omnibus-gitlab/docker", "dev.gitlab.org:5005/gitlab/omnibus-gitlab/gitlab-ce", "latest")
22-
expect(Docker::Image).to receive(:build_from_dir).with("/tmp/omnibus-gitlab/docker", { t: "dev.gitlab.org:5005/gitlab/omnibus-gitlab/gitlab-ce:latest", pull: true })
21+
allow(DockerHelper).to receive(:authenticate).and_return(true)
22+
allow(DockerHelper).to receive(:build).and_return(true)
23+
allow(DockerHelper).to receive(:create_builder).and_return(true)
24+
25+
expect(DockerHelper).to receive(:build).with("/tmp/omnibus-gitlab/docker", "registry.com/group/repo/gitlab-ce", '9.0.0')
2326
Rake::Task['docker:build:image'].invoke
2427
end
2528
end
@@ -82,11 +85,15 @@
8285
describe 'docker:push:staging' do
8386
before do
8487
Rake::Task['docker:push:staging'].reenable
88+
allow(ENV).to receive(:[]).with('CI_COMMIT_REF_SLUG').and_return('foo-bar')
89+
allow(ENV).to receive(:[]).with('CI_REGISTRY_IMAGE').and_return('registry.gitlab.com/gitlab-org/omnibus-gitlab')
90+
allow(SkopeoHelper).to receive(:copy_image).and_return(true)
91+
allow(Build::Info::Package).to receive(:name).and_return('gitlab-ce')
92+
allow(Build::Info::Docker).to receive(:tag).and_return('1.2.3.4')
8593
end
8694

87-
it 'pushes to staging correctly' do
88-
expect(dummy_image).to receive(:push).with(dummy_creds, repo_tag: 'dev.gitlab.org:5005/gitlab/omnibus-gitlab/gitlab-ce:9.0.0')
89-
expect(dummy_image).to receive(:push).with(dummy_creds, repo_tag: 'dev.gitlab.org:5005/gitlab/omnibus-gitlab/gitlab-ce:foo-bar')
95+
it 'pushes triggered images correctly' do
96+
expect(SkopeoHelper).to receive(:copy_image).with('registry.gitlab.com/gitlab-org/omnibus-gitlab/gitlab-ce:1.2.3.4', 'registry.gitlab.com/gitlab-org/omnibus-gitlab/gitlab-ce:foo-bar')
9097
Rake::Task['docker:push:staging'].invoke
9198
end
9299
end
@@ -183,13 +190,13 @@
183190
Rake::Task['docker:push:triggered'].reenable
184191
allow(ENV).to receive(:[]).with('CI_COMMIT_REF_SLUG').and_return('foo-bar')
185192
allow(ENV).to receive(:[]).with('CI_REGISTRY_IMAGE').and_return('registry.gitlab.com/gitlab-org/omnibus-gitlab')
186-
allow(ENV).to receive(:[]).with("IMAGE_TAG").and_return("omnibus-12345")
187-
allow(Build::Info::Docker).to receive(:tag).and_call_original
193+
allow(SkopeoHelper).to receive(:copy_image).and_return(true)
194+
allow(Build::Info::Package).to receive(:name).and_return('gitlab-ce')
195+
allow(Build::Info::Docker).to receive(:tag).and_return('1.2.3.4')
188196
end
189197

190198
it 'pushes triggered images correctly' do
191-
expect(dummy_image).to receive(:push).with(dummy_creds, repo_tag: 'registry.gitlab.com/gitlab-org/omnibus-gitlab/gitlab-ce:omnibus-12345')
192-
expect(dummy_image).to receive(:push).with(dummy_creds, repo_tag: 'registry.gitlab.com/gitlab-org/omnibus-gitlab/gitlab-ce:foo-bar')
199+
expect(SkopeoHelper).to receive(:copy_image).with('registry.gitlab.com/gitlab-org/omnibus-gitlab/gitlab-ce:1.2.3.4', 'registry.gitlab.com/gitlab-org/omnibus-gitlab/gitlab-ce:foo-bar')
193200
Rake::Task['docker:push:triggered'].invoke
194201
end
195202
end

0 commit comments

Comments
 (0)