Skip to content

Commit 0702eb4

Browse files
balasankarcbalasankarc
committed
Make AWS AMI builder fetch package from artifact instead of S3 bucket
Signed-off-by: Balasankar "Balu" C <[email protected]>
1 parent d506125 commit 0702eb4

27 files changed

+1079
-44
lines changed

‎gitlab-ci-config/gitlab-com.yml

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -713,12 +713,12 @@ validate_packer_changes:
713713
image: "${PUBLIC_BUILDER_IMAGE_REGISTRY}/debian_packer:${BUILDER_IMAGE_REVISION}"
714714
stage: check
715715
script:
716-
- cd "${CI_PROJECT_DIR}/support/packer" && packer validate -var aws_access_key=XXX -var aws_secret_key=XXX -var download_url=XXX ce-arm64.pkr.hcl
717-
- cd "${CI_PROJECT_DIR}/support/packer" && packer validate -var aws_access_key=XXX -var aws_secret_key=XXX -var download_url=XXX ce.pkr.hcl
718-
- cd "${CI_PROJECT_DIR}/support/packer" && packer validate -var aws_access_key=XXX -var aws_secret_key=XXX -var download_url=XXX ee-arm64.pkr.hcl
719-
- cd "${CI_PROJECT_DIR}/support/packer" && packer validate -var aws_access_key=XXX -var aws_secret_key=XXX -var download_url=XXX ee-premium.pkr.hcl
720-
- cd "${CI_PROJECT_DIR}/support/packer" && packer validate -var aws_access_key=XXX -var aws_secret_key=XXX -var download_url=XXX ee-ultimate.pkr.hcl
721-
- cd "${CI_PROJECT_DIR}/support/packer" && packer validate -var aws_access_key=XXX -var aws_secret_key=XXX -var download_url=XXX ee.pkr.hcl
716+
- cd "${CI_PROJECT_DIR}/support/packer" && packer validate -var ci_job_token=XXX -var aws_access_key=XXX -var aws_secret_key=XXX -var download_url=XXX ce-arm64.pkr.hcl
717+
- cd "${CI_PROJECT_DIR}/support/packer" && packer validate -var ci_job_token=XXX -var aws_access_key=XXX -var aws_secret_key=XXX -var download_url=XXX ce.pkr.hcl
718+
- cd "${CI_PROJECT_DIR}/support/packer" && packer validate -var ci_job_token=XXX -var aws_access_key=XXX -var aws_secret_key=XXX -var download_url=XXX ee-arm64.pkr.hcl
719+
- cd "${CI_PROJECT_DIR}/support/packer" && packer validate -var ci_job_token=XXX -var aws_access_key=XXX -var aws_secret_key=XXX -var download_url=XXX ee-premium.pkr.hcl
720+
- cd "${CI_PROJECT_DIR}/support/packer" && packer validate -var ci_job_token=XXX -var aws_access_key=XXX -var aws_secret_key=XXX -var download_url=XXX ee-ultimate.pkr.hcl
721+
- cd "${CI_PROJECT_DIR}/support/packer" && packer validate -var ci_job_token=XXX -var aws_access_key=XXX -var aws_secret_key=XXX -var download_url=XXX ee.pkr.hcl
722722
rules:
723723
- if: '$PIPELINE_TYPE == "_TEST_PIPELINE"'
724724
changes:

‎lib/gitlab/aws_helper.rb

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ def initialize(version, type)
1414
@type = type || 'ce'
1515
end
1616

17-
def create_ami
17+
def create_ami_old
1818
release_type = Gitlab::Util.get_env('AWS_RELEASE_TYPE')
1919
architecture = Gitlab::Util.get_env('AWS_ARCHITECTURE')
2020
args = {}
@@ -31,7 +31,27 @@ def create_ami
3131

3232
@download_url = Build::Info.ami_deb_package_download_url(**args)
3333

34-
system(*%W[support/packer/packer_ami.sh #{@version} #{@type} #{@download_url} #{@license_file}])
34+
system(*%W[support/packer_old/packer_ami.sh #{@version} #{@type} #{@download_url} #{@license_file}])
35+
end
36+
37+
def create_ami
38+
release_type = Gitlab::Util.get_env('AWS_RELEASE_TYPE')
39+
architecture = Gitlab::Util.get_env('AWS_ARCHITECTURE')
40+
41+
if (@type == 'ee') && release_type
42+
@type = "ee-#{release_type}"
43+
@license_file = "AWS_#{release_type}_LICENSE_FILE".upcase
44+
end
45+
46+
if architecture
47+
@type = "#{@type}-#{architecture}"
48+
else
49+
architecture = 'amd64'
50+
end
51+
52+
@download_url = Build::Info::CI.package_download_url(job_name: "Ubuntu-20.04", arch: architecture)
53+
54+
system(*%W[support/packer/packer_ami.sh #{@version} #{@type} #{@download_url} #{Build::Info::CI.job_token} #{@license_file}])
3555
end
3656

3757
def set_marketplace_details

‎lib/gitlab/tasks/aws.rake

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,11 @@ namespace :aws do
1212
next if Build::Check.is_auto_deploy? || Build::Check.is_rc_tag?
1313

1414
Omnibus.load_configuration('omnibus.rb')
15-
AWSHelper.new(Omnibus::BuildVersion.semver, Build::Info.edition).create_ami
15+
if Gitlab::Util.get_env('AMI_USE_OLD_BUILD_PROCESS') == "true"
16+
AWSHelper.new(Omnibus::BuildVersion.semver, Build::Info.edition).create_ami_old
17+
else
18+
AWSHelper.new(Omnibus::BuildVersion.semver, Build::Info.edition).create_ami
19+
end
1620
end
1721
end
1822

‎spec/lib/gitlab/tasks/aws_spec.rb

Lines changed: 31 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -40,21 +40,42 @@ def deregister_image(parameters)
4040
before do
4141
Rake::Task['aws:ami:create'].reenable
4242
allow_any_instance_of(Kernel).to receive(:system).and_return(true)
43+
allow(ENV).to receive(:[]).and_call_original
44+
stub_env_var('CI_JOB_TOKEN', 'CI-NO-JOB-TOKEN')
4345
end
4446

45-
describe 'on a regular tag' do
47+
context 'when using `AMI_USE_OLD_BUILD_PROCESS` environment variable' do
4648
before do
49+
stub_env_var('AMI_USE_OLD_BUILD_PROCESS', 'true')
4750
allow(Build::Check).to receive(:on_tag?).and_return(true)
4851
allow(Build::Check).to receive(:is_auto_deploy?).and_return(false)
4952
allow(Build::Check).to receive(:is_rc_tag?).and_return(false)
5053
allow(Build::Info).to receive(:ami_deb_package_download_url).and_return('http://example.com')
5154
end
5255

56+
it 'should call the old script' do
57+
allow(Build::Info).to receive(:edition).and_return('ce')
58+
allow(Omnibus::BuildVersion).to receive(:semver).and_return('9.3.0')
59+
60+
expect_any_instance_of(Kernel).to receive(:system).with(*["support/packer_old/packer_ami.sh", "9.3.0", "ce", "http://example.com", ""])
61+
62+
Rake::Task['aws:ami:create'].invoke
63+
end
64+
end
65+
66+
describe 'on a regular tag' do
67+
before do
68+
allow(Build::Check).to receive(:on_tag?).and_return(true)
69+
allow(Build::Check).to receive(:is_auto_deploy?).and_return(false)
70+
allow(Build::Check).to receive(:is_rc_tag?).and_return(false)
71+
allow(Build::Info::CI).to receive(:package_download_url).and_return('http://example.com')
72+
end
73+
5374
it 'should identify ce category correctly, if specified' do
5475
allow(Build::Info).to receive(:edition).and_return('ce')
5576
allow(Omnibus::BuildVersion).to receive(:semver).and_return('9.3.0')
5677

57-
expect_any_instance_of(Kernel).to receive(:system).with(*["support/packer/packer_ami.sh", "9.3.0", "ce", "http://example.com", ""])
78+
expect_any_instance_of(Kernel).to receive(:system).with(*["support/packer/packer_ami.sh", "9.3.0", "ce", "http://example.com", "CI-NO-JOB-TOKEN", ""])
5879

5980
Rake::Task['aws:ami:create'].invoke
6081
end
@@ -63,7 +84,7 @@ def deregister_image(parameters)
6384
allow(Build::Info).to receive(:edition).and_return(nil)
6485
allow(Omnibus::BuildVersion).to receive(:semver).and_return('9.3.0')
6586

66-
expect_any_instance_of(Kernel).to receive(:system).with(*["support/packer/packer_ami.sh", "9.3.0", "ce", "http://example.com", ""])
87+
expect_any_instance_of(Kernel).to receive(:system).with(*["support/packer/packer_ami.sh", "9.3.0", "ce", "http://example.com", "CI-NO-JOB-TOKEN", ""])
6788

6889
Rake::Task['aws:ami:create'].invoke
6990
end
@@ -72,7 +93,7 @@ def deregister_image(parameters)
7293
allow(Build::Info).to receive(:edition).and_return('ee')
7394
allow(Omnibus::BuildVersion).to receive(:semver).and_return('9.3.0')
7495

75-
expect_any_instance_of(Kernel).to receive(:system).with(*["support/packer/packer_ami.sh", "9.3.0", "ee", "http://example.com", ""])
96+
expect_any_instance_of(Kernel).to receive(:system).with(*["support/packer/packer_ami.sh", "9.3.0", "ee", "http://example.com", "CI-NO-JOB-TOKEN", ""])
7697

7798
Rake::Task['aws:ami:create'].invoke
7899
end
@@ -83,7 +104,7 @@ def deregister_image(parameters)
83104
allow(Build::Info).to receive(:edition).and_return(nil)
84105
allow(Omnibus::BuildVersion).to receive(:semver).and_return('9.3.0')
85106

86-
expect_any_instance_of(Kernel).to receive(:system).with(*["support/packer/packer_ami.sh", "9.3.0", "ce-arm64", "http://example.com", ""])
107+
expect_any_instance_of(Kernel).to receive(:system).with(*["support/packer/packer_ami.sh", "9.3.0", "ce-arm64", "http://example.com", "CI-NO-JOB-TOKEN", ""])
87108

88109
Rake::Task['aws:ami:create'].invoke
89110
end
@@ -94,7 +115,7 @@ def deregister_image(parameters)
94115
allow(Build::Info).to receive(:edition).and_return('ee')
95116
allow(Omnibus::BuildVersion).to receive(:semver).and_return('9.3.0')
96117

97-
expect_any_instance_of(Kernel).to receive(:system).with(*["support/packer/packer_ami.sh", "9.3.0", "ee-arm64", "http://example.com", ""])
118+
expect_any_instance_of(Kernel).to receive(:system).with(*["support/packer/packer_ami.sh", "9.3.0", "ee-arm64", "http://example.com", "CI-NO-JOB-TOKEN", ""])
98119

99120
Rake::Task['aws:ami:create'].invoke
100121
end
@@ -105,7 +126,7 @@ def deregister_image(parameters)
105126
allow(Gitlab::Util).to receive(:get_env).with("AWS_RELEASE_TYPE").and_return('ultimate')
106127
allow(Omnibus::BuildVersion).to receive(:semver).and_return('9.3.0')
107128

108-
expect_any_instance_of(Kernel).to receive(:system).with(*["support/packer/packer_ami.sh", "9.3.0", "ee-ultimate", "http://example.com", "AWS_ULTIMATE_LICENSE_FILE"])
129+
expect_any_instance_of(Kernel).to receive(:system).with(*["support/packer/packer_ami.sh", "9.3.0", "ee-ultimate", "http://example.com", "CI-NO-JOB-TOKEN", "AWS_ULTIMATE_LICENSE_FILE"])
109130

110131
Rake::Task['aws:ami:create'].invoke
111132
end
@@ -116,7 +137,7 @@ def deregister_image(parameters)
116137
allow(Gitlab::Util).to receive(:get_env).with("AWS_RELEASE_TYPE").and_return('premium')
117138
allow(Omnibus::BuildVersion).to receive(:semver).and_return('9.3.0')
118139

119-
expect_any_instance_of(Kernel).to receive(:system).with(*["support/packer/packer_ami.sh", "9.3.0", "ee-premium", "http://example.com", "AWS_PREMIUM_LICENSE_FILE"])
140+
expect_any_instance_of(Kernel).to receive(:system).with(*["support/packer/packer_ami.sh", "9.3.0", "ee-premium", "http://example.com", "CI-NO-JOB-TOKEN", "AWS_PREMIUM_LICENSE_FILE"])
120141

121142
Rake::Task['aws:ami:create'].invoke
122143
end
@@ -127,7 +148,7 @@ def deregister_image(parameters)
127148
allow(Build::Check).to receive(:on_tag?).and_return(true)
128149
allow(Build::Check).to receive(:is_auto_deploy?).and_return(false)
129150
allow(Build::Check).to receive(:is_rc_tag?).and_return(true)
130-
allow(Build::Info).to receive(:ami_deb_package_download_url).and_return('http://example.com')
151+
allow(Build::Info::CI).to receive(:package_download_url).and_return('http://example.com')
131152
end
132153

133154
it 'does not do anything' do
@@ -142,7 +163,7 @@ def deregister_image(parameters)
142163
allow(Build::Check).to receive(:on_tag?).and_return(true)
143164
allow(Build::Check).to receive(:is_auto_deploy?).and_return(true)
144165
allow(Build::Check).to receive(:is_rc_tag?).and_return(false)
145-
allow(Build::Info).to receive(:ami_deb_package_download_url).and_return('http://example.com')
166+
allow(Build::Info::CI).to receive(:package_download_url).and_return('http://example.com')
146167
end
147168

148169
it 'does not do anything' do

‎support/packer/ce-arm64.pkr.hcl

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,11 @@ variable "download_url" {
1414
type = string
1515
}
1616

17+
# ci_job_token is the token used to download the package from CI artifacts
18+
variable "ci_job_token" {
19+
type = string
20+
}
21+
1722
# license_file, somewhat of a misnomer, is the contents of the license to
1823
# install on the image. Due to the size of the license contents, it is usually
1924
# better to use a shell variable to hold the contents and then use the variable
@@ -121,7 +126,7 @@ build {
121126
}
122127

123128
provisioner "shell" {
124-
environment_vars = ["DOWNLOAD_URL=${var.download_url}"]
129+
environment_vars = ["DOWNLOAD_URL=${var.download_url}", "CI_JOB_TOKEN=${var.ci_job_token}"]
125130
script = "update-script-ce.sh"
126131
}
127132

‎support/packer/ce.pkr.hcl

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,11 @@ variable "download_url" {
1414
type = string
1515
}
1616

17+
# ci_job_token is the token used to download the package from CI artifacts
18+
variable "ci_job_token" {
19+
type = string
20+
}
21+
1722
# license_file, somewhat of a misnomer, is the contents of the license to
1823
# install on the image. Due to the size of the license contents, it is usually
1924
# better to use a shell variable to hold the contents and then use the variable
@@ -121,7 +126,7 @@ build {
121126
}
122127

123128
provisioner "shell" {
124-
environment_vars = ["DOWNLOAD_URL=${var.download_url}"]
129+
environment_vars = ["DOWNLOAD_URL=${var.download_url}", "CI_JOB_TOKEN=${var.ci_job_token}"]
125130
script = "update-script-ce.sh"
126131
}
127132

‎support/packer/ee-arm64.pkr.hcl

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,11 @@ variable "download_url" {
1414
type = string
1515
}
1616

17+
# ci_job_token is the token used to download the package from CI artifacts
18+
variable "ci_job_token" {
19+
type = string
20+
}
21+
1722
# license_file, somewhat of a misnomer, is the contents of the license to
1823
# install on the image. Due to the size of the license contents, it is usually
1924
# better to use a shell variable to hold the contents and then use the variable
@@ -118,7 +123,7 @@ build {
118123
}
119124

120125
provisioner "shell" {
121-
environment_vars = ["DOWNLOAD_URL=${var.download_url}"]
126+
environment_vars = ["DOWNLOAD_URL=${var.download_url}", "CI_JOB_TOKEN=${var.ci_job_token}"]
122127
script = "update-script-ee.sh"
123128
}
124129

‎support/packer/ee-premium.pkr.hcl

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,11 @@ variable "download_url" {
1414
type = string
1515
}
1616

17+
# ci_job_token is the token used to download the package from CI artifacts
18+
variable "ci_job_token" {
19+
type = string
20+
}
21+
1722
# license_file, somewhat of a misnomer, is the contents of the license to
1823
# install on the image. Due to the size of the license contents, it is usually
1924
# better to use a shell variable to hold the contents and then use the variable
@@ -117,7 +122,7 @@ build {
117122
}
118123

119124
provisioner "shell" {
120-
environment_vars = ["DOWNLOAD_URL=${var.download_url}", "GITLAB_LICENSE_FILE=${var.license_file}"]
125+
environment_vars = ["DOWNLOAD_URL=${var.download_url}", "GITLAB_LICENSE_FILE=${var.license_file}", "CI_JOB_TOKEN=${var.ci_job_token}"]
121126
script = "update-script-ee-premium.sh"
122127
}
123128

‎support/packer/ee-ultimate.pkr.hcl

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,11 @@ variable "download_url" {
1414
type = string
1515
}
1616

17+
# ci_job_token is the token used to download the package from CI artifacts
18+
variable "ci_job_token" {
19+
type = string
20+
}
21+
1722
# license_file, somewhat of a misnomer, is the contents of the license to
1823
# install on the image. Due to the size of the license contents, it is usually
1924
# better to use a shell variable to hold the contents and then use the variable
@@ -117,7 +122,7 @@ build {
117122
}
118123

119124
provisioner "shell" {
120-
environment_vars = ["DOWNLOAD_URL=${var.download_url}", "GITLAB_LICENSE_FILE=${var.license_file}"]
125+
environment_vars = ["DOWNLOAD_URL=${var.download_url}", "GITLAB_LICENSE_FILE=${var.license_file}", "CI_JOB_TOKEN=${var.ci_job_token}"]
121126
script = "update-script-ee-ultimate.sh"
122127
}
123128

‎support/packer/ee.pkr.hcl

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,11 @@ variable "download_url" {
1414
type = string
1515
}
1616

17+
# ci_job_token is the token used to download the package from CI artifacts
18+
variable "ci_job_token" {
19+
type = string
20+
}
21+
1722
# license_file, somewhat of a misnomer, is the contents of the license to
1823
# install on the image. Due to the size of the license contents, it is usually
1924
# better to use a shell variable to hold the contents and then use the variable
@@ -118,7 +123,7 @@ build {
118123
}
119124

120125
provisioner "shell" {
121-
environment_vars = ["DOWNLOAD_URL=${var.download_url}"]
126+
environment_vars = ["DOWNLOAD_URL=${var.download_url}", "CI_JOB_TOKEN=${var.ci_job_token}"]
122127
script = "update-script-ee.sh"
123128
}
124129

0 commit comments

Comments
 (0)