Skip to content

Commit f8f678d

Browse files
mtojekmtojek
authored
Implement minimal pipeline to publish a package (#731)
* WIP * Call the minimal pipeline * Two pipelines starting * Use ubuntu 20 * Fix: paths * Fix: install elastic-package * Fix * Fix * Fix * Fix * Fix: zip build * Fix: artifactsPath * packageStoragePublish * Fix * Fix * Fix
1 parent e28b434 commit f8f678d

File tree

2 files changed

+121
-81
lines changed

2 files changed

+121
-81
lines changed

‎.ci/Jenkinsfile‎

Lines changed: 2 additions & 78 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
@Library('apm@current') _
44

55
pipeline {
6-
agent { label 'ubuntu-18 && immutable' }
6+
agent { label 'ubuntu-20 && immutable' }
77
environment {
88
REPO = "elastic-package"
99

@@ -22,21 +22,6 @@ pipeline {
2222
JOB_GCS_EXT_CREDENTIALS = 'beats-ci-gcs-plugin-file-credentials'
2323
ELASTIC_PACKAGE_GCP_SECRET = 'secret/observability-team/ci/service-account/elastic-package-gcp'
2424
ELASTIC_OBSERVABILITY_PROJECT_ID = 'elastic-observability'
25-
26-
JOB_SIGNING_CREDENTIALS = 'sign-artifacts-with-gpg-job'
27-
INTERNAL_CI_JOB_GCS_CREDENTIALS = 'internal-ci-gcs-plugin'
28-
29-
REPO_BUILD_TAG = "${env.REPO}/${env.BUILD_TAG}"
30-
INFRA_SIGNING_BUCKET_NAME = 'internal-ci-artifacts'
31-
INFRA_SIGNING_BUCKET_SIGNED_ARTIFACTS_SUBFOLDER = "${env.REPO_BUILD_TAG}/signed-artifacts"
32-
INFRA_SIGNING_BUCKET_ARTIFACTS_PATH = "gs://${env.INFRA_SIGNING_BUCKET_NAME}/${env.REPO_BUILD_TAG}"
33-
INFRA_SIGNING_BUCKET_SIGNED_ARTIFACTS_PATH = "gs://${env.INFRA_SIGNING_BUCKET_NAME}/${env.INFRA_SIGNING_BUCKET_SIGNED_ARTIFACTS_SUBFOLDER}"
34-
35-
INTEGRATIONS_SIGNATURES_PATH = 'build/integrations-elastic-signatures' // different path not to override signatures archived in the "build-zip" step
36-
37-
PACKAGE_STORAGE_UPLOADER_CREDENTIALS = 'upload-package-to-package-storage'
38-
PACKAGE_STORAGE_UPLOADER_GCP_SERVICE_ACCOUNT = 'secret/gce/elastic-bekitzur/service-account/package-storage-uploader'
39-
PACKAGE_STORAGE_INTERNAL_BUCKET_QUEUE_PUBLISHING_PATH = "gs://elastic-bekitzur-package-storage-internal/queue-publishing/${env.REPO_BUILD_TAG}"
4025
}
4126
options {
4227
timeout(time: 1, unit: 'HOURS')
@@ -117,8 +102,7 @@ pipeline {
117102
'check-packages-with-kind': generateTestCommandStage(command: 'test-check-packages-with-kind', artifacts: ['build/test-results/*.xml', 'build/kubectl-dump.txt', 'build/elastic-stack-dump/check-*/logs/*.log', 'build/elastic-stack-dump/check-*/logs/fleet-server-internal/*'], junitArtifacts: true, publishCoverage: true),
118103
'check-packages-other': generateTestCommandStage(command: 'test-check-packages-other', artifacts: ['build/test-results/*.xml', 'build/elastic-stack-dump/check-*/logs/*.log', 'build/elastic-stack-dump/check-*/logs/fleet-server-internal/*'], junitArtifacts: true, publishCoverage: true),
119104
'build-zip': generateTestCommandStage(command: 'test-build-zip', artifacts: ['build/elastic-stack-dump/build-zip/logs/*.log', 'build/integrations/*.sig']),
120-
'profiles-command': generateTestCommandStage(command: 'test-profiles-command'),
121-
'publish-to-package-storage': generateTestPublishToPackageStorageStage()
105+
'profiles-command': generateTestCommandStage(command: 'test-profiles-command')
122106
]
123107

124108
def checkSinglePackageTasks = generateTestCheckSinglePackageStage()
@@ -163,66 +147,6 @@ def cleanup(){
163147
unstash 'source'
164148
}
165149

166-
def generateTestPublishToPackageStorageStage() {
167-
return {
168-
withNode(labels: "ubuntu-20 && immutable", sleepMax: 20, forceWorkspace: true) {
169-
cleanup()
170-
dir("${BASE_DIR}"){
171-
withMageEnv(){
172-
sh(label: 'Install elastic-package',script: "make install")
173-
dir("test/packages/package-storage/package_storage_candidate") {
174-
sh(label: 'Lint package',script: "elastic-package lint")
175-
sh(label: 'Build zipped package',script: "elastic-package build --zip")
176-
}
177-
signArtifactsWithElastic('build/integrations', env.INTEGRATIONS_SIGNATURES_PATH)
178-
179-
// Add the package candidate to the "queue-publishing"
180-
withGCPEnv(secret: env.PACKAGE_STORAGE_UPLOADER_GCP_SERVICE_ACCOUNT) {
181-
sh(label: 'Upload package .zip file', script: "gsutil cp ${env.INTEGRATIONS_SIGNATURES_PATH}/package_storage_candidate-0.0.1.zip ${env.PACKAGE_STORAGE_INTERNAL_BUCKET_QUEUE_PUBLISHING_PATH}/")
182-
sh(label: 'Upload package .sig file', script: "gsutil cp ${env.INTEGRATIONS_SIGNATURES_PATH}/package_storage_candidate-0.0.1.zip.sig ${env.PACKAGE_STORAGE_INTERNAL_BUCKET_QUEUE_PUBLISHING_PATH}/")
183-
}
184-
185-
// Call the publishing job
186-
withCredentials([string(credentialsId: env.PACKAGE_STORAGE_UPLOADER_CREDENTIALS, variable: 'TOKEN')]) {
187-
triggerRemoteJob(auth: CredentialsAuth(credentials: 'local-readonly-api-token'),
188-
job: 'https://internal-ci.elastic.co/job/package_storage/job/publishing-job-remote',
189-
token: TOKEN,
190-
parameters: """
191-
dry_run=true
192-
gs_package_build_zip_path=${env.PACKAGE_STORAGE_INTERNAL_BUCKET_QUEUE_PUBLISHING_PATH}/package_storage_candidate-0.0.1.zip
193-
gs_package_signature_path=${env.PACKAGE_STORAGE_INTERNAL_BUCKET_QUEUE_PUBLISHING_PATH}/package_storage_candidate-0.0.1.zip.sig
194-
""",
195-
useCrumbCache: false,
196-
useJobInfoCache: false)
197-
}
198-
}
199-
}
200-
}
201-
}
202-
}
203-
204-
def signArtifactsWithElastic(artifactsSourcePath, signaturesDestinationPath) {
205-
googleStorageUpload(bucket: env.INFRA_SIGNING_BUCKET_ARTIFACTS_PATH,
206-
credentialsId: env.INTERNAL_CI_JOB_GCS_CREDENTIALS,
207-
pathPrefix: artifactsSourcePath + '/',
208-
pattern: artifactsSourcePath + '/*.zip',
209-
sharedPublicly: false,
210-
showInline: true)
211-
withCredentials([string(credentialsId: env.JOB_SIGNING_CREDENTIALS, variable: 'TOKEN')]) {
212-
triggerRemoteJob(auth: CredentialsAuth(credentials: 'local-readonly-api-token'),
213-
job: 'https://internal-ci.elastic.co/job/elastic+unified-release+master+sign-artifacts-with-gpg',
214-
token: TOKEN,
215-
parameters: "gcs_input_path=${env.INFRA_SIGNING_BUCKET_ARTIFACTS_PATH}",
216-
useCrumbCache: false,
217-
useJobInfoCache: false)
218-
}
219-
googleStorageDownload(bucketUri: "${env.INFRA_SIGNING_BUCKET_SIGNED_ARTIFACTS_PATH}/*",
220-
credentialsId: env.INTERNAL_CI_JOB_GCS_CREDENTIALS,
221-
localDirectory: signaturesDestinationPath + '/',
222-
pathPrefix: "${env.INFRA_SIGNING_BUCKET_SIGNED_ARTIFACTS_SUBFOLDER}")
223-
sh(label: 'Rename .asc to .sig', script: 'for f in ' + signaturesDestinationPath + '/*.asc; do mv "$f" "${f%.asc}.sig"; done')
224-
}
225-
226150
def generateTestCheckSinglePackageStage(Map args = [:]) {
227151
def artifacts = ['build/test-results/*.xml', 'build/elastic-stack-dump/check-*/logs/*.log', 'build/elastic-stack-dump/check-*/logs/fleet-server-internal/*']
228152

‎.ci/package-storage-publish.groovy‎

Lines changed: 119 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,25 @@ pipeline {
66
agent { label 'ubuntu-20 && immutable' }
77
environment {
88
REPO = "elastic-package"
9+
REPO_BUILD_TAG = "${env.REPO}/${env.BUILD_TAG}"
910

1011
BASE_DIR="src/github.com/elastic/elastic-package"
1112
JOB_GIT_CREDENTIALS = "f6c7695a-671e-4f4f-a331-acdce44ff9ba"
1213
GITHUB_TOKEN_CREDENTIALS = "2a9602aa-ab9f-4e52-baf3-b71ca88469c7"
1314
PIPELINE_LOG_LEVEL='INFO'
15+
16+
// Signing
17+
JOB_SIGNING_CREDENTIALS = 'sign-artifacts-with-gpg-job'
18+
INFRA_SIGNING_BUCKET_NAME = 'internal-ci-artifacts'
19+
INFRA_SIGNING_BUCKET_SIGNED_ARTIFACTS_SUBFOLDER = "${env.REPO_BUILD_TAG}/signed-artifacts"
20+
INFRA_SIGNING_BUCKET_ARTIFACTS_PATH = "gs://${env.INFRA_SIGNING_BUCKET_NAME}/${env.REPO_BUILD_TAG}"
21+
INFRA_SIGNING_BUCKET_SIGNED_ARTIFACTS_PATH = "gs://${env.INFRA_SIGNING_BUCKET_NAME}/${env.INFRA_SIGNING_BUCKET_SIGNED_ARTIFACTS_SUBFOLDER}"
22+
23+
// Publishing
24+
INTERNAL_CI_JOB_GCS_CREDENTIALS = 'internal-ci-gcs-plugin'
25+
PACKAGE_STORAGE_UPLOADER_CREDENTIALS = 'upload-package-to-package-storage'
26+
PACKAGE_STORAGE_UPLOADER_GCP_SERVICE_ACCOUNT = 'secret/gce/elastic-bekitzur/service-account/package-storage-uploader'
27+
PACKAGE_STORAGE_INTERNAL_BUCKET_QUEUE_PUBLISHING_PATH = "gs://elastic-bekitzur-package-storage-internal/queue-publishing/${env.REPO_BUILD_TAG}"
1428
}
1529
options {
1630
timeout(time: 1, unit: 'HOURS')
@@ -31,7 +45,30 @@ pipeline {
3145
pipelineManager([ cancelPreviousRunningBuilds: [ when: 'PR' ] ])
3246
deleteDir()
3347
gitCheckout(basedir: "${BASE_DIR}")
34-
stash allowEmpty: true, name: 'source', useDefaultExcludes: false
48+
stash(allowEmpty: true, name: 'source', useDefaultExcludes: false)
49+
}
50+
}
51+
stage('Build package') {
52+
steps {
53+
cleanup()
54+
withGoEnv() {
55+
dir("${BASE_DIR}") {
56+
sh(label: 'Install elastic-package',script: "make install")
57+
// sh(label: 'Install elastic-package', script: 'go build github.com/elastic/elastic-package')
58+
dir("test/packages/package-storage/package_storage_candidate") {
59+
sh(label: 'Build package', script: "elastic-package build -v --zip")
60+
}
61+
}
62+
}
63+
stash(allowEmpty: true, name: 'build-package', includes: "${BASE_DIR}/build/integrations/*.zip", useDefaultExcludes: false)
64+
}
65+
}
66+
stage('Sign and publish package') {
67+
steps {
68+
cleanup(source: 'build-package')
69+
dir("${BASE_DIR}") {
70+
packageStoragePublish('build/integrations')
71+
}
3572
}
3673
}
3774
}
@@ -42,9 +79,88 @@ pipeline {
4279
}
4380
}
4481

45-
def cleanup(){
82+
def packageStoragePublish(builtPackagesPath) {
83+
signUnpublishedArtifactsWithElastic(builtPackagesPath)
84+
uploadUnpublishedToPackageStorage(builtPackagesPath)
85+
}
86+
87+
def signUnpublishedArtifactsWithElastic(builtPackagesPath) {
88+
def unpublished = false
89+
dir(builtPackagesPath) {
90+
findFiles()?.findAll{ it.name.endsWith('.zip') }?.collect{ it.name }?.sort()?.each {
91+
def packageZip = it
92+
if (isAlreadyPublished(packageZip)) {
93+
return
94+
}
95+
96+
unpublished = true
97+
googleStorageUpload(bucket: env.INFRA_SIGNING_BUCKET_ARTIFACTS_PATH,
98+
credentialsId: env.INTERNAL_CI_JOB_GCS_CREDENTIALS,
99+
pattern: '*.zip',
100+
sharedPublicly: false,
101+
showInline: true)
102+
}
103+
}
104+
105+
if (!unpublished) {
106+
return
107+
}
108+
109+
withCredentials([string(credentialsId: env.JOB_SIGNING_CREDENTIALS, variable: 'TOKEN')]) {
110+
triggerRemoteJob(auth: CredentialsAuth(credentials: 'local-readonly-api-token'),
111+
job: 'https://internal-ci.elastic.co/job/elastic+unified-release+master+sign-artifacts-with-gpg',
112+
token: TOKEN,
113+
parameters: "gcs_input_path=${env.INFRA_SIGNING_BUCKET_ARTIFACTS_PATH}",
114+
useCrumbCache: false,
115+
useJobInfoCache: false)
116+
}
117+
googleStorageDownload(bucketUri: "${env.INFRA_SIGNING_BUCKET_SIGNED_ARTIFACTS_PATH}/*",
118+
credentialsId: env.INTERNAL_CI_JOB_GCS_CREDENTIALS,
119+
localDirectory: builtPackagesPath + '/',
120+
pathPrefix: "${env.INFRA_SIGNING_BUCKET_SIGNED_ARTIFACTS_SUBFOLDER}")
121+
sh(label: 'Rename .asc to .sig', script: 'for f in ' + builtPackagesPath + '/*.asc; do mv "$f" "${f%.asc}.sig"; done')
122+
}
123+
124+
def uploadUnpublishedToPackageStorage(builtPackagesPath) {
125+
dir(builtPackagesPath) {
126+
withGCPEnv(secret: env.PACKAGE_STORAGE_UPLOADER_GCP_SERVICE_ACCOUNT) {
127+
withCredentials([string(credentialsId: env.PACKAGE_STORAGE_UPLOADER_CREDENTIALS, variable: 'TOKEN')]) {
128+
findFiles()?.findAll{ it.name.endsWith('.zip') }?.collect{ it.name }?.sort()?.each {
129+
def packageZip = it
130+
if (isAlreadyPublished(packageZip)) {
131+
return
132+
}
133+
134+
sh(label: 'Upload package .zip file', script: "gsutil cp ${packageZip} ${env.PACKAGE_STORAGE_INTERNAL_BUCKET_QUEUE_PUBLISHING_PATH}/")
135+
sh(label: 'Upload package .sig file', script: "gsutil cp ${packageZip}.sig ${env.PACKAGE_STORAGE_INTERNAL_BUCKET_QUEUE_PUBLISHING_PATH}/")
136+
137+
triggerRemoteJob(auth: CredentialsAuth(credentials: 'local-readonly-api-token'),
138+
job: 'https://internal-ci.elastic.co/job/package_storage/job/publishing-job-remote',
139+
token: TOKEN,
140+
parameters: """
141+
dry_run=true
142+
gs_package_build_zip_path=${env.PACKAGE_STORAGE_INTERNAL_BUCKET_QUEUE_PUBLISHING_PATH}/${packageZip}
143+
gs_package_signature_path=${env.PACKAGE_STORAGE_INTERNAL_BUCKET_QUEUE_PUBLISHING_PATH}/${packageZip}.sig
144+
""",
145+
useCrumbCache: true,
146+
useJobInfoCache: true)
147+
}
148+
}
149+
}
150+
}
151+
}
152+
153+
def isAlreadyPublished(packageZip) {
154+
def responseCode = httpRequest(method: "HEAD",
155+
url: "https://package-storage.elastic.co/artifacts/packages/${packageZip}",
156+
response_code_only: true)
157+
return responseCode == 200
158+
}
159+
160+
def cleanup(Map args = [:]) {
161+
def source = args.containsKey('source') ? args.source : 'source'
46162
dir("${BASE_DIR}"){
47163
deleteDir()
48164
}
49-
unstash 'source'
165+
unstash source
50166
}

0 commit comments

Comments
 (0)