update release scripts from master

SylvainJuge · SylvainJuge · commit 60155ab98989 · 2021-12-21T16:17:36.000+01:00
diff --git a/.ci/jobs/apm-agent-java-mbp.yml b/.ci/jobs/apm-agent-java-mbp.yml
@@ -42,4 +42,4 @@
             reference-repo: /var/lib/jenkins/.git-references/apm-agent-java.git
             use-author: true
             wipe-workspace: true
-    periodic-folder-trigger: 1h
+    periodic-folder-trigger: 4h
diff --git a/.ci/release/Jenkinsfile b/.ci/release/Jenkinsfile
@@ -2,7 +2,7 @@
 
 @Library('apm@current') _
 pipeline {
-  agent { label 'linux && immutable' }
+  agent { label 'ubuntu && linux-immutable-docker' }
   environment {
     REPO = 'apm-agent-java'
     BASE_DIR = "src/github.com/elastic/${env.REPO}"
@@ -23,7 +23,6 @@ pipeline {
     timestamps()
     ansiColor('xterm')
     durabilityHint('PERFORMANCE_OPTIMIZED')
-
   }
   parameters {
     string(name: 'branch_specifier', defaultValue: 'stable', description: "What branch to release from?")
@@ -45,17 +44,9 @@ pipeline {
             dir("${BASE_DIR}") {
               setupAPMGitEmail()
             }
-
-            dir("${BASE_DIR}") {
-              script {
-                env.TAG_BARE = sh(script: "./scripts/jenkins/tags.sh bare", returnStdout: true)
-                env.TAG_VER = sh(script: "./scripts/jenkins/tags.sh ver", returnStdout: true)
-                env.TAG_DOT_X = sh(script: "./scripts/jenkins/tags.sh dot_x", returnStdout: true)
-              }
-            }
             dir("${BASE_DIR}") {
-              withEnv(["BRANCH_NAME=tags/${env.TAG_BARE}"]){
-                withGitRelease() {
+              withEnv(["BRANCH_NAME=${BRANCH_SPECIFIER}"]){
+                withGitRelease() { // sets up the git release setup (remote branches, current workspace state, ...)
                   stash(allowEmpty: true, name: 'source', useDefaultExcludes: false)
                 }
               }
@@ -88,12 +79,14 @@ pipeline {
           when {
             allOf {
               expression { params.check_branch_ci_status }
-              not { branch 'stable' }
+              expression { env.BRANCH_SPECIFIER != 'stable' }
             }
           }
           steps {
             // If this build is not green: https://apm-ci.elastic.co/job/apm-agent-java/job/apm-agent-java-mbp/job/"${BRANCH_SPECIFIER}"/
             whenTrue(!buildStatus(host: 'apm-ci.elastic.co', job: ['apm-agent-java', 'apm-agent-java-mbp', "${BRANCH_SPECIFIER}"], return_boolean: true)) {
+              notifyStatus(slackStatus: 'warning', subject: "[${env.REPO}] The ${BRANCH_SPECIFIER} build is not passing",
+                           body: "Please go to (<${env.BUILD_URL}input|here>) to approve or reject within 12 hours.")
               input(message: "WARNING! The ${BRANCH_SPECIFIER} build is not passing. Do you wish to continue?")
             }
           }
@@ -125,6 +118,7 @@ pipeline {
                     defaultValue: "${release_version}",
                     description: "Current project version is ${snapshot_version}, will be released as ${release_version} if unchanged. Input release version without '-SNAPSHOT' suffix"
                   ]])
+
                 if( release_version.equals(user_release_version) ) {
                     echo "changing project version '${snapshot_version}' not required to release ${release_version}"
                 } else {
@@ -133,6 +127,10 @@ pipeline {
                     sh(script: "git commit -a -m 'Version bump ${user_release_version}'")
                     gitPush()
                 }
+
+                env.RELEASE_TAG = "v" + user_release_version
+                env.RELEASE_VERSION = user_release_version
+                env.BRANCH_DOT_X = user_release_version.substring(0, user_release_version.indexOf('.'))+'.x'
               }
             }
           }
@@ -146,42 +144,39 @@ pipeline {
         }
         stage('Nexus release') {
           steps {
-            script {
-              dir("${BASE_DIR}"){
-                withSecretVault(secret: 'secret/apm-team/ci/nexus', user_var_name: '__unused__', pass_var_name: 'SPID', role_id: 'apm-vault-role-id', secret_id: 'apm-vault-secret-id'){
-                  def foundStagingId = nexusFindStagingId(stagingProfileId: "${SPID}", groupId: "co.elastic.apm")
-                  nexusCloseStagingRepository(stagingProfileId: "${SPID}", stagingId: foundStagingId)
-                  nexusReleaseStagingRepository(stagingProfileId: "${SPID}", stagingId: foundStagingId)
-                }
-              }
-            }
+            notifyStatus(slackStatus: 'warning', subject: "[${env.REPO}] Release ready to be published in Nexus",
+                         body: "Please go to (<https://oss.sonatype.org/|here>) to proceed with the manual nexus release. Login details in LastPass")
+            input(message: "Go to https://oss.sonatype.org and proceed with the steps to close and release the staging artifact.")
           }
         }
-        stage('Branch creation') {
+        stage('Major Branch create/update') {
           steps {
             dir("${BASE_DIR}") {
               script {
-                def curVer = mvnVersion()
-                sh(script: "./scripts/jenkins/branch_creation.sh ${curVer}")
-                def isMajor = sh(script: "./scripts/jenkins/is_major.sh ${curVer}", returnStatus: true)
-                if (isMajor == 0) {
+                sh(script: ".ci/release/update_major_branch.sh ${RELEASE_VERSION}")
+                gitPush(args: "-f ${BRANCH_DOT_X}")
+
+                def isMajor = env.RELEASE_VERSION.endsWith(".0.0")
+                if (isMajor) {
                   input message: "This was a major version release. Please update the conf.yml in the docs repo before continuing", ok "Continue"
                 }
               }
-              gitPush()
             }
           }
         }
         stage('Create GitHub release draft') {
           steps {
             dir("${BASE_DIR}"){
               script {
-                def curVer = mvnVersion(showQualifiers: false)
                 // Construct the URL with anchor for the release notes
                 // Ex: https://www.elastic.co/guide/en/apm/agent/java/current/release-notes-1.x.html#release-notes-1.13.0
-                def finalUrl = sh(script: "scripts/jenkins/generate_release_notes_url.sh ${curVer}", returnStdout: true)
+                def finalUrl = "https://www.elastic.co/guide/en/apm/agent/java/current/release-notes-${BRANCH_DOT_X}.html#release-notes-${RELEASE_VERSION}"
                 githubEnv()
-                def ret = githubReleaseCreate(draft: true, name: "Release ${curVer}", body: "[Release Notes for ${curVer}](${finalUrl})")
+                def ret = githubReleaseCreate(
+                        draft: true,
+                        tagName: "${RELEASE_TAG}",
+                        releaseName: "Release ${RELEASE_VERSION}",
+                        body: "[Release Notes for ${RELEASE_VERSION}](${finalUrl})")
                 env.RELEASE_ID = ret['id']
                 env.RELEASE_NOTES_URL = finalUrl
               }
@@ -192,12 +187,9 @@ pipeline {
           steps {
             dir("${BASE_DIR}"){
               script {
-                def fullUrl = sh(script: "./scripts/jenkins/maven_artifact_url.sh ")
-                echo "Sleeping for ten minutes to wait for Sonatype"
-                sleep(time: 10, unit: "MINUTES")
                 waitUntil(initialRecurrencePeriod: 60000) {
                   script {
-                    def ret = sh(script: "curl -fs $fullUrl >/dev/null 2>&1", returnStatus: true)
+                    def ret = sh(script: ".ci/release/wait_maven_artifact_published.sh ${RELEASE_VERSION}", returnStatus: true)
                     echo "Waiting for the artifacts to be published on Sonatype"
                     return ret == 0
                   }
@@ -209,38 +201,41 @@ pipeline {
         stage('Update Cloudfoundry') {
           steps {
             dir("${BASE_DIR}"){
-              sh(script: "./scripts/jenkins/update_cloudfoundry.sh")
+              sh(script: ".ci/release/update_cloudfoundry.sh ${RELEASE_VERSION}")
               gitPush()
             }
           }
         }
         stage('Publish release on GitHub') {
           steps {
-            waitUntil(initialRecurrencePeriod: 60000) {
-              dir("${BASE_DIR}"){
+            dir("${BASE_DIR}"){
+              waitUntil(initialRecurrencePeriod: 60000) {
                 script {
                   echo "Waiting for the release notes to be available"
-                  def ret = sh(script: "./scripts/jenkins/docs_check.sh $TAG_VER")
-                  return ret == 0
+                    def ret = sh(script: ".ci/release/wait_release_notes_published.sh ${RELEASE_VERSION}", returnStatus: true)
+                    return ret == 0
                 }
               }
+              githubEnv()
+              githubReleasePublish(id: "${env.RELEASE_ID}", name: "Release ${RELEASE_VERSION}")
             }
-            githubEnv()
-            githubReleasePublish(id: "${env.RELEASE_ID}")
           }
         }
         stage('Build and push Docker images') {
           steps {
             dir("${BASE_DIR}"){
-              sh(label: "Build Docker image", script: "./scripts/jenkins/build_docker.sh")
-              // Get Docker registry credentials
-              dockerLogin(secret: "${ELASTIC_DOCKER_SECRET}", registry: 'docker.elastic.co')
-              sh(label: "Push Docker image", script: "./scripts/jenkins/push_docker.sh")
+              // fetch agent artifact from remote repository
+              withEnv(["SONATYPE_FALLBACK=1"]) {
+                sh(label: "Build Docker image", script: "./scripts/jenkins/build_docker.sh")
+                  // Get Docker registry credentials
+                  dockerLogin(secret: "${ELASTIC_DOCKER_SECRET}", registry: 'docker.elastic.co', role_id: 'apm-vault-role-id', secret_id: 'apm-vault-secret-id')
+                  sh(label: "Push Docker image", script: "./scripts/jenkins/push_docker.sh")
+              }
             }
           }
           post {
             success {
-              notifyStatus(slackStatus: 'good', subject: "[${env.REPO}] Release published", body: "Great news! The release has completed successfully. (<${env.RUN_DISPLAY_URL}|Open>).")
+              notifyStatus(slackStatus: 'good', subject: "[${env.REPO}] Release published", body: "Great news! The release ${env.RELEASE_VERSION} has completed successfully. (<${env.RUN_DISPLAY_URL}|Open>).")
             }
           }
         }
diff --git a/.ci/release/update_cloudfoundry.sh b/.ci/release/update_cloudfoundry.sh
@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+source "$(dirname "${0}")/util.sh"
+
+REMOTE_NAME=origin
+BRANCH_NAME=master
+BASE_URL=https://repo1.maven.org/maven2/co/elastic/apm/elastic-apm-agent
+CF_FILE=cloudfoundry/index.yml
+
+check_version "${1:-}"
+v="${1:-}"
+
+echo -e "\n--- fetch & ensure clean state of ${REMOTE_NAME}/${BRANCH_NAME}"
+git fetch ${REMOTE_NAME} ${BRANCH_NAME}
+git checkout ${BRANCH_NAME}
+git reset --hard ${REMOTE_NAME}/${BRANCH_NAME}
+
+echo -e "\n--- update ${CF_FILE} if required"
+
+# make script idempotent if release is already in CF descriptor
+set +e
+grep -e "^${v}:" ${CF_FILE}
+[[ $? == 0 ]] && exit 0
+set -e
+
+echo "${v}: ${BASE_URL}/${v}/elastic-apm-agent-${v}.jar" >> ${CF_FILE}
+git commit ${CF_FILE} -m "Update cloudfoundry for ${v} release"
diff --git a/.ci/release/update_major_branch.sh b/.ci/release/update_major_branch.sh
@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+source "$(dirname "${0}")/util.sh"
+
+REMOTE_NAME=origin
+
+# Final state (when there is no error): local checkout in the major branch '1.x' that matches the release tag
+# Pushing to remote repository is managed by the caller
+
+check_version "${1:-}"
+v="${1:-}"
+
+tag=$(version_tag ${v})
+major_branch="$(version_major_branch "${v}")"
+
+echo ""
+echo "release version: ${v}"
+echo "release tag:     ${tag}"
+echo "major branch:    ${major_branch}"
+
+check_ref_exists () {
+  set +e
+  git show-ref --verify --quiet "${1}" \
+    && echo 1 \
+    || echo 0
+  set -e
+}
+
+
+# shallow clone does not allow to use other branches by default
+# found in https://stackoverflow.com/questions/23708231/git-shallow-clone-clone-depth-misses-remote-branches
+git remote set-branches ${REMOTE_NAME} '*'
+
+echo -e "\n--- delete any local ${major_branch} branch if it exists"
+set +e
+git rev-parse --verify ${major_branch} 2>/dev/null && git branch -D ${major_branch}
+set -e
+
+echo -e "\n--- ensure remote tag ${tag} and branch ${major_branch} are fetched for local checkout"
+git fetch --tags ${REMOTE_NAME}
+
+if [[ '1' != $(check_ref_exists "refs/tags/${tag}") ]]; then
+  echo "ERROR remote tag '${tag}' does not exists on remote '${REMOTE_NAME}'"
+  exit 1
+fi
+
+checkout_options=''
+
+echo -e "\n--- checkout or create ${major_branch} branch"
+if [[ "0" == "$(check_ref_exists "refs/remotes/${REMOTE_NAME}/${major_branch}")" ]]; then
+  # remote branch does not exists, creating new local branch
+  checkout_options="-b ${major_branch}"
+else
+  # remote branch exists, create local branch to update it
+  checkout_options="--track ${REMOTE_NAME}/${major_branch}"
+fi
+set -e
+
+echo -e "\n--- checkout local ${major_branch} branch"
+git checkout --force ${checkout_options}
+
+echo -e "\n--- move local branch ${major_branch} to match tag ${tag}"
+git reset --hard ${tag}
diff --git a/.ci/release/util.sh b/.ci/release/util.sh
@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+check_version() {
+  v=${1:-}
+
+  if [[ "${v}" == "" ]]; then
+    echo "usage $0 <version>" # here $0 will be the calling script
+    echo "where <version> in format '1.2.3'"
+    exit 1
+  fi
+
+  if [[ ! "$v" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+    echo "invalid version format '${v}'"
+    exit 1
+  fi
+}
+
+version_tag() {
+    echo "v${1:?usage version_tag <version>}"
+}
+
+version_major_branch() {
+  v="${1:?usage version_major_branch <version>}"
+  echo "${v%%.*}.x"
+}
diff --git a/.ci/release/wait_maven_artifact_published.sh b/.ci/release/wait_maven_artifact_published.sh
@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+source "$(dirname "${0}")/util.sh"
+
+# returns '0' (zero) status code when artifact exists
+# return non-zero status when artifact does not exists or unable to execute request
+
+check_version "${1:-}"
+v="${1:-}"
+
+full_url="https://repo1.maven.org/maven2/co/elastic/apm/elastic-apm-agent/${v}/elastic-apm-agent-${v}.pom"
+curl -fs "${full_url}" 2>&1 > /dev/null
diff --git a/.ci/release/wait_release_notes_published.sh b/.ci/release/wait_release_notes_published.sh
@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+source "$(dirname "${0}")/util.sh"
+
+# returns '0' (zero) status code when release notes have been published
+# return non-zero status when they haven't been published yet or unable to execute request
+
+check_version "${1:-}"
+v="${1:-}"
+major_branch="$(version_major_branch "${v}")"
+
+full_url="https://www.elastic.co/guide/en/apm/agent/java/current/release-notes-${major_branch}.html"
+curl -fs "${full_url}" | grep "${v}"
