Merge pull request #1016 from gursewak1997/add-gc-cloud-prune

jobs/garbage-collection: Add Garbage Collection job to the pipeline

Author: gursewak1997

gc-policy.yaml

@@ -0,0 +1,4 @@
+branched:
+    cloud-uploads: 3y
+rawhide:
+    cloud-uploads: 3y

jobs/garbage-collection.Jenkinsfile

@@ -0,0 +1,100 @@
+node {
+    checkout scm
+    // these are script global vars
+    pipeutils = load("utils.groovy")
+    pipecfg = pipeutils.load_pipecfg()
+    gc_policy_data = pipeutils.load_gc()
+    repo = "coreos/fedora-coreos-config"
+}
+
+// This job runs the Garbage collection on the selected stream in accordance with the
+// gc-policy.yaml available in https://github.com/coreos/fedora-coreos-pipeline. If
+// the pruning step succeeds, it uploads the updated builds/builds.json to s3.
+properties([
+    pipelineTriggers([]),
+    parameters([
+        choice(name: 'STREAM',
+               choices: pipeutils.get_streams_choices(pipecfg),
+               description: 'CoreOS stream to run GC'),
+        booleanParam(name: 'DRY_RUN',
+                     defaultValue: true,
+                     description: 'Only print what would be deleted')
+    ])
+])
+
+def build_description = "[${params.STREAM}]"
+def cosa_img = 'quay.io/coreos-assembler/coreos-assembler:main'
+def container_env = pipeutils.get_env_vars_for_stream(pipecfg, params.STREAM)
+def s3_stream_dir = pipeutils.get_s3_streams_dir(pipecfg, params.STREAM)
+def dry_run = params.DRY_RUN ? "--dry-run" : ""
+
+lock(resource: "gc-${params.STREAM}") {
+    cosaPod(image: cosa_img, env: container_env,
+            memory: "1024Mi",
+            serviceAccount: "jenkins") {
+        try {
+            currentBuild.description = "${build_description} Running"
+
+            stage('Init') {
+                def branch = params.STREAM
+                sh "cosa init --branch \"${branch}\" https://github.com/${repo}"
+            }
+
+            // Write YAML data to a new file in cosaPod
+            def new_gc_policy_path = 'tmp/gc-policy.yaml'
+            writeYaml file: new_gc_policy_path, data: gc_policy_data
+
+            stage('BuildFetch') {
+                pipeutils.shwrapWithAWSBuildUploadCredentials("""
+                cosa buildfetch --aws-config-file \${AWS_BUILD_UPLOAD_CONFIG} \
+                    --stream ${params.STREAM} --url=s3://${s3_stream_dir}/builds
+                """)
+            }
+            def originalBuildsJson = readJSON file: 'builds/builds.json'
+            def originalTimestamp = originalBuildsJson.timestamp
+
+            withCredentials([file(variable: 'GCP_KOLA_TESTS_CONFIG', credentialsId: 'gcp-image-upload-config')]) {
+                def acl = pipecfg.s3.acl ?: 'public-read'
+
+                stage('Garbage Collection') {
+                    pipeutils.shwrapWithAWSBuildUploadCredentials("""
+                    cosa cloud-prune --policy ${new_gc_policy_path} \
+                    --stream ${params.STREAM} ${dry_run} \
+                    --gcp-json-key=\${GCP_KOLA_TESTS_CONFIG} \
+                    --acl=${acl} \
+                    --aws-config-file \${AWS_BUILD_UPLOAD_CONFIG}
+                    """)
+                }
+            }
+
+            def currentBuildsJson = readJSON file: 'builds/builds.json'
+            def currentTimestamp = currentBuildsJson.timestamp
+
+            // If the timestamp on builds.json after the 'Garbage Collection' step
+            // is the same as before, that means, there were no resources to be pruned
+            // and hence, no need to update the builds.json.
+            if (originalTimestamp != currentTimestamp) {
+                // Nested lock for the Upload Builds JSON step
+                lock(resource: "builds-json-${params.STREAM}") {
+                    stage('Upload Builds JSON') {
+                        pipeutils.shwrapWithAWSBuildUploadCredentials("""
+                        cosa cloud-prune --policy ${new_gc_policy_path} \
+                        --stream ${params.STREAM} \
+                        --upload-builds-json ${dry_run} \
+                        --aws-config-file \${AWS_BUILD_UPLOAD_CONFIG}
+                        """)
+                    }
+                }
+            }
+            currentBuild.result = 'SUCCESS'
+
+        } catch (e) {
+            currentBuild.result = 'FAILURE'
+            throw e
+        } finally {
+            if (currentBuild.result != 'SUCCESS') {
+                pipeutils.trySlackSend(message: ":wastebasket: garbage-collection #${env.BUILD_NUMBER} <${env.BUILD_URL}|:jenkins:> <${env.RUN_DISPLAY_URL}|:ocean:> [${params.STREAM}]")
+            }
+        }
+    }
+}

utils.groovy

@@ -94,6 +94,21 @@ def validate_pipecfg(pipecfg, is_hotfix) {
     }
 }
 
+def load_gc() {
+    def jenkinscfg = load_jenkins_config()
+    def url = jenkinscfg['pipecfg-url']
+    def gc_policy_data
+
+    if (url == 'in-tree') {
+        gc_policy_data = readYaml(file: "gc-policy.yaml")
+    } else {
+        // assume the user called `load_pipecfg()` in this workdir; if not, let error out
+        gc_policy_data = readYaml(file: "pipecfg/gc-policy.yaml")
+    }
+
+    return gc_policy_data
+}
+
 def add_hotfix_parameters_if_supported() {
     def supported = true
     if (env.JENKINS_URL in PROTECTED_JENKINSES) {
@@ -202,7 +217,7 @@ def bump_builds_json(stream, buildid, arch, s3_stream_dir, acl) {
     // unlock
     //
     // XXX: should fold this into `cosa buildupload` somehow
-    lock(resource: "bump-builds-json-${stream}") {
+    lock(resource: "builds-json-${stream}") {
         def remotejson = "s3://${s3_stream_dir}/builds/builds.json"
         aws_s3_cp_allow_noent(remotejson, './remote-builds.json')
         shwrap("""