ROX-15523: Copy expiration to cloud resources (GKE)

chart/infra-server/static/workflow-gke-default.yaml

@@ -2,8 +2,15 @@ apiVersion: argoproj.io/v1alpha1
 kind: Workflow
 metadata:
   generateName: gke-default-
+  labels:
+    needsExit: "true"
+  annotations:
+    infra.stackrox.com/lifespan: 0s
+    infra.stackrox.com/flavor: unknown
+    infra.stackrox.com/owner: unknown
 spec:
   entrypoint: start
+  onExit: stop
   arguments:
     parameters:
       - name: name
@@ -27,15 +34,127 @@ spec:
     - name: start
       steps:
         - - name: create
-            template: create
+            template: echo
+            arguments:
+              parameters:
+                - name: message
+                  value: "create"
         - - name: wait
-            template: wait
+            template: running
+            arguments:
+              parameters:
+                - name: lifespan
+                  value: "{{workflow.annotations.infra.stackrox.com/lifespan}}"
+
+    - name: stop
+      steps:
         - - name: destroy
-            template: destroy
+            template: echo
+            arguments:
+              parameters:
+                - name: message
+                  value: "{{workflow.parameters.name}}"
+
+    - name: running
+      inputs:
+        parameters:
+          - name: lifespan
+      failFast: false
+      steps:
+        - - name: label
+            template: label
+            arguments:
+              parameters:
+                - name: lifespan
+                  value: "{{inputs.parameters.lifespan}}"
+        - - name: wait
+            template: wait
+            arguments:
+              parameters:
+                - name: lifespan
+                  value: "{{inputs.parameters.lifespan}}"
+
+    - name: wait
+      inputs:
+        parameters:
+          - name: lifespan
+      steps:
+        - - name: delay
+            inline:
+              suspend:
+                duration: "1"
+        - - name: echo
+            template: echo
+            arguments:
+              parameters:
+                - name: message
+                  value: "'{{inputs.parameters.lifespan}}' =~ '{{workflow.annotations.infra.stackrox.com/lifespan}}'"
+        - - name: loop
+            template: wait
+            when: "'{{inputs.parameters.lifespan}}' =~ '{{workflow.annotations.infra.stackrox.com/lifespan}}'"
+            arguments:
+              parameters:
+                - name: lifespan
+                  value: "{{inputs.parameters.lifespan}}"  # If we send the annotation, it could change between the 'when' check and sending.
+        - - name: break
+            template: running
+            arguments:
+              parameters:
+                - name: lifespan
+                  value: "{{workflow.annotations.infra.stackrox.com/lifespan}}"
+
+    - name: label
+      inputs:
+        parameters:
+          - name: lifespan
+      steps:
+        - - name: label
+            template: label-gke
             arguments:
               parameters:
-                - name: name
-                  value: "{{steps.create.outputs.parameters.cluster_name}}"
+              - name: labels
+                value: "\
+                  flavor={{workflow.annotations.infra.stackrox.com/flavor}},\
+                  owner={{workflow.annotations.infra.stackrox.com/owner}},\
+                  lifespan={{inputs.parameters.lifespan}},\
+                  expiration={{= sprig.date('2006-01-02T15:04:05Z07:00',\
+                    sprig.dateModify(inputs.parameters.lifespan,\
+                    sprig.toDate('2006-01-02T15:04:05Z07:00',\
+                      workflow.creationTimestamp.RFC3339)))\
+                  }}"
+
+    - name: label-gke
+      inputs:
+        parameters:
+        - name: labels
+      script:
+        image: quay.io/stackrox-io/ci:automation-flavors-gke-default-0.7.6-3-g1ce65fe441-snapshot
+        imagePullPolicy: Always
+        env:
+          - name: GOOGLE_APPLICATION_CREDENTIALS
+            value: /tmp/google-credentials.json
+        command: [bash, -x]
+        source: |
+          labels="{{inputs.parameters.labels}}"
+          gcloud auth activate-service-account --key-file /tmp/google-credentials.json;
+          gcloud auth list;
+          gcloud config set compute/zone "{{workflow.parameters.gcp-zone}}";
+          gcloud config set core/disable_prompts True;
+          gcloud container clusters update \
+            "{{workflow.parameters.name}}"\
+            --project=srox-temp-dev-test\
+            "--update-labels=${labels}" || true
+        volumeMounts:
+          - name: credentials
+            mountPath: /tmp
+
+    - name: echo
+      inputs:
+        parameters:
+        - name: message
+      container:
+        image: alpine:3.7
+        command: [echo, "{{inputs.parameters.message}}"]
 
     - name: create
       activeDeadlineSeconds: 3600
@@ -79,9 +198,6 @@ spec:
           - name: credentials
             mountPath: /tmp
 
-    - name: wait
-      suspend: {}
-
     - name: destroy
       activeDeadlineSeconds: 3600
       container:

service/cluster/cluster.go

@@ -528,6 +528,21 @@ func (s *clusterImpl) Delete(ctx context.Context, req *v1.ResourceByID) (*empty.
 			"error", err,
 		)
 	}
+	if workflow.Spec.HasExitHook() {
+		log.Infow("stopping argo workflow", "workflow-name", workflow.GetName())
+		_, err = s.argoWorkflowsClient.StopWorkflow(s.argoClientCtx, &workflowpkg.WorkflowStopRequest{
+			Name:              workflow.GetName(),
+			Namespace:         s.workflowNamespace,
+			NodeFieldSelector: "",
+			Message:           "Destroying cluster. End workflow loop.",
+		})
+		if err != nil {
+			log.Warnw("failed to stop workflow, this is OK if the workflow is not running",
+				"workflow-name", req.GetId(),
+				"error", err,
+			)
+		}
+	}
 
 	return &empty.Empty{}, nil
 }
@@ -638,13 +653,30 @@ func (s *clusterImpl) cleanupExpiredClusters() {
 				continue
 			}
 
-			log.Infow("resuming an argo workflow that has expired", "workflow-name", workflow.GetName())
-			_, err = s.argoWorkflowsClient.ResumeWorkflow(s.argoClientCtx, &workflowpkg.WorkflowResumeRequest{
-				Name:      workflow.GetName(),
-				Namespace: s.workflowNamespace,
-			})
-			if err != nil {
-				log.Warnw("failed to resume argo workflow", "workflow-name", workflow.GetName(), "error", err)
+			if workflow.Spec.Suspend != nil && *workflow.Spec.Suspend {
+				log.Infow("resuming an argo workflow that has expired", "workflow-name", workflow.GetName())
+				_, err = s.argoWorkflowsClient.ResumeWorkflow(s.argoClientCtx, &workflowpkg.WorkflowResumeRequest{
+					Name:      workflow.GetName(),
+					Namespace: s.workflowNamespace,
+				})
+				if err != nil {
+					log.Warnw("failed to resume argo workflow", "workflow-name", workflow.GetName(), "error", err)
+				}
+			}
+			if workflow.Spec.HasExitHook() {
+				log.Infow("stopping argo workflow with exit hook", "workflow-name", workflow.GetName())
+				_, err = s.argoWorkflowsClient.StopWorkflow(s.argoClientCtx, &workflowpkg.WorkflowStopRequest{
+					Name:              workflow.GetName(),
+					Namespace:         s.workflowNamespace,
+					NodeFieldSelector: "",
+					Message:           "Destroying cluster. End workflow loop.",
+				})
+				if err != nil {
+					log.Warnw("failed to stop workflow, this is OK if the workflow is not running",
+						"workflow-name", workflow.GetName(),
+						"error", err,
+					)
+				}
 			}
 		}
 

service/cluster/helpers.go

@@ -204,18 +204,27 @@ func workflowStatus(workflowStatus v1alpha1.WorkflowStatus) v1.Status {
 				}
 			} else if node.Type == v1alpha1.NodeTypeSuspend {
 				switch node.Phase {
-				case v1alpha1.NodeSucceeded:
-					return v1.Status_DESTROYING
 				case v1alpha1.NodeError, v1alpha1.NodeFailed, v1alpha1.NodeSkipped:
 					panic("a suspend should not be able to fail?")
 				case v1alpha1.NodeRunning, v1alpha1.NodePending:
 					return v1.Status_READY
 				}
 			}
+			if node.GetName() == "destroy" || node.IsExitNode() {
+				return v1.Status_DESTROYING
+			}
+			if node.GetName() == "create" {
+				switch node.Phase {
+				case v1alpha1.NodeError, v1alpha1.NodeFailed, v1alpha1.NodeSkipped:
+					return v1.Status_FAILED
+				case v1alpha1.NodeRunning, v1alpha1.NodePending:
+					return v1.Status_CREATING
+				}
+			}
 		}
 
-		// No suspend node was found, which means one hasn't been run yet, which means that this cluster is still creating.
-		return v1.Status_CREATING
+		// If no "create" or "destroy"/onExit node active, then we're ready.
+		return v1.Status_READY
 
 	case "":
 		return v1.Status_CREATING