added wait for service external Ip assignment (#11325)

* service wait for ip

* updated task.json

* code refactoring

* addressed review comments

* addressing review comments

* some more review comments

* added test for service IP

Author: anuragc617

Tasks/KubernetesManifestV0/Strings/resources.resjson/en-US/resources.resjson

@@ -74,5 +74,9 @@
   "loc.messages.InvalidPromotetActionDeploymentStrategy": "Promote action works only with strategy: canary",
   "loc.messages.AllContainersNotInReadyState": "All the containers are not in a ready state.",
   "loc.messages.CouldNotDeterminePodStatus": "Could not determine the pod's status due to the error: %s",
-  "loc.messages.KubectlShouldBeUpgraded": "kubectl client version equal to v1.14 or higher is required to use kustomize features."
+  "loc.messages.KubectlShouldBeUpgraded": "kubectl client version equal to v1.14 or higher is required to use kustomize features.",
+  "loc.messages.CouldNotDetermineServiceStatus": "Could not determine the service %s status due to the error: %s",
+  "loc.messages.waitForServiceIpAssignment": "Waiting for service %s external IP assignment",
+  "loc.messages.waitForServiceIpAssignmentTimedOut": "Wait for service %s external IP assignment timed out",
+  "loc.messages.ServiceExternalIP": "service %s external IP is %s"
 }

Tasks/KubernetesManifestV0/Tests/L0.ts

@@ -45,6 +45,7 @@ describe('Kubernetes Manifests Suite', function () {
         process.env[shared.TestEnvVars.imagePullSecrets] = 'test-key1\ntest-key2';
         tr.run();
         assert(tr.succeeded, 'task should have succeeded');
+        assert(tr.stdout.indexOf('nginx-service 104.211.243.77') != -1, 'nginx-service external IP is 104.211.243.77')
         done();
     });
 

Tasks/KubernetesManifestV0/Tests/TestSetup.ts

@@ -247,6 +247,11 @@ a.exec[`${kubectlPath} scale ${process.env[shared.TestEnvVars.kind]}/${process.e
     stdout: 'created secret'
 }
 
+a.exec[`${kubectlPath} get service/nginx-service -o json --namespace testnamespace`] = {
+    'code': 0,
+    'stdout': '{\r\n     "apiVersion": "v1",\r\n     "kind": "Service",\r\n     "metadata": {\r\n         "annotations": {\r\n             "azure-pipelines/jobName": "Agent phase",\r\n             "azure-pipelines/org": "https://codedev.ms/anchauh/",\r\n             "azure-pipelines/pipeline": "aksCd-153 - 64 - CD",\r\n             "azure-pipelines/pipelineId": "40",\r\n             "azure-pipelines/project": "nginx",\r\n             "azure-pipelines/run": "41",\r\n             "azure-pipelines/runuri": "https://codedev.ms/anchauh/nginx/_releaseProgress?releaseId=41",\r\n             "kubectl.kubernetes.io/last-applied-configuration": "{\\"apiVersion\\":\\"v1\\",\\"kind\\":\\"Service\\",\\"metadata\\":{\\"annotations\\":{},\\"labels\\":{\\"app\\":\\"nginx\\"},\\"name\\":\\"nginx-service\\",\\"namespace\\":\\"testnamespace\\"},\\"spec\\":{\\"ports\\":[{\\"name\\":\\"http\\",\\"port\\":80,\\"protocol\\":\\"TCP\\",\\"targetPort\\":\\"http\\"}],\\"selector\\":{\\"app\\":\\"nginx\\"},\\"type\\":\\"LoadBalancer\\"}}\\n"\r\n         },\r\n         "creationTimestamp": "2019-09-11T10:09:09Z",\r\n         "labels": {\r\n             "app": "nginx"\r\n         },\r\n         "name": "nginx-service",\r\n         "namespace": "testnamespace",\r\n         "resourceVersion": "8754335",\r\n         "selfLink": "/api/v1/namespaces/testnamespace/services/nginx-service",\r\n         "uid": "31f02713-d47c-11e9-9448-16b93c17a2b4"\r\n     },\r\n     "spec": {\r\n         "clusterIP": "10.0.157.189",\r\n         "externalTrafficPolicy": "Cluster",\r\n         "ports": [\r\n             {\r\n                 "name": "http",\r\n                 "nodePort": 32112,\r\n                 "port": 80,\r\n                 "protocol": "TCP",\r\n                 "targetPort": "http"\r\n             }\r\n         ],\r\n         "selector": {\r\n             "app": "nginx"\r\n         },\r\n         "sessionAffinity": "***",\r\n         "type": "LoadBalancer"\r\n     },\r\n     "status": {\r\n         "loadBalancer": {\r\n             "ingress": [\r\n                 {\r\n                     "ip": "104.211.243.77"\r\n                 }\r\n             ]\r\n         }\r\n     }\r\n }'
+}
+
 const pipelineAnnotations: string = [
     `azure-pipelines/run=${buildNumber}`,
     `azure-pipelines/pipeline="${definitionName}"`,
@@ -282,6 +287,7 @@ if (process.env[shared.TestEnvVars.arguments]) {
     };
 }
 
+
 tr.setAnswers(<any>a);
 tr.registerMock('azure-pipelines-task-lib/toolrunner', require('azure-pipelines-task-lib/mock-toolrunner'));
 
@@ -324,6 +330,7 @@ tr.registerMock('../utils/FileHelper', {
         });
 
         if (newFilePaths.length === 0) {
+            console.log(shared.ManifestFilesPath);
             newFilePaths.push(shared.ManifestFilesPath);
         }
         return newFilePaths;

Tasks/KubernetesManifestV0/Tests/manifests/deployment.yaml

@@ -20,4 +20,22 @@ spec:
         ports:
         - containerPort: 80
       imagePullSecrets:
-      - name: key1
+      - name: key1
+
+---
+
+apiVersion: v1
+kind: Service
+metadata:
+  name: nginx-service
+  labels:
+    app: nginx
+spec:
+  type: LoadBalancer
+  ports:
+    - port: 80
+      targetPort: http
+      protocol: TCP
+      name: http
+  selector:
+    app: nginx

Tasks/KubernetesManifestV0/src/models/constants.ts

@@ -13,6 +13,17 @@ export class KubernetesWorkload {
     public static cronjob: string = 'cronjob';
 }
 
+export class DiscoveryAndLoadBalancerResource {
+    public static service: string = 'service';
+    public static ingress: string = 'ingress';
+}
+
+export class ServiceTypes {
+    public static loadBalancer: string = 'LoadBalancer';
+    public static nodePort: string = 'NodePort';
+    public static clusterIP: string = 'ClusterIP'
+}
+
 export const deploymentTypes: string[] = ['deployment', 'replicaset', 'daemonset', 'pod', 'statefulset'];
 export const workloadTypes: string[] = ['deployment', 'replicaset', 'daemonset', 'pod', 'statefulset', 'job', 'cronjob'];
 export const workloadTypesWithRolloutStatus: string[] = ['deployment', 'daemonset', 'statefulset'];

Tasks/KubernetesManifestV0/src/utils/DeploymentHelper.ts

@@ -34,9 +34,15 @@ export async function deploy(kubectl: Kubectl, manifestFilePaths: string[], depl
     const deployedManifestFiles = deployManifests(inputManifestFiles, kubectl, isCanaryDeploymentStrategy(deploymentStrategy));
 
     // check manifest stability
-    const resourceTypes: Resource[] = KubernetesObjectUtility.getResources(deployedManifestFiles, models.deploymentTypes);
+    const resourceTypes: Resource[] = KubernetesObjectUtility.getResources(deployedManifestFiles, models.deploymentTypes.concat([constants.DiscoveryAndLoadBalancerResource.service]));
     await checkManifestStability(kubectl, resourceTypes);
 
+    // print ingress resources
+    const ingressResources: Resource[] = KubernetesObjectUtility.getResources(deployedManifestFiles, [constants.DiscoveryAndLoadBalancerResource.ingress]);
+    ingressResources.forEach(ingressResource => {
+        kubectl.getResource(constants.DiscoveryAndLoadBalancerResource.ingress, ingressResource.name);
+    });
+
     // annotate resources
     const allPods = JSON.parse((kubectl.getAllPods()).stdout);
     annotateResources(deployedManifestFiles, kubectl, resourceTypes, allPods);
@@ -92,6 +98,21 @@ async function checkManifestStability(kubectl: Kubectl, resources: Resource[]):
                 tl.warning(tl.loc('CouldNotDeterminePodStatus', JSON.stringify(ex)));
             }
         }
+        if (isEqual(resource.type, constants.DiscoveryAndLoadBalancerResource.service, StringComparer.OrdinalIgnoreCase)) {
+            try {
+                const service = getService(kubectl, resource.name);
+                const spec = service.spec;
+                const status = service.status;
+                if (isEqual(spec.type, constants.ServiceTypes.loadBalancer, StringComparer.OrdinalIgnoreCase)) {
+                    if(!isLoadBalancerIPAssigned(status)) {
+                        await waitForServiceExternalIPAssignment(kubectl, resource.name);
+                    }
+                    console.log(tl.loc('ServiceExternalIP', resource.name, status.loadBalancer.ingress[0].ip));
+                }
+            } catch (ex) {
+                tl.warning(tl.loc('CouldNotDetermineServiceStatus', resource.name, JSON.stringify(ex)));
+            }
+        }
     }
     utils.checkForErrors(rolloutStatusResults);
 }
@@ -263,6 +284,34 @@ function isPodReady(podStatus: any): boolean {
     return allContainersAreReady;
 }
 
+function getService(kubectl: Kubectl, serviceName) {
+    const serviceResult = kubectl.getResource(constants.DiscoveryAndLoadBalancerResource.service, serviceName);
+    utils.checkForErrors([serviceResult]);
+    return JSON.parse(serviceResult.stdout);
+}
+
+async function waitForServiceExternalIPAssignment(kubectl: Kubectl, serviceName: string): Promise<void> {
+    const sleepTimeout = 10 * 1000; // 10 seconds
+    const iterations = 18; // 18 * 10 seconds timeout = 3 minutes max timeout
+
+    for (let i = 0; i < iterations; i++) {
+        console.log(tl.loc('waitForServiceIpAssignment', serviceName));
+        await sleep(sleepTimeout);
+        let status = getService(kubectl, serviceName).status;
+        if (isLoadBalancerIPAssigned(status)) {
+            return;
+        }
+    }
+    tl.warning(tl.loc('waitForServiceIpAssignmentTimedOut', serviceName));
+}
+
+function isLoadBalancerIPAssigned(status: any) {
+    if (status && status.loadBalancer && status.loadBalancer.ingress && status.loadBalancer.ingress.length > 0) {
+        return true;
+    }
+    return false;
+}
+
 function sleep(timeout: number) {
     return new Promise(resolve => setTimeout(resolve, timeout));
 }

Tasks/KubernetesManifestV0/task.json

@@ -333,6 +333,10 @@
         "InvalidPromotetActionDeploymentStrategy": "Promote action works only with strategy: canary",
         "AllContainersNotInReadyState": "All the containers are not in a ready state.",
         "CouldNotDeterminePodStatus": "Could not determine the pod's status due to the error: %s",
-        "KubectlShouldBeUpgraded": "kubectl client version equal to v1.14 or higher is required to use kustomize features."
+        "KubectlShouldBeUpgraded": "kubectl client version equal to v1.14 or higher is required to use kustomize features.",
+        "CouldNotDetermineServiceStatus":  "Could not determine the service %s status due to the error: %s",
+        "waitForServiceIpAssignment": "Waiting for service %s external IP assignment",
+        "waitForServiceIpAssignmentTimedOut": "Wait for service %s external IP assignment timed out",
+        "ServiceExternalIP": "service %s external IP is %s"
     }
 }

Tasks/KubernetesManifestV0/task.loc.json

@@ -333,6 +333,10 @@
     "InvalidPromotetActionDeploymentStrategy": "ms-resource:loc.messages.InvalidPromotetActionDeploymentStrategy",
     "AllContainersNotInReadyState": "ms-resource:loc.messages.AllContainersNotInReadyState",
     "CouldNotDeterminePodStatus": "ms-resource:loc.messages.CouldNotDeterminePodStatus",
-    "KubectlShouldBeUpgraded": "ms-resource:loc.messages.KubectlShouldBeUpgraded"
+    "KubectlShouldBeUpgraded": "ms-resource:loc.messages.KubectlShouldBeUpgraded",
+    "CouldNotDetermineServiceStatus": "ms-resource:loc.messages.CouldNotDetermineServiceStatus",
+    "waitForServiceIpAssignment": "ms-resource:loc.messages.waitForServiceIpAssignment",
+    "waitForServiceIpAssignmentTimedOut": "ms-resource:loc.messages.waitForServiceIpAssignmentTimedOut",
+    "ServiceExternalIP": "ms-resource:loc.messages.ServiceExternalIP"
   }
 }