Updated helm deploy task to extract and push deployment metadata to e… (#11327)

* Updated helm deploy task to extract and push deployment metadata to evidence store

* Handling manifest url as per build source

* Handling file path when configuration file path is provided and arguments are null

* Review comments

Author: dabasnidhi

Tasks/Common/kubernetes-common-v2/image-metadata-helper.ts

@@ -1,33 +1,42 @@
 import * as tl from "azure-pipelines-task-lib/task";
 import * as util from "util";
+import * as yaml from 'js-yaml';
 
 const matchPatternForImageName = new RegExp(/\:\/\/(.+?)\@/);
 const matchPatternForDigest = new RegExp(/\@sha256\:(.+)/);
 const matchPatternForFileArgument = new RegExp(/-f\s|-filename\s/);
 const matchPatternForServerUrl = new RegExp(/https\:\/\/(.+)/);
+const matchPatternForSource = new RegExp(/source:(.+)/ig);
+const matchPatternForChartPath = new RegExp(/chart path:(.+)/i);
 const orgUrl = tl.getVariable('System.TeamFoundationCollectionUri');
 const build = "build";
 const hostType = tl.getVariable("System.HostType").toLowerCase();
 const isBuild = hostType === build;
 const deploymentTypes: string[] = ["deployment", "replicaset", "daemonset", "pod", "statefulset"];
+const workingDirectory = tl.getVariable("System.DefaultWorkingDirectory");
+const branch = tl.getVariable("Build.SourceBranchName") || tl.getVariable("Build.SourceBranch");
+const repositoryProvider = tl.getVariable("Build.Repository.Provider");
+const repositoryUrl = tl.getVariable("Build.Repository.Uri");
 
 // ToDo: Add UTs for public methods
-export function getDeploymentMetadata(deploymentObject: any, allPods: any, deploymentStrategy: string, clusterInfo: any, manifestFilePaths?: string[]): any {
+export function getDeploymentMetadata(deploymentObject: any, allPods: any, deploymentStrategy: string, clusterInfo: any, manifestUrls: string[]): any {
     let imageIds: string[] = [];
+    let containers = [];
     let kind: string = deploymentObject.kind;
     try {
         if (isPodEntity(kind)) {
-            imageIds = getImageIdsForPod(deploymentObject);
+            containers = deploymentObject.spec.containers;
         }
         else {
-            let containers = deploymentObject.spec.template.spec.containers;
-            if (containers && containers.length > 0) {
-                containers.forEach(container => {
-                    // Filter all pods using the container names in this deployment,
-                    // and get the imageIds from pod status
-                    imageIds = getImageIdsForPodsInDeployment(container.name, allPods.items);
-                });
-            }
+            containers = deploymentObject.spec.template.spec.containers;
+        }
+
+        if (containers && containers.length > 0) {
+            containers.forEach(container => {
+                // Filter all pods using the container names in this deployment,
+                // and get the imageIds from pod status
+                imageIds = getImageIdsForPodsInDeployment(container.name, allPods.items);
+            });
         }
     }
     catch (e) {
@@ -42,14 +51,8 @@ export function getDeploymentMetadata(deploymentObject: any, allPods: any, deplo
         relatedUrls.push(clusterUrl);
     }
 
-    if (manifestFilePaths) {
-        relatedUrls.push(...manifestFilePaths);
-    }
-    else {
-        let manifestPaths = getManifestFilePaths();
-        if (manifestPaths.length > 0) {
-            relatedUrls.push(...manifestPaths);
-        }
+    if (manifestUrls.length > 0) {
+        relatedUrls.push(...manifestUrls);
     }
 
     const metadataDetails = {
@@ -192,19 +195,130 @@ function getServerUrl(clusterInfo: any): string {
     return serverUrl;
 }
 
-function getManifestFilePaths(): string[] {
-    let manifestFilePaths: string[] = [];
-    const commandArguments = tl.getInput("arguments", false);
-    const filePathMatch: string[] = commandArguments.split(matchPatternForFileArgument);
-    if (filePathMatch && filePathMatch.length >= 0) {
+export function extractManifestsFromHelmOutput(helmOutput: string): any {
+    let manifestObjects = [];
+    let manifestFiles = "";
+    // The output stream contains the manifest file between the manifest and last deployed fields
+    const manifestString = "manifest:";
+    const lastDeployedString = "last deployed:";
+    let indexOfManifests = helmOutput.toLowerCase().indexOf(manifestString);
+    let indexOfLastDeployed = helmOutput.toLowerCase().indexOf(lastDeployedString);
+    if (indexOfManifests >= 0 && indexOfLastDeployed >= 0) {
+        manifestFiles = helmOutput.substring(indexOfManifests + manifestString.length, indexOfLastDeployed);
+    }
+
+    if (manifestFiles) {
+        // Each of the source manifests is separated in output stream via string '---'
+        const files = manifestFiles.split("---");
+        files.forEach(file => {
+            file = file.trim();
+            if (file) {
+                const parsedObject = yaml.safeLoad(file);
+                manifestObjects.push(parsedObject);
+            }
+        });
+    }
+
+    return manifestObjects;
+}
+
+export function getManifestFileUrlsFromArgumentsInput(fileArgs: string): string[] {
+    let manifestFileUrls: string[] = [];
+    const filePathMatch: string[] = fileArgs.split(matchPatternForFileArgument);
+    if (filePathMatch && filePathMatch.length > 0) {
         filePathMatch.forEach(manifestPath => {
             if (!!manifestPath) {
-                manifestFilePaths.push(manifestPath.trim())
+                if (manifestPath.startsWith("http") || manifestPath.startsWith("https:")) {
+                    manifestFileUrls.push(manifestPath);
+                }
+                else {
+                    manifestFileUrls.push(...getManifestUrls([manifestPath]));
+                }
             }
         });
     }
 
-    return manifestFilePaths;
+    return manifestFileUrls;
+}
+
+export function getManifestFileUrlsFromHelmOutput(helmOutput: string): string[] {
+    const chartType = tl.getInput("chartType", true);
+    // Raw github links are supported only for chart names not chart paths
+    if (chartType === "Name") {
+        const chartName = tl.getInput("chartName", true);
+        if (chartName.startsWith("http:") || chartName.startsWith("https:")) {
+            return [chartName];
+        }
+    }
+
+    let manifestFilePaths: string[] = [];
+    // Extract the chart directory
+    const directoryName = getChartDirectoryName(helmOutput);
+    // Extract all source paths; source path example - # Source: MyChart/templates/pod.yaml
+    const filePathMatches = helmOutput.match(matchPatternForSource);
+    if (filePathMatches && filePathMatches.length >= 1) {
+        filePathMatches.forEach(filePathMatch => {
+            // Strip the Chart name from source path to get the template path
+            let indexOfTemplate = filePathMatch.toLowerCase().indexOf("templates");
+            const templatePath = indexOfTemplate >= 0 ? filePathMatch.substr(indexOfTemplate) : filePathMatch;
+            manifestFilePaths.push(directoryName + "/" + templatePath.trim());
+        });
+    }
+
+    return getManifestUrls(manifestFilePaths);
+}
+
+export function getChartDirectoryName(helmOutput: string): string {
+    // The output contains chart path in the following format - CHART PATH: C:\agent\_work\2\s\helm-chart-directory
+    let directoryName = "";
+    const chartPathMatch = helmOutput.match(matchPatternForChartPath);
+    if (chartPathMatch && chartPathMatch.length >= 1) {
+        let fullPath = chartPathMatch[1];
+        let indexOfLastSeparator = fullPath.lastIndexOf("\\");
+        directoryName = indexOfLastSeparator >= 0 ? fullPath.substr(indexOfLastSeparator + 1) : fullPath;
+    }
+
+    return directoryName;
+}
+
+export function getManifestUrls(manifestFilePaths: string[]): string[] {
+    let manifestUrls = [];
+    const branchName = getBranchName(branch);
+    for (const path of manifestFilePaths) {
+        let manifestUrl = "";
+        let normalisedPath = path.indexOf(workingDirectory) === 0 ? path.substr(workingDirectory.length) : path;
+        normalisedPath = normalisedPath.replace(/\\/g, "/");
+
+        if (repositoryProvider && (repositoryProvider.toLowerCase() === "githubenterprise" || repositoryProvider.toLowerCase() === "github")) {
+            if (normalisedPath.indexOf("/") != 0) {
+                // Prepend "/" if not present in path beginning as the path is appended as it is in manifest url to access github repo
+                normalisedPath = "/" + normalisedPath;
+            }
+
+            manifestUrl = repositoryUrl + "/blob/" + branchName + normalisedPath;
+        }
+        else if (repositoryProvider && repositoryProvider.toLowerCase() === "tfsgit") {
+            if (normalisedPath.indexOf("/") === 0) {
+                // Remove "/" from path if present in the beginning as we need to append path as a query string in manifest url to access tfs repo
+                normalisedPath = normalisedPath.substr(1);
+            }
+
+            manifestUrl = repositoryUrl + "?path=" + normalisedPath;
+        }
+
+        manifestUrls.push(manifestUrl);
+    }
+
+    return manifestUrls;
+}
+
+function getBranchName(ref: string): string {
+    const gitRefsHeadsPrefix = "refs/heads/";
+    if (ref && ref.indexOf(gitRefsHeadsPrefix) === 0) {
+        return ref.substr(gitRefsHeadsPrefix.length);
+    }
+
+    return ref;
 }
 
 function getPlatform(): string {

Tasks/HelmDeployV0/make.json

@@ -16,6 +16,12 @@
 		"type": "node",
 		"dest" : "./",
 		"compile" : true
+	},
+    {
+		"module": "../Common/utility-common-v2",
+		"type": "node",
+		"dest" : "./",
+		"compile" : true
 	}
 ],
 	"rm": [