[AzureRG Deploy] File encoding check before parsing (#3749)

* Detecting encoding of file before parsing

* refactoring code

* bug fix and message fixes

* Fixing messages

Author: thesattiraju

Tasks/AzureResourceGroupDeployment/Strings/resources.resjson/en-US/resources.resjson

@@ -158,5 +158,8 @@
   "loc.messages.DeletingMGAgentOnVMs": "Deleting machine group agent on virtual machines",
   "loc.messages.AddingExtensionFailed": "Addition of extension on vm %s failed",
   "loc.messages.TimeoutWhileWaiting": "Timed out while waiting",
-  "loc.messages.InvalidTemplateLocation": "The template location supplied is invalid. Task only supports 'Linked artifact' or 'URL of the file'"
+  "loc.messages.InvalidTemplateLocation": "The template location supplied is invalid. Task only supports 'Linked artifact' or 'URL of the file'",
+  "loc.messages.EncodingNotSupported": "Encoding of the file '%s' is '%s' which is not supported. Supported encodings are ['utf-8', 'utf-16le']",
+  "loc.messages.CouldNotDetectEncoding": "Could not detect encoding of file '%s'",
+  "loc.messages.ShortFileBufferError": "Short file buffer error on file '%s'"
 }

Tasks/AzureResourceGroupDeployment/operations/FileEncoding.ts

@@ -0,0 +1,97 @@
+//File Encoding detected to be : utf-32be, which is not supported by Node.js
+//'Unable to detect encoding of file ' + typeCode
+//'File buffer is too short to detect encoding type'
+var fs = require('fs');
+import tl = require('vsts-task-lib');
+
+export class FileEncoding {
+    public type: string;
+    public usesBOM: boolean;
+    constructor(type: string, usesBOM: boolean) {
+        this.type = type;
+        this.usesBOM = usesBOM;
+    }
+}
+
+function detectFileEncodingWithBOM(fileName: string, buffer: Buffer) {
+    tl.debug('Detecting file encoding using BOM');
+    var type: string;
+    if (buffer.slice(0, 3).equals(new Buffer([239, 187, 191]))) {
+        type = 'utf-8';
+    }
+    else if (buffer.slice(0, 4).equals(new Buffer([255, 254, 0, 0]))) {
+        type = 'UTF-32LE';
+    }
+    else if (buffer.slice(0, 2).equals(new Buffer([254, 255]))) {
+        type = 'UTF-16BE';
+    }
+    else if (buffer.slice(0, 2).equals(new Buffer([255, 254]))) {
+        type = 'utf-16le';
+    }
+    else if (buffer.slice(0, 4).equals(new Buffer([0, 0, 254, 255]))) {
+        type = 'UTF-32BE';
+    }
+    else {
+        tl.debug('Unable to detect File encoding using BOM');
+        return null;
+    }
+    return new FileEncoding(type, true);
+}
+
+function detectFileEncodingWithoutBOM(fileName: string, buffer: Buffer) {
+    tl.debug('Detecting file encoding without BOM');
+    var typeCode = 0;
+    var type: string;
+    for (var index = 0; index < 4; index++) {
+        typeCode = typeCode << 1;
+        typeCode = typeCode | (buffer[index] > 0 ? 1 : 0);
+    }
+    switch (typeCode) {
+        case 1:
+            type = 'UTF-32BE';
+            break;
+        case 5:
+            type = 'UTF-16BE';
+            break;
+        case 8:
+            type = 'UTF-32LE';
+            break;
+        case 10:
+            type = 'utf-16le';
+            break;
+        case 15:
+            type = 'utf-8';
+            break;
+        default:
+            return null;
+    }
+    return new FileEncoding(type, false);
+}
+export function detectFileEncoding(fileName: string, buffer: Buffer): FileEncoding {
+    if (buffer.length < 4) {
+        tl.debug(tl.loc('ShortFileBufferError', fileName))
+        throw Error(tl.loc("CouldNotDetectEncoding", fileName));
+    }
+    var fileEncoding: FileEncoding = detectFileEncodingWithBOM(fileName, buffer);
+    if (fileEncoding == null)
+        fileEncoding = detectFileEncodingWithoutBOM(fileName, buffer);
+
+    if (fileEncoding == null) {
+        throw new Error(tl.loc("CouldNotDetectEncoding", fileName));
+    }
+    return fileEncoding;
+}
+
+export function readFileContentsAsText(fileName: string): string {
+    var buffer = fs.readFileSync(fileName);
+    var supportedFileEncodings = ["utf-8", "utf-16le"]
+    var fileEncoding = detectFileEncoding(fileName, buffer);
+    if (supportedFileEncodings.indexOf(fileEncoding.type) < 0) {
+        throw new Error(tl.loc('EncodingNotSupported', fileName, fileEncoding.type));
+    }
+    var fileContents: string = buffer.toString(fileEncoding.type);
+    if (fileEncoding.usesBOM) {
+        fileContents = fileContents.slice(1);
+    }
+    return fileContents;
+}

Tasks/AzureResourceGroupDeployment/operations/ResourceGroup.ts

@@ -15,6 +15,7 @@ import winRM = require("./WinRMExtensionHelper");
 import mgExtensionHelper = require("./MachineGroupExtensionHelper");
 var parameterParser = require("./ParameterParser").parse;
 import utils = require("./Utils");
+import fileEncoding = require('./FileEncoding');
 
 var httpClient = require('vso-node-api/HttpClient');
 var httpObj = new httpClient.HttpCallbackClient("VSTS_AGENT");
@@ -190,7 +191,7 @@ export class ResourceGroup {
         var template: Object;
         try {
             tl.debug("Loading CSM Template File.. " + this.taskParameters.csmFile);
-            template = JSON.parse(fs.readFileSync(this.taskParameters.csmFile, 'UTF-8'));
+            template = JSON.parse(fileEncoding.readFileContentsAsText(this.taskParameters.csmFile));
             tl.debug("Loaded CSM File");
         }
         catch (error) {
@@ -202,9 +203,9 @@ export class ResourceGroup {
             if (utils.isNonEmpty(this.taskParameters.csmParametersFile)) {
                 if (!fs.lstatSync(this.taskParameters.csmParametersFile).isDirectory()) {
                     tl.debug("Loading Parameters File.. " + this.taskParameters.csmParametersFile);
-                    var parameterFile = fs.readFileSync(this.taskParameters.csmParametersFile, 'UTF-8');
+                    var parameterFile = JSON.parse(fileEncoding.readFileContentsAsText(this.taskParameters.csmParametersFile));
                     tl.debug("Loaded Parameters File");
-                    parameters = JSON.parse(parameterFile).parameters;
+                    parameters = parameterFile["parameters"];
                 }
             }
         }

Tasks/AzureResourceGroupDeployment/task.json

@@ -14,7 +14,7 @@
     "version": {
         "Major": 2,
         "Minor": 0,
-        "Patch": 3
+        "Patch": 4
     },
     "demands": [],
     "minimumAgentVersion": "2.0.0",
@@ -367,6 +367,9 @@
         "DeletingMGAgentOnVMs": "Deleting machine group agent on virtual machines",
         "AddingExtensionFailed": "Addition of extension on vm %s failed",
         "TimeoutWhileWaiting": "Timed out while waiting",
-        "InvalidTemplateLocation": "The template location supplied is invalid. Task only supports 'Linked artifact' or 'URL of the file'"
+        "InvalidTemplateLocation": "The template location supplied is invalid. Task only supports 'Linked artifact' or 'URL of the file'",
+        "EncodingNotSupported": "Encoding of the file '%s' is '%s' which is not supported. Supported encodings are ['utf-8', 'utf-16le']",
+        "CouldNotDetectEncoding": "Could not detect encoding of file '%s'",
+        "ShortFileBufferError": "Short file buffer error on file '%s'"
     }
 }

Tasks/AzureResourceGroupDeployment/task.loc.json

@@ -14,7 +14,7 @@
   "version": {
     "Major": 2,
     "Minor": 0,
-    "Patch": 3
+    "Patch": 4
   },
   "demands": [],
   "minimumAgentVersion": "2.0.0",
@@ -367,6 +367,9 @@
     "DeletingMGAgentOnVMs": "ms-resource:loc.messages.DeletingMGAgentOnVMs",
     "AddingExtensionFailed": "ms-resource:loc.messages.AddingExtensionFailed",
     "TimeoutWhileWaiting": "ms-resource:loc.messages.TimeoutWhileWaiting",
-    "InvalidTemplateLocation": "ms-resource:loc.messages.InvalidTemplateLocation"
+    "InvalidTemplateLocation": "ms-resource:loc.messages.InvalidTemplateLocation",
+    "EncodingNotSupported": "ms-resource:loc.messages.EncodingNotSupported",
+    "CouldNotDetectEncoding": "ms-resource:loc.messages.CouldNotDetectEncoding",
+    "ShortFileBufferError": "ms-resource:loc.messages.ShortFileBufferError"
   }
 }