Hotfix for packer task

Author: bishal-pdMSFT

Tasks/PackerBuild/DefaultTemplates/custom.linux.template.json

@@ -13,7 +13,8 @@
     "script_arguments": "{{env `script_arguments`}}",
     "package_path": "{{env `package_path`}}",
     "package_name": "{{env `package_name`}}",
-    "capture_name_prefix": "{{env `capture_name_prefix`}}"
+    "capture_name_prefix": "{{env `capture_name_prefix`}}",
+    "skip_clean": "{{env `skip_clean`}}"
   },
   "builders": [{
         "type": "azure-arm",
@@ -62,6 +63,7 @@
         "execute_command": "chmod +x {{ .Path }}; {{ .Vars }} sudo -E sh '{{ .Path }}'",
         "inline": [
             "/usr/sbin/waagent -force -deprovision+user && export HISTSIZE=0 && sync"
-        ]
+        ],
+        "skip_clean": "{{user `skip_clean`}}"
     }]
 }

Tasks/PackerBuild/DefaultTemplates/default.linux.template.json

@@ -15,7 +15,8 @@
     "script_arguments": "",
     "package_path": "",
     "package_name": "",
-    "capture_name_prefix": ""
+    "capture_name_prefix": "",
+    "skip_clean":""
   },
   "builders": [{
         "type": "azure-arm",
@@ -66,6 +67,7 @@
         "execute_command": "chmod +x {{ .Path }}; {{ .Vars }} sudo -E sh '{{ .Path }}'",
         "inline": [
             "/usr/sbin/waagent -force -deprovision+user && export HISTSIZE=0 && sync"
-        ]
+        ],
+        "skip_clean": "{{user `skip_clean`}}"
     }]
 }

Tasks/PackerBuild/Strings/resources.resjson/en-US/resources.resjson

@@ -6,6 +6,7 @@
   "loc.releaseNotes": "- Works with Xplat agents (Windows, Linux or OSX)\n- Supports building images for both Windows and Linux. These images can be used to deploy Azure Virtual machine scale set.",
   "loc.group.displayName.AzureDetails": "Azure Details",
   "loc.group.displayName.DeploymentInputs": "Deployment Inputs",
+  "loc.group.displayName.Advanced": "Advanced",
   "loc.group.displayName.Output": "Output",
   "loc.input.label.templateType": "Packer template",
   "loc.input.help.templateType": "Select whether you want the task to auto generate Packer template or use custom template provided by you.",
@@ -32,6 +33,10 @@
   "loc.input.help.deployScriptPath": "Specify the relative path to powershell script(for Windows) or shell script(for Linux) which deploys the package. This script should be contained in the package path selected above. Supports minimatch pattern. Example path: deploy/**/scripts/windows/deploy.ps1",
   "loc.input.label.deployScriptArguments": "Deployment script arguments",
   "loc.input.help.deployScriptArguments": "Specify the arguments to be passed to deployment script.",
+  "loc.input.label.additionalBuilderParameters": "Additional Builder parameters",
+  "loc.input.help.additionalBuilderParameters": "In auto generated Packer template mode the task creates a Packer template with an Azure builder. This builder is used to generate a machine image. You can add keys to the Azure builder to customize the generated Packer template. For example setting ssh_tty=true in case you are using a CentOS base image and you need to have a tty to run sudo.<br/>However certain keys like builder type, access_key etc that are set by taking user input in the task cannot be overridden by adding keys here. To view/edit the additional parameters in a grid, click on “…” next to textbox.",
+  "loc.input.label.skipTempFileCleanupDuringVMDeprovision": "Skip temporary file cleanup during deprovision",
+  "loc.input.help.skipTempFileCleanupDuringVMDeprovision": "During deprovisioning of VM, skip clean-up of temporary files uploaded to VM. Refer [here](https://www.packer.io/docs/builders/azure.html#skip_clean)",
   "loc.input.label.imageUri": "Image URL",
   "loc.input.help.imageUri": "Provide a name for the output variable which will store generated machine image url.",
   "loc.messages.OSTypeNotSupported": "This OS type is not supported for image creation: %s. Please use either windows or linux.",
@@ -52,7 +57,7 @@
   "loc.messages.CustomTemplateNotFoundErrorMessagePathName": "Custom template not found at location: %s",
   "loc.messages.CouldNotDeleteTemporaryTemplateDirectory": "Could not delete temporary template directory %s. Please delete manually.",
   "loc.messages.TaskParametersConstructorFailed": "Error happened while initializing task: %s.",
-  "loc.messages.PackerFixFailed": "Packer fix command failed. This could happen if task does not support packer version.",
+  "loc.messages.PackerFixFailed": "Packer fix command failed with error : '%s'. This could happen if task does not support packer version.",
   "loc.messages.PackerValidateFailed": "Packer validate command failed. This could happen if task does not support packer version.",
   "loc.messages.ResolvedPathNotFound": "Not found any file matching pattern: %s under rool folder: %s.",
   "loc.messages.ResolvedDeployPackgePath": "Resolved deploy package path: %s.",
@@ -63,5 +68,11 @@
   "loc.messages.ExtractingPackerCompleted": "Packer extracted successfully at path: %s.",
   "loc.messages.CreatedStagingDirectory": "Created staging directory for keeping packer binary and templates: %s.",
   "loc.messages.CouldNotDeleteStagingDirectory": "Could not delete staging directory %s. Please delete manually.",
-  "loc.messages.InstalledPackerVersion": "Current installed packer version is %s."
+  "loc.messages.InstalledPackerVersion": "Current installed packer version is %s.",
+  "loc.messages.AdditionalParameterIgnored": "Additional parameter key '%s' already exists in builder '%s'. New value will be ignored.",
+  "loc.messages.PackerToolBusy": "Packer tool seems to be busy. Retrying after 1 second...",
+  "loc.messages.ResolvingDeployPackageInput": "Resolving deploy package path.",
+  "loc.messages.ResolvingDeployScriptInput": "Resolving deploy script path.",
+  "loc.messages.ParsingAdditionalBuilderParameters": "Parsing additional builder parameters json.",
+  "loc.messages.ParsingTemplateFileContentFailed": "Unable to parse json content from template file %s with error: %s."
 }

Tasks/PackerBuild/Tests/L0.ts

@@ -220,7 +220,8 @@ describe('PackerBuild Suite', function() {
             process.env["__packer_fix_fails__"] = "true";
             let tp = path.join(__dirname, 'L0WindowsFail.js');
             let tr : ttm.MockTestRunner = new ttm.MockTestRunner(tp);
-            tr.run();   
+            tr.run();
+            process.env["__packer_fix_fails__"] = "false";  
 
             assert(tr.failed, 'task should have failed');
             assert(tr.invokedToolCount == 2, 'should not invoke packer validate and build commands. actual: ' + tr.invokedToolCount);        
@@ -229,8 +230,7 @@ describe('PackerBuild Suite', function() {
             done();
         });
 
-        it('should fail if packer validate exits with non zero code', (done:MochaDone) => {
-            process.env["__packer_fix_fails__"] = "false";     
+        it('should fail if packer validate exits with non zero code', (done:MochaDone) => {   
             process.env["__packer_validate_fails__"] = "true";
             let tp = path.join(__dirname, 'L0WindowsFail.js');
             let tr : ttm.MockTestRunner = new ttm.MockTestRunner(tp);
@@ -354,7 +354,7 @@ describe('PackerBuild Suite', function() {
             let tr : ttm.MockTestRunner = new ttm.MockTestRunner(tp);
             tr.run();
 
-            assert(tr.invokedToolCount == 3, 'should have invoked tool thrice. actual: ' + tr.invokedToolCount);
+            assert(tr.invokedToolCount == 4, 'should have invoked tool 4 times. actual: ' + tr.invokedToolCount);
             assert(tr.stderr.length == 0 || tr.errorIssues.length, 'should not have written to stderr');
             assert(tr.succeeded, 'task should have succeeded');
             assert(tr.stdout.indexOf("loc_mock_DownloadingPackerRequired") != -1, "should show message that packer will be downloaded");
@@ -376,7 +376,7 @@ describe('PackerBuild Suite', function() {
             process.env["__packer_exists__"] = "false";
             process.env["__lower_version__"] = "false";
 
-            assert(tr.invokedToolCount == 4, 'should have invoked tool four times. actual: ' + tr.invokedToolCount);
+            assert(tr.invokedToolCount == 5, 'should have invoked tool 5 times. actual: ' + tr.invokedToolCount);
             assert(tr.stderr.length == 0 || tr.errorIssues.length, 'should not have written to stderr');
             assert(tr.succeeded, 'task should have succeeded');
             assert(tr.stdout.indexOf("loc_mock_DownloadingPackerRequired") != -1, "should show message that packer will be downloaded");
@@ -430,6 +430,17 @@ describe('PackerBuild Suite', function() {
             done();
         });
 
+        it('Should add additional parameters to builder section in builtin template', (done:MochaDone) => {
+            let tp = path.join(__dirname, 'L0WindowsBuiltinTemplateAdditionalParameters.js');
+            let tr : ttm.MockTestRunner = new ttm.MockTestRunner(tp);
+            tr.run();
+
+            assert(tr.stderr.length == 0 || tr.errorIssues.length, 'should not have written to stderr');
+            assert(tr.stdout.indexOf("writing to file F:\\somedir\\tempdir\\100\\default.windows.template-builderUpdated.json content: {\"builders\":[{\"type\":\"azure-arm\",\"ssh_pty\":\"true\"}]}") != -1, "additional parameters should be written to updated template file");
+            assert(tr.succeeded, 'task should have succeeded');
+            done();
+        });
+
     } else {
         it('Runs successfully for linux template', (done:MochaDone) => {
             let tp = path.join(__dirname, 'L0Linux.js');
@@ -575,5 +586,16 @@ describe('PackerBuild Suite', function() {
             assert(tr.stdout.indexOf("rmRF /tmp/tempdir/100") != -1, "rmRF should be called on temp template folder");
             done();
         });
+
+        it('Should add additional parameters to builder section in builtin template', (done:MochaDone) => {
+            let tp = path.join(__dirname, 'L0LinuxBuiltinTemplateAdditionalParameters.js');
+            let tr : ttm.MockTestRunner = new ttm.MockTestRunner(tp);
+            tr.run();
+
+            assert(tr.stderr.length == 0 || tr.errorIssues.length, 'should not have written to stderr');
+            assert(tr.stdout.indexOf("writing to file /tmp/tempdir/100/default.linux.template-builderUpdated.json content: {\"builders\":[{\"type\":\"azure-arm\",\"ssh_pty\":\"true\"}]}") != -1, "additional parameters should be written to updated template file");
+            assert(tr.succeeded, 'task should have succeeded');
+            done();
+        });
     }
 });

Tasks/PackerBuild/Tests/L0CustomTemplate.ts

@@ -9,6 +9,7 @@ tr.setInput('templateType', 'custom');
 tr.setInput('customTemplateLocation', 'C:\\custom.template.json');
 tr.setInput('imageUri', 'imageUri');
 tr.setInput('imageStorageAccount', 'imageStorageAccount');
+tr.setInput("additionalBuilderParameters", "{}");
 
 process.env["RELEASE_RELEASENAME"] = "Release-1";
 

Tasks/PackerBuild/Tests/L0Linux.ts

@@ -19,6 +19,8 @@ tr.setInput('deployScriptArguments', "\"subdir 1\" false");
 tr.setInput('ConnectedServiceName', 'AzureRMSpn');
 tr.setInput('imageUri', 'imageUri');
 tr.setInput('imageStorageAccount', 'imageStorageAccount');
+tr.setInput("additionalBuilderParameters", "{}");
+tr.setInput("skipTempFileCleanupDuringVMDeprovision", "true");
 
 process.env["ENDPOINT_AUTH_SCHEME_AzureRMSpn"] = "ServicePrincipal";
 process.env["ENDPOINT_AUTH_PARAMETER_AzureRMSpn_SERVICEPRINCIPALID"] = "spId";
@@ -49,23 +51,23 @@ let a: any = <any>{
             "code": 0,
             "stdout": "{ \"some-key\": \"some-value\" }"
         },
-        "packer validate -var resource_group=testrg -var storage_account=teststorage -var image_publisher=Canonical -var image_offer=UbuntuServer -var image_sku=14.04.4-LTS -var location=South India -var capture_name_prefix=Release-1 -var script_relative_path=dir3/somedir/deploy.sh -var package_path=/tmp/dir1/somedir/dir2 -var package_name=dir2 -var script_arguments=\"subdir 1\" false -var subscription_id=sId -var client_id=spId -var client_secret=spKey -var tenant_id=tenant -var object_id=oId /tmp/tempdir/100/default.linux.template-fixed.json": {
+        "packer validate -var resource_group=testrg -var storage_account=teststorage -var image_publisher=Canonical -var image_offer=UbuntuServer -var image_sku=14.04.4-LTS -var location=South India -var capture_name_prefix=Release-1 -var skip_clean=true -var script_relative_path=dir3/somedir/deploy.sh -var package_path=/tmp/dir1/somedir/dir2 -var package_name=dir2 -var script_arguments=\"subdir 1\" false -var subscription_id=sId -var client_id=spId -var client_secret=spKey -var tenant_id=tenant -var object_id=oId /tmp/tempdir/100/default.linux.template-fixed.json": {
             "code": 0,
             "stdout": "Executed Successfully"
         },
-        "packer build -force -var resource_group=testrg -var storage_account=teststorage -var image_publisher=Canonical -var image_offer=UbuntuServer -var image_sku=14.04.4-LTS -var location=South India -var capture_name_prefix=Release-1 -var script_relative_path=dir3/somedir/deploy.sh -var package_path=/tmp/dir1/somedir/dir2 -var package_name=dir2 -var script_arguments=\"subdir 1\" false -var subscription_id=sId -var client_id=spId -var client_secret=spKey -var tenant_id=tenant -var object_id=oId /tmp/tempdir/100/default.linux.template-fixed.json": {
+        "packer build -force -var resource_group=testrg -var storage_account=teststorage -var image_publisher=Canonical -var image_offer=UbuntuServer -var image_sku=14.04.4-LTS -var location=South India -var capture_name_prefix=Release-1 -var skip_clean=true -var script_relative_path=dir3/somedir/deploy.sh -var package_path=/tmp/dir1/somedir/dir2 -var package_name=dir2 -var script_arguments=\"subdir 1\" false -var subscription_id=sId -var client_id=spId -var client_secret=spKey -var tenant_id=tenant -var object_id=oId /tmp/tempdir/100/default.linux.template-fixed.json": {
             "code": process.env["__packer_build_fails__"] === "true" ? 1 : 0,
             "stdout": process.env["__packer_build_fails__"] === "true" ? "packer build failed\r\nsome error" : "Executed Successfully\nOSDiskUri: https://bishalpackerimages.blob.core.windows.net/system/Microsoft.Compute/Images/packer/packer-osDisk.e2e08a75-2d73-49ad-97c2-77f8070b65f5.vhd\nStorageAccountLocation: SouthIndia",
         },
         "packer fix -validate=false \\tmp\\tempdir\\100\\default.linux.template.json": {
             "code": 0,
             "stdout": "Executed Successfully"
         },
-        "packer validate -var resource_group=testrg -var storage_account=teststorage -var image_publisher=Canonical -var image_offer=UbuntuServer -var image_sku=14.04.4-LTS -var location=South India -var capture_name_prefix=Release-1 -var script_relative_path=dir3/somedir/deploy.sh -var package_path=/tmp/dir1/somedir/dir2 -var package_name=dir2 -var script_arguments=\"subdir 1\" false -var subscription_id=sId -var client_id=spId -var client_secret=spKey -var tenant_id=tenant -var object_id=oId \\tmp\\tempdir\\100\\default.linux.template-fixed.json": {
+        "packer validate -var resource_group=testrg -var storage_account=teststorage -var image_publisher=Canonical -var image_offer=UbuntuServer -var image_sku=14.04.4-LTS -var location=South India -var capture_name_prefix=Release-1 -var skip_clean=true -var script_relative_path=dir3/somedir/deploy.sh -var package_path=/tmp/dir1/somedir/dir2 -var package_name=dir2 -var script_arguments=\"subdir 1\" false -var subscription_id=sId -var client_id=spId -var client_secret=spKey -var tenant_id=tenant -var object_id=oId \\tmp\\tempdir\\100\\default.linux.template-fixed.json": {
             "code": 0,
             "stdout": "Executed Successfully"
         },
-        "packer build -force -var resource_group=testrg -var storage_account=teststorage -var image_publisher=Canonical -var image_offer=UbuntuServer -var image_sku=14.04.4-LTS -var location=South India -var capture_name_prefix=Release-1 -var script_relative_path=dir3/somedir/deploy.sh -var package_path=/tmp/dir1/somedir/dir2 -var package_name=dir2 -var script_arguments=\"subdir 1\" false -var subscription_id=sId -var client_id=spId -var client_secret=spKey -var tenant_id=tenant -var object_id=oId \\tmp\\tempdir\\100\\default.linux.template-fixed.json": {
+        "packer build -force -var resource_group=testrg -var storage_account=teststorage -var image_publisher=Canonical -var image_offer=UbuntuServer -var image_sku=14.04.4-LTS -var location=South India -var capture_name_prefix=Release-1 -var skip_clean=true -var script_relative_path=dir3/somedir/deploy.sh -var package_path=/tmp/dir1/somedir/dir2 -var package_name=dir2 -var script_arguments=\"subdir 1\" false -var subscription_id=sId -var client_id=spId -var client_secret=spKey -var tenant_id=tenant -var object_id=oId \\tmp\\tempdir\\100\\default.linux.template-fixed.json": {
             "code": process.env["__packer_build_fails__"] === "true" ? 1 : 0,
             "stdout": process.env["__packer_build_fails__"] === "true" ? "packer build failed\r\nsome error" : "Executed Successfully\nOSDiskUri: https://bishalpackerimages.blob.core.windows.net/system/Microsoft.Compute/Images/packer/packer-osDisk.e2e08a75-2d73-49ad-97c2-77f8070b65f5.vhd\nStorageAccountLocation: SouthIndia",
         }