Merge pull request #63 from YannickRe/2.0.0

2.0.0

Author: YannickRe

README.md

@@ -8,9 +8,8 @@ Currently supports Windows Server 2019, Windows Server 2022, Ubuntu 2004 and Ubu
   - Checkout the latest `main` branch from [actions/runner-images](https://github.com/actions/runner-images)
   - Build the VM with Packer  
   - Clean up remaining temporary Azure resources  
-  - Turn VM disk into Azure Managed Image  
   - Add Azure Managed Image to Azure Compute Gallery or Update Virtual Machine Scale Set with the new image
-  - Remove Azure Managed Image
+  - Remove Azure Managed Image when using Azure Compute Gallery
 - __[managedimage-cleanup.yml](./managedimage-cleanup.yml)__  
   - Remove unused Azure Managed Images or old Gallery image versions, depending on selection
 
@@ -37,26 +36,18 @@ Connect-AzAccount -UseDeviceAuthentication
 ```
 New-AzResourceGroup -Name "DevOps-PackerResources" -Location "West Europe"
 ```
-3. Create an Azure Storage Account to store the generated VHD
-```
-New-AzStorageAccount -ResourceGroupName "DevOps-PackerResources" -AccountName "devopspacker" -Location "West Europe" -SkuName "Standard_LRS"
-```
-4. Create Azure AD Service Principal, output client secret and client id
+3. Create Azure AD Service Principal, output client secret and client id
 ```
 $sp = New-AzADServicePrincipal -DisplayName "DevOps-Packer"
 $BSTR = [System.Runtime.InteropServices.Marshal]::SecureStringToBSTR($sp.Secret)
 $plainPassword = [System.Runtime.InteropServices.Marshal]::PtrToStringAuto($BSTR)
 $plainPassword
 $sp.ApplicationId
 ```
-5. Make the Service Principal a Contributor on the subscription
+4. Make the Service Principal a Contributor on the subscription
 ```
 New-AzRoleAssignment -RoleDefinitionName Contributor -ServicePrincipalName $sp.ApplicationId
 ```
-6. Make the Service Principal a Storage Blob Data Contributor on the subscription
-```
-New-AzRoleAssignment -RoleDefinitionName "Storage Blob Data Contributor" -ServicePrincipalName $sp.ApplicationId
-```
 
 ### Azure Virtual Machine Scale Set
 To use an Azure Virtual Machine Scale Set as an Azure DevOps Scale Set Agent it has to adhere to a certain set of requirements. [The documentation](https://docs.microsoft.com/en-us/azure/devops/pipelines/agents/scale-set-agents?view=azure-devops&WT.mc_id=M365-MVP-5003400#create-the-scale-set) contains all the required information, but at the time of writing the following things were important:
@@ -77,8 +68,7 @@ Create a Variable Group in the Azure DevOps project running the pipeline, and gi
 |---|---|
 | AZURE_AGENTS_RESOURCE_GROUP | Resource Group that contains the Virtual Machine Scale Sets to be used as Scale Set Agents in Azure DevOps |
 | AZURE_LOCATION | Azure location where Packer will create the temporary resources |
-| AZURE_RESOURCE_GROUP | Resource group containing the Azure Storage Account that will be used by Packer. The resulting Azure Managed Image will also be put in this Resource Group |
-| AZURE_STORAGE_ACCOUNT | Storage Account that Packer will use to store the temporary OSDisk and the resulting sysprepped .vhd |
+| AZURE_RESOURCE_GROUP | Resource group that will be used by Packer to put the resulting Azure Managed Image. |
 | AZURE_SUBSCRIPTION | Subscription ID of the Azure Subscription that is used to host the temporary resources. |
 | BUILD_AGENT_VNET_NAME | Name of the existing VNet to use for the VM created by Packer, put $null if you want packer to create a new one |
 | BUILD_AGENT_VNET_RESOURCE_GROUP | Name of the resource group containing the existing VNet to use for the VM created by Packer, put $null if you don't have this |
@@ -158,7 +148,7 @@ When calling a template, you must provide certain parameters. For reference, ple
 There is one important element you must be aware of: 
 
 - repository_base_path
-  - This variable dictactes how the agent should resolve the assets within this repository. When used, two things will happen:
+  - This variable dictates how the agent should resolve the assets within this repository. When used, two things will happen:
     - First, it will clone the repository resource specified within your YML file, which represents _this_ repository
     - It will also use it to properly resolve the path where this repository resides on your pipeline agent
   - When a remote template is referenced within an Azure Pipeline YML file, it doesn't clone the repository. Providing this parameter will make sure these templates understands they need to clone it before being able to run any of the scripts.

buildagent-generation-galleryvm.yml

@@ -19,9 +19,8 @@ jobs:
   cancelTimeoutInMinutes: 90
   pool: 
     name: ${{ parameters.agent_pool }}
+    
   variables:
-  - name: ManagedImageId
-    value: $[stageDependencies.convert_template_to_managedvm.managedimagegeneration.outputs['createmanagedimage.ManagedImageId'] ]
   - name: ImageType
     value: $[stageDependencies.buildagent_template_vm.imagegeneration.outputs['setVars.ImageType'] ]
   - name: ResourcesNamePrefix
@@ -41,11 +40,10 @@ jobs:
       arguments: -ResourcesNamePrefix $(ResourcesNamePrefix) `
                  -ClientId $(CLIENT_ID) `
                  -ClientSecret $(CLIENT_SECRET) `
-                 -ResourceGroup $(AZURE_AGENTS_RESOURCE_GROUP) `
+                 -ResourceGroup $(AZURE_RESOURCE_GROUP) `
                  -SubscriptionId $(AZURE_SUBSCRIPTION) `
                  -TenantId $(AZURE_TENANT) `
                  -Location $(AZURE_LOCATION) `
                  -GalleryName $(GALLERY_NAME) `
                  -GalleryResourceGroup $(GALLERY_RESOURCE_GROUP) `
-                 -ImageType $(ImageType) `
-                 -ManagedImageId $(ManagedImageId)
+                 -ImageType $(ImageType) 

buildagent-generation-managedvm.yml

@@ -138,13 +138,16 @@ stages:
                   -ClientSecret $(CLIENT_SECRET) `
                   -TemplatePath $(TemplatePath) `
                   -ResourceGroup $(AZURE_RESOURCE_GROUP) `
-                  -StorageAccount $(AZURE_STORAGE_ACCOUNT) `
                   -SubscriptionId $(AZURE_SUBSCRIPTION) `
                   -TenantId $(AZURE_TENANT) `
                   -VirtualNetworkName $(BUILD_AGENT_VNET_NAME) `
                   -VirtualNetworkRG $(BUILD_AGENT_VNET_RESOURCE_GROUP) `
-                  -VirtualNetworkSubnet $(BUILD_AGENT_SUBNET_NAME)
-                  -Location $(AZURE_LOCATION)
+                  -VirtualNetworkSubnet $(BUILD_AGENT_SUBNET_NAME) `
+                  -Location $(AZURE_LOCATION) `
+                  -GalleryResourceGroup $(GALLERY_RESOURCE_GROUP) `
+                  -GalleryName $(GALLERY_NAME) `
+                  -CaptureNamePrefix $(setVars.ResourcesNamePrefix) `
+                  -ImageType ${{ parameters.image_type }}
       env:
         PACKER_LOG: 1
         PACKER_LOG_PATH: $(Build.ArtifactStagingDirectory)/packer-log.txt
@@ -176,25 +179,13 @@ stages:
                   -ClientSecret $(CLIENT_SECRET) `
                   -Image $(setVars.ImageType) `
                   -SubscriptionId $(AZURE_SUBSCRIPTION) `
-                  -TenantId $(AZURE_TENANT) `
-                  -StorageAccount $(AZURE_STORAGE_ACCOUNT)
-
-- stage: convert_template_to_managedvm
-  displayName: 'Convert Template to Managed VM'
-  dependsOn: buildagent_template_vm
-  jobs:
-  - template: buildagent-generation-managedvm.yml
-    parameters: 
-      variable_group: ${{ parameters.variable_group }}
-      agent_pool: ${{ parameters.agent_pool }}
-      repository_base_path: ${{ parameters.repository_base_path }}
+                  -TenantId $(AZURE_TENANT)
 
 - stage: create_gallery_imagevm
   displayName:  'Create Gallery Image Version VM'
   condition: and(succeeded(), eq('${{ parameters.update_type }}', 'galleryvm'))
   dependsOn: 
     - buildagent_template_vm
-    - convert_template_to_managedvm
   jobs:
   - template: buildagent-generation-galleryvm.yml
     parameters: 
@@ -206,7 +197,7 @@ stages:
   displayName:  'Update VM Scale Set to new Image'
   dependsOn: 
     - buildagent_template_vm
-    - convert_template_to_managedvm
+    # - convert_template_to_managedvm
   condition: and(succeeded(), eq('${{ parameters.update_type }}', 'vmss'))
   variables:
   - ${{ if eq(parameters.image_type, 'windows2019') }}:
@@ -227,4 +218,4 @@ stages:
       variable_group: ${{ parameters.variable_group }}
       agent_pool: ${{ parameters.agent_pool }}
       repository_base_path: ${{ parameters.repository_base_path }}
-      vmss_names: $(VmssNames)
+      vmss_names: $(VmssNames)

buildagent-generation-template.yml

@@ -14,17 +14,21 @@ parameters:
 - name: vmss_names
   displayName: VmssNames
   type: string
-
+  
 jobs:
 - job: updatevmss
   displayName: Update Virtual Machine Scale Set
   timeoutInMinutes: 600
   cancelTimeoutInMinutes: 30
   pool: 
     name: ${{ parameters.agent_pool }}
+
   variables:
-  - name: ManagedImageId
-    value: $[stageDependencies.convert_template_to_managedvm.managedimagegeneration.outputs['createmanagedimage.ManagedImageId'] ]
+  - name: ImageType
+    value: $[stageDependencies.buildagent_template_vm.imagegeneration.outputs['setVars.ImageType'] ]
+  - name: ResourcesNamePrefix
+    value: $[stageDependencies.buildagent_template_vm.imagegeneration.outputs['setVars.ResourcesNamePrefix'] ]
+  - group: ${{ parameters.variable_group }}
   - name: VmssNames
     value: ${{ parameters.vmss_names }}
   - group: ${{ parameters.variable_group }}
@@ -40,8 +44,10 @@ jobs:
       filePath: ${{ parameters.repository_base_path }}/scripts/update-vmss.ps1
       arguments: -ClientId $(CLIENT_ID) `
                  -ClientSecret $(CLIENT_SECRET) `
-                 -ResourceGroup $(AZURE_AGENTS_RESOURCE_GROUP) `
+                 -ResourceGroup $(AZURE_RESOURCE_GROUP) `
+                 -AgentsResourceGroup $(AZURE_AGENTS_RESOURCE_GROUP) `
                  -SubscriptionId $(AZURE_SUBSCRIPTION) `
                  -TenantId $(AZURE_TENANT) `
+                 -ResourceNamePrefix $(ResourcesNamePrefix)
+                 -ImageType $(ImageType)
                  -VmssNames $(VmssNames) `
-                 -ManagedImageId $(ManagedImageId)

buildagent-generation-update-vmss.yml

@@ -1,16 +1,18 @@
+trigger: none
+
 parameters:
 - name: variable_group
   displayName: Variable Group
   type: string
   default: BuildAgents
-- name: gallery_threshold
+- name: gallery_images_to_keep
   displayName: Number of image versions to keep in gallery
   type: number
-  default: 3
+  default: 2
 - name: agent_pool
   displayName: Agent Pool
-  type: string
   default: 'Host Pool - Image'
+  type: string
 - name: repository_base_path
   displayName: Scripts Path
   type: string
@@ -24,6 +26,7 @@ jobs:
   variables:
   - group: ${{ parameters.variable_group }}
 
+
   steps:
   - checkout: self
   - ${{ if ne(parameters.repository_base_path, '.') }}:
@@ -40,4 +43,5 @@ jobs:
                  -ResourceGroup $(AZURE_AGENTS_RESOURCE_GROUP) `
                  -GalleryName $(GALLERY_NAME) `
                  -GalleryResourceGroup $(GALLERY_RESOURCE_GROUP) `
-                 -ImageCountThreshold ${{ parameters.gallery_threshold }}
+                 -GalleryImagesToKeep ${{ parameters.gallery_images_to_keep }}
+

managedimage-cleanup-galleryvm.yml

@@ -5,13 +5,17 @@ param(
     [String] [Parameter (Mandatory=$true)] $ResourcesNamePrefix,
     [String] [Parameter (Mandatory=$true)] $Location,
     [String] [Parameter (Mandatory=$true)] $ResourceGroup,
-    [String] [Parameter (Mandatory=$true)] $StorageAccount,
     [String] [Parameter (Mandatory=$true)] $SubscriptionId,
     [String] [Parameter (Mandatory=$true)] $TenantId,
+    [String] [Parameter (Mandatory=$true)] $CaptureNamePrefix,
+    [String] [Parameter (Mandatory=$true)] $GalleryResourceGroup,
+    [String] [Parameter (Mandatory=$true)] $GalleryName,
+    [String] [Parameter (Mandatory=$true)] $ImageType, 
     [String] [Parameter (Mandatory=$false)] $VirtualNetworkName,
     [String] [Parameter (Mandatory=$false)] $VirtualNetworkRG,
     [String] [Parameter (Mandatory=$false)] $VirtualNetworkSubnet
-)
+
+    )
 
 if (-not (Test-Path $TemplatePath))
 {
@@ -39,20 +43,19 @@ Write-Host "Show Packer Version"
 packer --version
 
 Write-Host "Build $Image VM"
-packer build    -var "capture_name_prefix=$ResourcesNamePrefix" `
-                -var "client_id=$ClientId" `
+packer build    -var "client_id=$ClientId" `
                 -var "client_secret=$ClientSecret" `
                 -var "install_password=$InstallPassword" `
                 -var "location=$Location" `
-                -var "resource_group=$ResourceGroup" `
-                -var "storage_account=$StorageAccount" `
                 -var "subscription_id=$SubscriptionId" `
                 -var "temp_resource_group_name=$TempResourceGroupName" `
                 -var "tenant_id=$TenantId" `
                 -var "virtual_network_name=$VirtualNetworkName" `
                 -var "virtual_network_resource_group_name=$VirtualNetworkRG" `
                 -var "virtual_network_subnet_name=$VirtualNetworkSubnet" `
                 -var "run_validation_diskspace=$env:RUN_VALIDATION_FLAG" `
+                -var "managed_image_name=$ImageType-$ResourcesNamePrefix" `
+                -var "managed_image_resource_group_name=$ResourceGroup" `
                 -color=false `
                 $TemplatePath `
         | Foreach-Object { 
@@ -69,4 +72,4 @@ packer build    -var "capture_name_prefix=$ResourcesNamePrefix" `
 
             $sensitiveString = $SensitiveData | Where-Object { $currentString -match $_ }
             $sensitiveString -eq $null
-        }
+        }

scripts/build-image.ps1

@@ -1,4 +1,3 @@
-
 param(
     [String] [Parameter (Mandatory=$true)] $ClientId,
     [String] [Parameter (Mandatory=$true)] $ClientSecret,
@@ -7,7 +6,7 @@ param(
     [String] [Parameter (Mandatory=$true)] $ResourceGroup,
     [String] [Parameter (Mandatory=$true)] $GalleryName,
     [String] [Parameter (Mandatory=$true)] $GalleryResourceGroup,
-    [int] [Parameter (Mandatory=$true)] $ImageCountThreshold
+    [int] [Parameter (Mandatory=$true)] $GalleryImagesToKeep 
 )
 
 # Variables
@@ -17,6 +16,7 @@ $imageDefinitions = @(
     "windows2019-agentpool-full",
     "windows2022-agentpool-full"
 )
+$ImageCountThreshold = $GalleryImagesToKeep + 1
 
 # install required modules
 install-module Az.Compute -Scope CurrentUser -AllowClobber -Force
@@ -58,15 +58,17 @@ if ($gallery) {
 
             if ($images.Count -ge $ImageCountThreshold) {
                 # Sort the images by creation timestamp in ascending order
-                $sortedImages = $images | Sort-Object -Property CreatedTime
-
-                # Remove the oldest image
-                $imageToRemove = $sortedImages[0]
+                #$sortedImages = $images | Sort-Object -Property { [DateTime]::ParseExact($_.PublishingProfile.PublishedDate, 'dd/MM/yyyy HH:mm:ss', $null) } 
+                $sortedImages = $images | Sort-Object -Property $_.PublishingProfile.PublishedDate 
 
-                Write-Host "##[section]Removing the oldest image version for image definition '$imageDefinition': $($imageToRemove.Name)"
-                Remove-AzGalleryImageVersion -ResourceGroupName $GalleryResourceGroup -GalleryName $gallery.Name -GalleryImageDefinitionName $imageDefinition -Name $imageToRemove.Name -Force -AsJob 
-            } else {
-                Write-Host "##[section]The number of images for image definition '$imageDefinition' is less than $ImageCountThreshold"
+                # Remove all images except the most recent ones
+                $imagesToRemove = $sortedImages[0..($sortedImages.Count - $GalleryImagesToKeep - 1 )]
+                foreach ($imageToRemove in $imagesToRemove) {
+                    Write-Host "##[section]Removing image version for image definition '$imageDefinition': $($imageToRemove.Name) with $($imageToRemove.PublishingProfile.PublishedDate)"
+                    Remove-AzGalleryImageVersion -ResourceGroupName $GalleryResourceGroup -GalleryName $gallery.Name -GalleryImageDefinitionName $imageDefinition -Name $imageToRemove.Name -Force -AsJob
+                }
+                } else {
+                Write-Host "##[section]The number of images for image definition '$imageDefinition' has no more than $GalleryImagesToKeep images. No images will be removed."
             }
         }
    }