Merge pull request #298 from BrianLPeterson/user/bpet/retryfix7

Adding retries to Invoke-UseCase and GetAzureStackScaleUnitNode.

Author: MatthewMcGlynn

CanaryValidator/Canary.Tests.ps1

@@ -232,7 +232,7 @@ while ($runCount -le $NumberOfIterations)
             Get-AzsScaleUnit -Location $ResourceLocation
         }
 
-        Invoke-Usecase -Name 'GetAzureStackScaleUnitNode' -Description "List nodes in scale unit" -UsecaseBlock `
+        Invoke-Usecase -Name 'GetAzureStackScaleUnitNode' -Description "List nodes in scale unit" -RetryCount 2 -RetryDelayInSec 20 -UsecaseBlock `
         {
             Get-AzsScaleUnitNode -Location $ResourceLocation
         }
@@ -295,7 +295,7 @@ while ($runCount -le $NumberOfIterations)
         }
     }
 
-    if ((Get-Volume ((Get-Item -Path $ENV:TMP).PSDrive.Name)).SizeRemaining/1GB -gt 35)
+    if ((Get-Volume ((Get-Item -Path $ENV:TMP).PSDrive.Name) -ErrorAction SilentlyContinue).SizeRemaining/1GB -gt 35)
     {
         [boolean]$invalidUri = $false
         try {Invoke-WebRequest -Uri $LinuxImagePath -UseBasicParsing -DisableKeepAlive -Method Head -ErrorAction SilentlyContinue | Out-Null} 

CanaryValidator/Canary.Utilities.psm1

@@ -283,7 +283,11 @@ function Invoke-Usecase
         [string]$Description, 
         [parameter(Mandatory=$true, Position = 2)]
         [ValidateNotNullOrEmpty()]
-        [ScriptBlock]$UsecaseBlock    
+        [ScriptBlock]$UsecaseBlock,
+        [parameter(Mandatory=$false, Position = 3)]
+        [int]$RetryCount = 0, 
+        [parameter(Mandatory=$false, Position = 4)]
+        [int]$RetryDelayInSec = 10
     )
 
     if ($Global:listAvailableUsecases)
@@ -324,7 +328,54 @@ function Invoke-Usecase
 
     try
     {
-        $result = Invoke-Command -ScriptBlock $UsecaseBlock
+        $currentRetryCount = 0
+
+        do
+        {
+            $useCaseError = $null
+
+            try
+            {
+                $result = Invoke-Command -ScriptBlock $UsecaseBlock
+            }
+            catch
+            {
+                $useCaseError = $_
+            }
+
+            # Dont retry or log errors on the parent cases
+            if ($UsecaseBlock.ToString().Contains("Invoke-Usecase"))
+            {
+                $useCaseError = $null
+                break;
+            }
+
+            if ($useCaseError)
+            {       
+                $currentRetryCount++
+
+                if($currentRetryCount -gt $retryCount)
+                {
+                    break
+                }
+                else
+                {
+                    Start-Sleep -Seconds $RetryDelayInSec
+                }
+            }
+        }
+        while($useCaseError)
+
+        if ($useCaseError -and $RetryCount -gt 0)
+        {
+            throw "Test '$Name' failed and retried '$RetryCount' times. The last exception was: $useCaseError" 
+        }
+
+        if ($useCaseError)
+        {
+            throw $useCaseError
+        }
+
         if ($result -and (-not $UsecaseBlock.ToString().Contains("Invoke-Usecase")))
         {
             Log-Info ($result)