this probably needs cleanup

Author: rvdwegen

Modules/CIPPCore/Public/GraphHelper/New-CIPPAzRestRequest.ps1

@@ -35,6 +35,8 @@ function New-CIPPAzRestRequest {
         Use basic parsing (for older PowerShell versions)
     .PARAMETER WebSession
         Web session object for maintaining cookies/state
+    .PARAMETER MaxRetries
+        Maximum number of retry attempts for transient failures. Defaults to 3.
     .EXAMPLE
         New-CIPPAzRestRequest -Uri 'https://management.azure.com/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Web/sites/{name}?api-version=2020-06-01'
         Gets Azure Resource Manager resource using managed identity
@@ -92,7 +94,10 @@ function New-CIPPAzRestRequest {
         [switch]$UseBasicParsing,
 
         [Parameter(Mandatory = $false)]
-        [Microsoft.PowerShell.Commands.WebRequestSession]$WebSession
+        [Microsoft.PowerShell.Commands.WebRequestSession]$WebSession,
+
+        [Parameter(Mandatory = $false)]
+        [int]$MaxRetries = 3
     )
 
     # Get Azure Managed Identity token
@@ -178,32 +183,140 @@ function New-CIPPAzRestRequest {
         $RestMethodParams['WebSession'] = $WebSession
     }
 
-    # Invoke the REST method
-    try {
-        $Response = Invoke-RestMethod @RestMethodParams
+    # Invoke the REST method with retry logic
+    $RetryCount = 0
+    $RequestSuccessful = $false
+    $Message = $null
+    $MessageObj = $null
 
-        # For compatibility with Invoke-AzRestMethod behavior, return object with Content property if response is a string
-        # Otherwise return the parsed object directly
-        if ($Response -is [string]) {
-            return [PSCustomObject]@{
-                Content = $Response
+    Write-Information "$($Method.ToUpper()) [ $Uri ] | attempt: $($RetryCount + 1) of $MaxRetries"
+
+    do {
+        try {
+            $Response = Invoke-RestMethod @RestMethodParams
+            $RequestSuccessful = $true
+
+            # For compatibility with Invoke-AzRestMethod behavior, return object with Content property if response is a string
+            # Otherwise return the parsed object directly
+            if ($Response -is [string]) {
+                return [PSCustomObject]@{
+                    Content = $Response
+                }
             }
-        }
 
-        return $Response
-    } catch {
-        $errorMessage = "Azure REST API call failed: $($_.Exception.Message)"
-        if ($_.Exception.Response) {
+            return $Response
+        } catch {
+            $ShouldRetry = $false
+            $WaitTime = 0
+
+            # Extract error message from JSON response if available
             try {
-                $reader = New-Object System.IO.StreamReader($_.Exception.Response.GetResponseStream())
-                $responseBody = $reader.ReadToEnd()
-                $reader.Close()
-                $errorMessage += "`nResponse: $responseBody"
+                if ($_.ErrorDetails.Message) {
+                    $MessageObj = $_.ErrorDetails.Message | ConvertFrom-Json -ErrorAction SilentlyContinue
+                    if ($MessageObj.error) {
+                        $MessageObj | Add-Member -NotePropertyName 'url' -NotePropertyValue $Uri -Force
+                        $Message = if ($MessageObj.error.message) {
+                            Get-NormalizedError -message $MessageObj.error.message
+                        } elseif ($MessageObj.error.code) {
+                            $MessageObj.error.code
+                        } else {
+                            $_.Exception.Message
+                        }
+                    } else {
+                        $Message = Get-NormalizedError -message $_.ErrorDetails.Message
+                    }
+                } else {
+                    $Message = $_.Exception.Message
+                }
             } catch {
-                # Ignore errors reading response stream
+                $Message = $_.Exception.Message
+            }
+
+            # If we couldn't extract a message, use the exception message
+            if ([string]::IsNullOrEmpty($Message)) {
+                $Message = $_.Exception.Message
+                $MessageObj = @{
+                    error = @{
+                        code    = $_.Exception.GetType().FullName
+                        message = $Message
+                        url     = $Uri
+                    }
+                }
+            }
+
+            # Check for 429 Too Many Requests (rate limiting)
+            if ($_.Exception.Response -and $_.Exception.Response.StatusCode -eq 429) {
+                $RetryAfterHeader = $_.Exception.Response.Headers['Retry-After']
+                if ($RetryAfterHeader) {
+                    $WaitTime = [int]$RetryAfterHeader
+                    Write-Warning "Rate limited (429). Waiting $WaitTime seconds before retry. Attempt $($RetryCount + 1) of $MaxRetries"
+                    $ShouldRetry = $true
+                } elseif ($RetryCount -lt $MaxRetries) {
+                    # Exponential backoff if no Retry-After header
+                    $WaitTime = [Math]::Min([Math]::Pow(2, $RetryCount), 60)  # Cap at 60 seconds
+                    Write-Warning "Rate limited (429) without Retry-After header. Waiting $WaitTime seconds before retry. Attempt $($RetryCount + 1) of $MaxRetries"
+                    $ShouldRetry = $true
+                }
+            }
+            # Check for 503 Service Unavailable or temporary errors
+            elseif ($_.Exception.Response -and $_.Exception.Response.StatusCode -eq 503) {
+                if ($RetryCount -lt $MaxRetries) {
+                    $WaitTime = Get-Random -Minimum 1.1 -Maximum 3.1  # Random sleep between 1-3 seconds
+                    Write-Warning "Service unavailable (503). Waiting $WaitTime seconds before retry. Attempt $($RetryCount + 1) of $MaxRetries"
+                    $ShouldRetry = $true
+                }
+            }
+            # Check for "Resource temporarily unavailable" or other transient errors
+            elseif ($Message -like '*Resource temporarily unavailable*' -or $Message -like '*temporarily*' -or $Message -like '*timeout*') {
+                if ($RetryCount -lt $MaxRetries) {
+                    $WaitTime = Get-Random -Minimum 1.1 -Maximum 3.1  # Random sleep between 1-3 seconds
+                    Write-Warning "Transient error detected. Waiting $WaitTime seconds before retry. Attempt $($RetryCount + 1) of $MaxRetries"
+                    $ShouldRetry = $true
+                }
+            }
+            # Check for 500/502/504 server errors (retryable)
+            elseif ($_.Exception.Response -and $_.Exception.Response.StatusCode -in @(500, 502, 504)) {
+                if ($RetryCount -lt $MaxRetries) {
+                    $WaitTime = Get-Random -Minimum 1.1 -Maximum 3.1  # Random sleep between 1-3 seconds
+                    Write-Warning "Server error ($($_.Exception.Response.StatusCode)). Waiting $WaitTime seconds before retry. Attempt $($RetryCount + 1) of $MaxRetries"
+                    $ShouldRetry = $true
+                }
+            }
+
+            # Retry if conditions are met
+            if ($ShouldRetry -and $RetryCount -lt $MaxRetries) {
+                $RetryCount++
+                if ($WaitTime -gt 0) {
+                    Start-Sleep -Seconds $WaitTime
+                }
+                Write-Information "$($Method.ToUpper()) [ $Uri ] | attempt: $($RetryCount + 1) of $MaxRetries"
+            } else {
+                # Final failure - build detailed error message
+                $errorMessage = "Azure REST API call failed: $Message"
+                if ($_.Exception.Response) {
+                    $errorMessage += " (Status: $($_.Exception.Response.StatusCode))"
+                    try {
+                        $reader = New-Object System.IO.StreamReader($_.Exception.Response.GetResponseStream())
+                        $responseBody = $reader.ReadToEnd()
+                        $reader.Close()
+                        if ($responseBody) {
+                            $errorMessage += "`nResponse: $responseBody"
+                        }
+                    } catch {
+                        # Ignore errors reading response stream
+                    }
+                }
+                $errorMessage += "`nURI: $Uri"
+
+                Write-Error -Message $errorMessage -ErrorAction $ErrorActionPreference
+                return
             }
         }
+    } while (-not $RequestSuccessful -and $RetryCount -le $MaxRetries)
 
+    # Should never reach here, but just in case
+    if (-not $RequestSuccessful) {
+        $errorMessage = "Azure REST API call failed after $MaxRetries attempts: $Message`nURI: $Uri"
         Write-Error -Message $errorMessage -ErrorAction $ErrorActionPreference
         return
     }