Skip to content

Commit ce08eb4

Browse files
KelvinTegelaarKelvinTegelaar
committed
backoff logic
1 parent baa198d commit ce08eb4

File tree

1 file changed

+88
-51
lines changed

1 file changed

+88
-51
lines changed

‎Modules/CIPPCore/Public/GraphHelper/New-GraphGetRequest.ps1‎

Lines changed: 88 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -56,71 +56,108 @@ function New-GraphGetRequest {
5656
}
5757

5858
$ReturnedData = do {
59-
try {
60-
$GraphRequest = @{
61-
Uri = $nextURL
62-
Method = 'GET'
63-
Headers = $headers
64-
ContentType = 'application/json; charset=utf-8'
65-
}
66-
if ($IncludeResponseHeaders) {
67-
$GraphRequest.ResponseHeadersVariable = 'ResponseHeaders'
68-
}
59+
$RetryCount = 0
60+
$MaxRetries = 3
61+
$RequestSuccessful = $false
6962

70-
if ($ReturnRawResponse) {
71-
$GraphRequest.SkipHttpErrorCheck = $true
72-
$Data = Invoke-WebRequest @GraphRequest
73-
} else {
74-
$Data = (Invoke-RestMethod @GraphRequest)
75-
}
63+
do {
64+
try {
65+
$GraphRequest = @{
66+
Uri = $nextURL
67+
Method = 'GET'
68+
Headers = $headers
69+
ContentType = 'application/json; charset=utf-8'
70+
}
71+
if ($IncludeResponseHeaders) {
72+
$GraphRequest.ResponseHeadersVariable = 'ResponseHeaders'
73+
}
7674

77-
if ($ReturnRawResponse) {
78-
if (Test-Json -Json $Data.Content) {
79-
$Content = $Data.Content | ConvertFrom-Json
75+
if ($ReturnRawResponse) {
76+
$GraphRequest.SkipHttpErrorCheck = $true
77+
$Data = Invoke-WebRequest @GraphRequest
8078
} else {
81-
$Content = $Data.Content
79+
$Data = (Invoke-RestMethod @GraphRequest)
8280
}
8381

84-
$Data | Select-Object -Property StatusCode, StatusDescription, @{Name = 'Content'; Expression = { $Content }}
85-
$nextURL = $null
86-
} elseif ($CountOnly) {
87-
$Data.'@odata.count'
88-
$NextURL = $null
89-
} else {
90-
if ($Data.PSObject.Properties.Name -contains 'value') { $data.value } else { $Data }
91-
if ($noPagination -eq $true) {
92-
if ($Caller -eq 'Get-GraphRequestList') {
93-
@{ 'nextLink' = $data.'@odata.nextLink' }
82+
# If we reach here, the request was successful
83+
$RequestSuccessful = $true
84+
85+
if ($ReturnRawResponse) {
86+
if (Test-Json -Json $Data.Content) {
87+
$Content = $Data.Content | ConvertFrom-Json
88+
} else {
89+
$Content = $Data.Content
9490
}
91+
92+
$Data | Select-Object -Property StatusCode, StatusDescription, @{Name = 'Content'; Expression = { $Content }}
9593
$nextURL = $null
94+
} elseif ($CountOnly) {
95+
$Data.'@odata.count'
96+
$NextURL = $null
9697
} else {
97-
$NextPageUriFound = $false
98-
if ($IncludeResponseHeaders) {
99-
if ($ResponseHeaders.NextPageUri) {
100-
$NextURL = $ResponseHeaders.NextPageUri
101-
$NextPageUriFound = $true
98+
if ($Data.PSObject.Properties.Name -contains 'value') { $data.value } else { $Data }
99+
if ($noPagination -eq $true) {
100+
if ($Caller -eq 'Get-GraphRequestList') {
101+
@{ 'nextLink' = $data.'@odata.nextLink' }
102+
}
103+
$nextURL = $null
104+
} else {
105+
$NextPageUriFound = $false
106+
if ($IncludeResponseHeaders) {
107+
if ($ResponseHeaders.NextPageUri) {
108+
$NextURL = $ResponseHeaders.NextPageUri
109+
$NextPageUriFound = $true
110+
}
102111
}
112+
if (!$NextPageUriFound) {
113+
$nextURL = $data.'@odata.nextLink'
114+
}
115+
}
116+
}
117+
} catch {
118+
$ShouldRetry = $false
119+
$WaitTime = 0
120+
121+
try {
122+
$Message = ($_.ErrorDetails.Message | ConvertFrom-Json -ErrorAction SilentlyContinue).error.message
123+
} catch { $Message = $null }
124+
if ($Message -eq $null) { $Message = $($_.Exception.Message) }
125+
126+
# Check for 429 Too Many Requests
127+
if ($_.Exception.Response.StatusCode -eq 429) {
128+
$RetryAfterHeader = $_.Exception.Response.Headers['Retry-After']
129+
if ($RetryAfterHeader) {
130+
$WaitTime = [int]$RetryAfterHeader
131+
Write-Warning "Rate limited (429). Waiting $WaitTime seconds before retry. Attempt $($RetryCount + 1) of $MaxRetries"
132+
$ShouldRetry = $true
103133
}
104-
if (!$NextPageUriFound) {
105-
$nextURL = $data.'@odata.nextLink'
134+
}
135+
# Check for "Resource temporarily unavailable"
136+
elseif ($Message -like "*Resource temporarily unavailable*") {
137+
if ($RetryCount -lt $MaxRetries) {
138+
$WaitTime = Get-Random -Minimum 1 -Maximum 10 # Random sleep between 1-10 seconds
139+
Write-Warning "Resource temporarily unavailable. Waiting $WaitTime seconds before retry. Attempt $($RetryCount + 1) of $MaxRetries"
140+
$ShouldRetry = $true
106141
}
107142
}
108-
}
109-
} catch {
110-
try {
111-
$Message = ($_.ErrorDetails.Message | ConvertFrom-Json -ErrorAction SilentlyContinue).error.message
112-
} catch { $Message = $null }
113-
if ($Message -eq $null) { $Message = $($_.Exception.Message) }
114-
if ($Message -ne 'Request not applicable to target tenant.' -and $Tenant) {
115-
$Tenant.LastGraphError = $Message
116-
if ($Tenant.PSObject.Properties.Name -notcontains 'GraphErrorCount') {
117-
$Tenant | Add-Member -MemberType NoteProperty -Name 'GraphErrorCount' -Value 0 -Force
143+
144+
if ($ShouldRetry -and $RetryCount -lt $MaxRetries) {
145+
$RetryCount++
146+
Start-Sleep -Seconds $WaitTime
147+
} else {
148+
# Final failure - update tenant error tracking and throw
149+
if ($Message -ne 'Request not applicable to target tenant.' -and $Tenant) {
150+
$Tenant.LastGraphError = $Message
151+
if ($Tenant.PSObject.Properties.Name -notcontains 'GraphErrorCount') {
152+
$Tenant | Add-Member -MemberType NoteProperty -Name 'GraphErrorCount' -Value 0 -Force
153+
}
154+
$Tenant.GraphErrorCount++
155+
Update-AzDataTableEntity -Force @TenantsTable -Entity $Tenant
156+
}
157+
throw $Message
118158
}
119-
$Tenant.GraphErrorCount++
120-
Update-AzDataTableEntity -Force @TenantsTable -Entity $Tenant
121159
}
122-
throw $Message
123-
}
160+
} while (-not $RequestSuccessful -and $RetryCount -le $MaxRetries)
124161
} until ([string]::IsNullOrEmpty($NextURL) -or $NextURL -is [object[]] -or ' ' -eq $NextURL)
125162
if ($Tenant.PSObject.Properties.Name -notcontains 'LastGraphError') {
126163
$Tenant | Add-Member -MemberType NoteProperty -Name 'LastGraphError' -Value '' -Force

0 commit comments

Comments
 (0)