Skip to content

Commit 440c1bf

Browse files
pull[bot]pull[bot]
authored
Merge pull request #442 from KelvinTegelaar/dev
[pull] dev from KelvinTegelaar:dev
2 parents bf39f66 + 3e6d1ba commit 440c1bf

File tree

5 files changed

+222
-95
lines changed

5 files changed

+222
-95
lines changed

‎Config/SchedulerRateLimits.json‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
[
2+
{
3+
"Command": "Sync-CIPPExtensionData",
4+
"MaxRequests": 50
5+
},
6+
{
7+
"Command": "Push-CIPPExtensionData",
8+
"MaxRequests": 30
9+
}
10+
]

‎Modules/CIPPCore/Public/Entrypoints/HTTP Functions/CIPP/Settings/Invoke-ExecAddTrustedIP.ps1‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
using namespace System.Net
22

3-
Function Invoke-ExecAddTrustedIP {
3+
function Invoke-ExecAddTrustedIP {
44
<#
55
.FUNCTIONALITY
66
Entrypoint
@@ -11,12 +11,13 @@ Function Invoke-ExecAddTrustedIP {
1111
param($Request, $TriggerMetadata)
1212

1313
$Table = Get-CippTable -tablename 'trustedIps'
14-
Add-CIPPAzDataTableEntity @Table -Entity @{
15-
PartitionKey = $Request.Body.tenantfilter
16-
RowKey = $Request.Body.IP
17-
state = $Request.Body.State
18-
} -Force
19-
14+
foreach ($IP in $Request.body.IP) {
15+
Add-CIPPAzDataTableEntity @Table -Entity @{
16+
PartitionKey = $Request.Body.tenantfilter
17+
RowKey = $IP
18+
state = $Request.Body.State
19+
} -Force
20+
}
2021
Push-OutputBinding -Name Response -Value ([HttpResponseContext]@{
2122
StatusCode = [HttpStatusCode]::OK
2223
Body = @{ results = "Added $($Request.Body.IP) to database with state $($Request.Body.State) for $($Request.Body.tenantfilter)" }

‎Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-UserTasksOrchestrator.ps1‎

Lines changed: 82 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -10,10 +10,39 @@ function Start-UserTasksOrchestrator {
1010
$1HourAgo = (Get-Date).AddHours(-1).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ssZ')
1111
$Filter = "PartitionKey eq 'ScheduledTask' and (TaskState eq 'Planned' or TaskState eq 'Failed - Planned' or (TaskState eq 'Running' and Timestamp lt datetime'$1HourAgo'))"
1212
$tasks = Get-CIPPAzDataTableEntity @Table -Filter $Filter
13+
14+
$RateLimitTable = Get-CIPPTable -tablename 'SchedulerRateLimits'
15+
$RateLimits = Get-CIPPAzDataTableEntity @RateLimitTable -Filter "PartitionKey eq 'SchedulerRateLimits'"
16+
17+
$CIPPCoreModuleRoot = Get-Module -Name CIPPCore | Select-Object -ExpandProperty ModuleBase
18+
$CIPPRoot = (Get-Item $CIPPCoreModuleRoot).Parent.Parent
19+
$DefaultRateLimits = Get-Content -Path "$CIPPRoot/Config/SchedulerRateLimits.json" | ConvertFrom-Json
20+
$NewRateLimits = foreach ($Limit in $DefaultRateLimits) {
21+
if ($Limit.Command -notin $RateLimits.RowKey) {
22+
@{
23+
PartitionKey = 'SchedulerRateLimits'
24+
RowKey = $Limit.Command
25+
MaxRequests = $Limit.MaxRequests
26+
}
27+
}
28+
}
29+
30+
if ($NewRateLimits) {
31+
$null = Add-CIPPAzDataTableEntity @RateLimitTable -Entity $NewRateLimits -Force
32+
$RateLimits = Get-CIPPAzDataTableEntity @RateLimitTable -Filter "PartitionKey eq 'SchedulerRateLimits'"
33+
}
34+
35+
# Create a hashtable for quick rate limit lookups
36+
$RateLimitLookup = @{}
37+
foreach ($limit in $RateLimits) {
38+
$RateLimitLookup[$limit.RowKey] = $limit.MaxRequests
39+
}
40+
1341
$Batch = [System.Collections.Generic.List[object]]::new()
1442
$TenantList = Get-Tenants -IncludeErrors
1543
foreach ($task in $tasks) {
1644
$tenant = $task.Tenant
45+
1746
$currentUnixTime = [int64](([datetime]::UtcNow) - (Get-Date '1/1/1970')).TotalSeconds
1847
if ($currentUnixTime -ge $task.ScheduledTime) {
1948
try {
@@ -113,21 +142,62 @@ function Start-UserTasksOrchestrator {
113142
}
114143
}
115144
}
145+
146+
Write-Information 'Batching tasks for execution...'
147+
Write-Information "Total tasks to process: $($Batch.Count)"
148+
116149
if (($Batch | Measure-Object).Count -gt 0) {
117-
# Create queue entry
118-
$Queue = New-CippQueueEntry -Name 'Scheduled Tasks' -TotalTasks ($Batch | Measure-Object).Count
119-
$QueueId = $Queue.RowKey
120-
$Batch = $Batch | Select-Object *, @{Name = 'QueueId'; Expression = { $QueueId } }, @{Name = 'QueueName'; Expression = { '{0} - {1}' -f $_.TaskInfo.Name, ($_.TaskInfo.Tenant -ne 'AllTenants' ? $_.TaskInfo.Tenant : $_.Parameters.TenantFilter) } }
121-
122-
$InputObject = [PSCustomObject]@{
123-
OrchestratorName = 'UserTaskOrchestrator'
124-
Batch = @($Batch)
125-
SkipLog = $true
150+
# Group commands by type and apply rate limits
151+
$CommandGroups = $Batch | Group-Object -Property Command
152+
$ProcessedBatches = [System.Collections.Generic.List[object]]::new()
153+
154+
foreach ($CommandGroup in $CommandGroups) {
155+
$CommandName = $CommandGroup.Name
156+
$Commands = [System.Collections.Generic.List[object]]::new($CommandGroup.Group)
157+
158+
# Get rate limit for this command (default to 100 if not found)
159+
$MaxItemsPerBatch = if ($RateLimitLookup.ContainsKey($CommandName)) {
160+
$RateLimitLookup[$CommandName]
161+
} else {
162+
100
163+
}
164+
165+
# Split into batches based on rate limit
166+
while ($Commands.Count -gt 0) {
167+
$BatchSize = [Math]::Min($Commands.Count, $MaxItemsPerBatch)
168+
$CommandBatch = [System.Collections.Generic.List[object]]::new()
169+
170+
for ($i = 0; $i -lt $BatchSize; $i++) {
171+
$CommandBatch.Add($Commands[0])
172+
$Commands.RemoveAt(0)
173+
}
174+
175+
$ProcessedBatches.Add($CommandBatch)
176+
}
126177
}
127-
#Write-Host ($InputObject | ConvertTo-Json -Depth 10)
128178

129-
if ($PSCmdlet.ShouldProcess('Start-UserTasksOrchestrator', 'Starting User Tasks Orchestrator')) {
130-
Start-NewOrchestration -FunctionName 'CIPPOrchestrator' -InputObject ($InputObject | ConvertTo-Json -Depth 10 -Compress)
179+
# Process each batch separately
180+
foreach ($ProcessedBatch in $ProcessedBatches) {
181+
Write-Information "Processing batch with $($ProcessedBatch.Count) tasks..."
182+
Write-Information 'Tasks by command:'
183+
$ProcessedBatch | Group-Object -Property Command | ForEach-Object {
184+
Write-Information " - $($_.Name): $($_.Count)"
185+
}
186+
187+
# Create queue entry for each batch
188+
$Queue = New-CippQueueEntry -Name "Scheduled Tasks - Batch #$($ProcessedBatches.IndexOf($ProcessedBatch) + 1) of $($ProcessedBatches.Count)"
189+
$QueueId = $Queue.RowKey
190+
$BatchWithQueue = $ProcessedBatch | Select-Object *, @{Name = 'QueueId'; Expression = { $QueueId } }, @{Name = 'QueueName'; Expression = { '{0} - {1}' -f $_.TaskInfo.Name, ($_.TaskInfo.Tenant -ne 'AllTenants' ? $_.TaskInfo.Tenant : $_.Parameters.TenantFilter) } }
191+
192+
$InputObject = [PSCustomObject]@{
193+
OrchestratorName = 'UserTaskOrchestrator'
194+
Batch = @($BatchWithQueue)
195+
SkipLog = $true
196+
}
197+
198+
if ($PSCmdlet.ShouldProcess('Start-UserTasksOrchestrator', 'Starting User Tasks Orchestrator')) {
199+
Start-NewOrchestration -FunctionName 'CIPPOrchestrator' -InputObject ($InputObject | ConvertTo-Json -Depth 10 -Compress)
200+
}
131201
}
132202
}
133203
}

‎Modules/CIPPCore/Public/Get-CIPPDrift.ps1‎

Lines changed: 33 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,24 @@ function Get-CIPPDrift {
2929
[switch]$AllTenants
3030
)
3131

32+
33+
$IntuneTable = Get-CippTable -tablename 'templates'
34+
$IntuneFilter = "PartitionKey eq 'IntuneTemplate'"
35+
$RawIntuneTemplates = (Get-CIPPAzDataTableEntity @IntuneTable -Filter $IntuneFilter)
36+
$AllIntuneTemplates = $RawIntuneTemplates | ForEach-Object {
37+
try {
38+
$JSONData = $_.JSON | ConvertFrom-Json -Depth 100 -ErrorAction SilentlyContinue
39+
$data = $JSONData.RAWJson | ConvertFrom-Json -Depth 100 -ErrorAction SilentlyContinue
40+
$data | Add-Member -NotePropertyName 'displayName' -NotePropertyValue $JSONData.Displayname -Force
41+
$data | Add-Member -NotePropertyName 'description' -NotePropertyValue $JSONData.Description -Force
42+
$data | Add-Member -NotePropertyName 'Type' -NotePropertyValue $JSONData.Type -Force
43+
$data | Add-Member -NotePropertyName 'GUID' -NotePropertyValue $_.RowKey -Force
44+
$data
45+
} catch {
46+
# Skip invalid templates
47+
}
48+
} | Sort-Object -Property displayName
49+
3250
try {
3351
$AlignmentData = Get-CIPPTenantAlignment -TenantFilter $TenantFilter -TemplateId $TemplateId | Where-Object -Property standardType -EQ 'drift'
3452
if (-not $AlignmentData) {
@@ -64,16 +82,24 @@ function Get-CIPPDrift {
6482
} else {
6583
'New'
6684
}
85+
#if the $ComparisonItem.StandardName contains *intuneTemplate*, then it's an Intune policy deviation, and we need to grab the correct displayname from the template table
86+
if ($ComparisonItem.StandardName -like '*intuneTemplate*') {
87+
$CompareGuid = $ComparisonItem.StandardName.Split('.') | Select-Object -Index 2
88+
Write-Host "Extracted GUID: $CompareGuid"
89+
$Template = $AllIntuneTemplates | Where-Object { $_.GUID -like "*$CompareGuid*" }
90+
if ($Template) { $displayName = $Template.displayName }
91+
}
6792
$reason = if ($ExistingDriftStates.ContainsKey($ComparisonItem.StandardName)) { $ExistingDriftStates[$ComparisonItem.StandardName].Reason }
6893
$User = if ($ExistingDriftStates.ContainsKey($ComparisonItem.StandardName)) { $ExistingDriftStates[$ComparisonItem.StandardName].User }
6994
$StandardsDeviations.Add([PSCustomObject]@{
70-
standardName = $ComparisonItem.StandardName
71-
expectedValue = 'Compliant'
72-
receivedValue = $ComparisonItem.StandardValue
73-
state = 'current'
74-
Status = $Status
75-
Reason = $reason
76-
lastChangedByUser = $User
95+
standardName = $ComparisonItem.StandardName
96+
standardDisplayName = $displayName
97+
expectedValue = 'Compliant'
98+
receivedValue = $ComparisonItem.StandardValue
99+
state = 'current'
100+
Status = $Status
101+
Reason = $reason
102+
lastChangedByUser = $User
77103
})
78104
}
79105
}
@@ -194,22 +220,6 @@ function Get-CIPPDrift {
194220
# Get actual Intune templates from templates table
195221
if ($IntuneTemplateIds.Count -gt 0) {
196222
try {
197-
$IntuneTable = Get-CippTable -tablename 'templates'
198-
$IntuneFilter = "PartitionKey eq 'IntuneTemplate'"
199-
$RawIntuneTemplates = (Get-CIPPAzDataTableEntity @IntuneTable -Filter $IntuneFilter)
200-
$AllIntuneTemplates = $RawIntuneTemplates | ForEach-Object {
201-
try {
202-
$JSONData = $_.JSON | ConvertFrom-Json -Depth 100 -ErrorAction SilentlyContinue
203-
$data = $JSONData.RAWJson | ConvertFrom-Json -Depth 100 -ErrorAction SilentlyContinue
204-
$data | Add-Member -NotePropertyName 'displayName' -NotePropertyValue $JSONData.Displayname -Force
205-
$data | Add-Member -NotePropertyName 'description' -NotePropertyValue $JSONData.Description -Force
206-
$data | Add-Member -NotePropertyName 'Type' -NotePropertyValue $JSONData.Type -Force
207-
$data | Add-Member -NotePropertyName 'GUID' -NotePropertyValue $_.RowKey -Force
208-
$data
209-
} catch {
210-
# Skip invalid templates
211-
}
212-
} | Sort-Object -Property displayName
213223

214224
$TemplateIntuneTemplates = $AllIntuneTemplates | Where-Object { $_.GUID -in $IntuneTemplateIds }
215225
} catch {
@@ -222,7 +232,6 @@ function Get-CIPPDrift {
222232
$PolicyFound = $false
223233
$tenantPolicy.policy | Add-Member -MemberType NoteProperty -Name 'URLName' -Value $TenantPolicy.Type -Force
224234
$TenantPolicyName = if ($TenantPolicy.Policy.displayName) { $TenantPolicy.Policy.displayName } else { $TenantPolicy.Policy.name }
225-
226235
foreach ($TemplatePolicy in $TemplateIntuneTemplates) {
227236
$TemplatePolicyName = if ($TemplatePolicy.displayName) { $TemplatePolicy.displayName } else { $TemplatePolicy.name }
228237

0 commit comments

Comments
 (0)