Skip to content

Commit bbd4241

Browse files
KelvinTegelaarKelvinTegelaar
committed
performance improvements
1 parent 3eb2fb2 commit bbd4241

File tree

1 file changed

+94
-40
lines changed

1 file changed

+94
-40
lines changed

‎Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Webhooks/Push-AuditLogIngestion.ps1‎

Lines changed: 94 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -10,36 +10,67 @@ function Push-AuditLogIngestion {
1010
$ContentTypes = $Item.ContentTypes
1111

1212
try {
13+
if (!$ContentTypes -or $ContentTypes.Count -eq 0) {
14+
Write-LogMessage -API 'AuditLogIngestion' -tenant $TenantFilter -message 'No content types specified' -sev Warn
15+
return $true
16+
}
17+
1318
Write-LogMessage -API 'AuditLogIngestion' -tenant $TenantFilter -message 'Starting Management API ingestion for tenant' -sev Info
19+
1420
$AuditLogStateTable = Get-CippTable -TableName 'AuditLogState'
1521
$CacheWebhooksTable = Get-CippTable -TableName 'CacheWebhooks'
16-
Write-LogMessage -API 'AuditLogIngestion' -tenant $TenantFilter -message "Content types to process: $($ContentTypes -join ', ')" -sev Info
22+
23+
$StateCache = @{}
24+
$StateUpdates = @{}
1725
foreach ($ContentType in $ContentTypes) {
1826
$StateRowKey = "$TenantFilter-$ContentType"
1927
$StateEntity = Get-CIPPAzDataTableEntity @AuditLogStateTable -Filter "PartitionKey eq 'AuditLogState' and RowKey eq '$StateRowKey'"
28+
$StateCache[$ContentType] = $StateEntity
29+
}
30+
31+
Write-LogMessage -API 'AuditLogIngestion' -tenant $TenantFilter -message "Content types to process: $($ContentTypes -join ', ')" -sev Info
32+
33+
$ContentTypesNeedingSubscription = [System.Collections.Generic.List[string]]::new()
34+
$EnabledContentTypes = [System.Collections.Generic.List[string]]::new()
35+
36+
foreach ($ContentType in $ContentTypes) {
37+
$StateEntity = $StateCache[$ContentType]
38+
2039
if ($StateEntity -and $StateEntity.SubscriptionEnabled) {
2140
Write-LogMessage -API 'AuditLogIngestion' -tenant $TenantFilter -message "Subscription already enabled for $ContentType" -sev Debug
41+
$EnabledContentTypes.Add($ContentType)
2242
continue
2343
}
44+
45+
$ContentTypesNeedingSubscription.Add($ContentType)
46+
}
47+
48+
foreach ($ContentType in $ContentTypesNeedingSubscription) {
49+
$StateRowKey = "$TenantFilter-$ContentType"
50+
$StateEntity = $StateCache[$ContentType]
51+
2452
$SubscriptionUri = "https://manage.office.com/api/v1.0/$TenantId/activity/feed/subscriptions/start?contentType=$ContentType"
2553
$SubscriptionParams = @{
2654
scope = 'https://manage.office.com/.default'
2755
Uri = $SubscriptionUri
2856
Method = 'POST'
2957
TenantId = $TenantFilter
3058
}
59+
3160
try {
3261
Write-LogMessage -API 'AuditLogIngestion' -tenant $TenantFilter -message "Starting subscription for $ContentType" -sev Debug
3362
$null = New-GraphPostRequest @SubscriptionParams -ErrorAction Stop
3463
Write-LogMessage -API 'AuditLogIngestion' -tenant $TenantFilter -message "Successfully started subscription for $ContentType" -sev Info
3564

36-
$StateEntity = @{
37-
PartitionKey = 'AuditLogState'
38-
RowKey = $StateRowKey
39-
SubscriptionEnabled = $true
40-
ContentType = $ContentType
65+
if (!$StateUpdates[$ContentType]) {
66+
$StateUpdates[$ContentType] = @{
67+
PartitionKey = 'AuditLogState'
68+
RowKey = $StateRowKey
69+
ContentType = $ContentType
70+
}
4171
}
42-
Add-CIPPAzDataTableEntity @AuditLogStateTable -Entity $StateEntity -Force
72+
$StateUpdates[$ContentType].SubscriptionEnabled = $true
73+
$EnabledContentTypes.Add($ContentType)
4374

4475
} catch {
4576
if ($_.Exception.Message -match 'AADSTS65001') {
@@ -52,42 +83,55 @@ function Push-AuditLogIngestion {
5283
Update-CippSamPermissions -UpdatedBy 'CIPP-API'
5384
Write-Host "Re-adding delegated permission for tenant: $TenantFilter"
5485
Add-CIPPDelegatedPermission -RequiredResourceAccess 'CIPPDefaults' -ApplicationId $env:ApplicationID -tenantfilter $TenantFilter
55-
$StateEntity = @{
56-
PartitionKey = 'AuditLogState'
57-
RowKey = "$TenantFilter-$ContentType"
58-
PermissionsUpdated = $true
59-
ContentType = $ContentType
86+
87+
if (!$StateUpdates[$ContentType]) {
88+
$StateUpdates[$ContentType] = @{
89+
PartitionKey = 'AuditLogState'
90+
RowKey = $StateRowKey
91+
ContentType = $ContentType
92+
}
6093
}
61-
Add-CIPPAzDataTableEntity @AuditLogStateTable -Entity $StateEntity
62-
return $true
94+
$StateUpdates[$ContentType].PermissionsUpdated = $true
95+
continue
6396
}
6497

6598
if ($_.Exception.Message -match 'already enabled|already exists|AF20024') {
6699
Write-LogMessage -API 'AuditLogIngestion' -tenant $TenantFilter -message "Subscription already exists for $ContentType" -sev Debug
67100

68-
$StateEntity = @{
69-
PartitionKey = 'AuditLogState'
70-
RowKey = $StateRowKey
71-
SubscriptionEnabled = $true
72-
ContentType = $ContentType
101+
if (!$StateUpdates[$ContentType]) {
102+
$StateUpdates[$ContentType] = @{
103+
PartitionKey = 'AuditLogState'
104+
RowKey = $StateRowKey
105+
ContentType = $ContentType
106+
}
73107
}
74-
Add-CIPPAzDataTableEntity @AuditLogStateTable -Entity $StateEntity -Force
108+
$StateUpdates[$ContentType].SubscriptionEnabled = $true
109+
$EnabledContentTypes.Add($ContentType)
75110

76111
} else {
77112
Write-LogMessage -API 'AuditLogIngestion' -tenant $TenantFilter -message "Failed to start subscription for $ContentType : $($_.Exception.Message)" -sev Error
78113
}
79114
}
80115
}
81116

117+
if ($EnabledContentTypes.Count -eq 0) {
118+
Write-LogMessage -API 'AuditLogIngestion' -tenant $TenantFilter -message 'No enabled content types to process' -sev Warn
119+
if ($StateUpdates.Count -gt 0) {
120+
$UpdateEntities = @($StateUpdates.Values)
121+
Add-CIPPAzDataTableEntity @AuditLogStateTable -Entity $UpdateEntities -Force
122+
}
123+
return $true
124+
}
125+
82126
$TotalProcessedRecords = 0
83127
$Now = Get-Date
84128

85-
foreach ($ContentType in $ContentTypes) {
129+
foreach ($ContentType in $EnabledContentTypes) {
86130
try {
87131
Write-LogMessage -API 'AuditLogIngestion' -tenant $TenantFilter -message "Processing content type: $ContentType" -sev Debug
88132

89133
$StateRowKey = "$TenantFilter-$ContentType"
90-
$StateEntity = Get-CIPPAzDataTableEntity @AuditLogStateTable -Filter "PartitionKey eq 'AuditLogState' and RowKey eq '$StateRowKey'"
134+
$StateEntity = $StateCache[$ContentType]
91135

92136
if ($StateEntity -and $StateEntity.LastContentCreatedUtc) { $StartTime = ([DateTime]$StateEntity.LastContentCreatedUtc).AddMinutes(-5).ToUniversalTime() } else { $StartTime = $Now.AddHours(-1).ToUniversalTime() }
93137
$EndTime = $Now.AddMinutes(-5).ToUniversalTime()
@@ -164,19 +208,23 @@ function Push-AuditLogIngestion {
164208

165209
Write-LogMessage -API 'AuditLogIngestion' -tenant $TenantFilter -message "Caching $($AuditRecords.Count) audit records for $ContentType" -sev Debug
166210

211+
$CacheEntities = [System.Collections.Generic.List[hashtable]]::new()
167212
foreach ($Record in $AuditRecords) {
168-
try {
169-
$CacheEntity = @{
213+
$CacheEntities.Add(@{
170214
RowKey = $Record.Id
171215
PartitionKey = $TenantFilter
172216
JSON = [string]($Record | ConvertTo-Json -Depth 10 -Compress)
173217
ContentId = $ContentItem.contentId
174218
ContentType = $ContentType
175-
}
176-
Add-CIPPAzDataTableEntity @CacheWebhooksTable -Entity $CacheEntity -Force
177-
$ProcessedRecords++
219+
})
220+
}
221+
222+
if ($CacheEntities.Count -gt 0) {
223+
try {
224+
Add-CIPPAzDataTableEntity @CacheWebhooksTable -Entity $CacheEntities -Force
225+
$ProcessedRecords += $CacheEntities.Count
178226
} catch {
179-
Write-LogMessage -API 'AuditLogIngestion' -tenant $TenantFilter -message "Failed to cache record for $ContentType : $($_.Exception.Message)" -sev Error
227+
Write-LogMessage -API 'AuditLogIngestion' -tenant $TenantFilter -message "Failed to batch cache records for $ContentType : $($_.Exception.Message)" -sev Error
180228
}
181229
}
182230

@@ -195,27 +243,33 @@ function Push-AuditLogIngestion {
195243
Write-LogMessage -API 'AuditLogIngestion' -tenant $TenantFilter -message "Cached $ProcessedRecords audit records for $ContentType" -sev Info
196244
$TotalProcessedRecords += $ProcessedRecords
197245

198-
$Entities = if ($LatestContentCreated) {
199-
$StateEntity = @{
200-
PartitionKey = 'AuditLogState'
201-
RowKey = $StateRowKey
202-
PermissionsUpdated = $true
203-
SubscriptionEnabled = $true
204-
LastContentCreatedUtc = $LatestContentCreated.ToString('yyyy-MM-ddTHH:mm:ss.fffZ')
205-
LastContentId = $LatestContentId
206-
LastProcessedUtc = $Now.ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ss.fffZ')
207-
ContentType = $ContentType
246+
if ($LatestContentCreated) {
247+
if (!$StateUpdates[$ContentType]) {
248+
$StateUpdates[$ContentType] = @{
249+
PartitionKey = 'AuditLogState'
250+
RowKey = $StateRowKey
251+
ContentType = $ContentType
252+
}
208253
}
254+
$StateUpdates[$ContentType].SubscriptionEnabled = $true
255+
$StateUpdates[$ContentType].LastContentCreatedUtc = $LatestContentCreated.ToString('yyyy-MM-ddTHH:mm:ss.fffZ')
256+
$StateUpdates[$ContentType].LastContentId = $LatestContentId
257+
$StateUpdates[$ContentType].LastProcessedUtc = $Now.ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ss.fffZ')
209258

210-
Write-LogMessage -API 'AuditLogIngestion' -tenant $TenantFilter -message "Updated watermark for $ContentType to $($StateEntity.LastContentCreatedUtc)" -sev Debug
259+
Write-LogMessage -API 'AuditLogIngestion' -tenant $TenantFilter -message "Updated watermark for $ContentType to $($LatestContentCreated.ToString('yyyy-MM-ddTHH:mm:ss.fffZ'))" -sev Debug
211260
}
212-
Add-CIPPAzDataTableEntity @AuditLogStateTable -Entity $Entities -Force
261+
213262
} catch {
214263
Write-LogMessage -API 'AuditLogIngestion' -tenant $TenantFilter -message "Error processing content type $ContentType : $($_.Exception.Message)" -sev Error -LogData (Get-CippException -Exception $_)
215264
continue
216265
}
217266
}
218267

268+
if ($StateUpdates.Count -gt 0) {
269+
$UpdateEntities = @($StateUpdates.Values)
270+
Add-CIPPAzDataTableEntity @AuditLogStateTable -Entity $UpdateEntities -Force
271+
}
272+
219273
Write-LogMessage -API 'AuditLogIngestion' -tenant $TenantFilter -message "Completed ingestion: $TotalProcessedRecords total records cached" -sev Info
220274

221275
return $true

0 commit comments

Comments
 (0)