Skip to content

Commit 8f04d56

Browse files
KelvinTegelaarKelvinTegelaar
committed
massive audit log update: seperate download and processing into different jobs
1 parent 1e9a80c commit 8f04d56

File tree

6 files changed

+194
-50
lines changed

6 files changed

+194
-50
lines changed

‎Modules/CIPPCore/Public/AuditLogs/Get-CippAuditLogSearches.ps1‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,6 @@ function Get-CippAuditLogSearches {
3131
return @()
3232
}
3333
$Queries = New-GraphBulkRequest -Requests @($BulkRequests) -AsApp $true -TenantId $TenantFilter | Select-Object -ExpandProperty body
34-
3534
$Queries = $Queries | Where-Object { $PendingQueries.RowKey -contains $_.id -and $_.status -eq 'succeeded' }
3635
} else {
3736
$Queries = New-GraphGetRequest -uri 'https://graph.microsoft.com/beta/security/auditLog/queries' -AsApp $true -tenantid $TenantFilter

‎Modules/CIPPCore/Public/AuditLogs/New-CIPPAuditLogSearchResultsCache.ps1‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
function New-CIPPAuditLogSearchResultsCache {
2+
<#
3+
.SYNOPSIS
4+
Cache audit log search results for more efficient processing
5+
.DESCRIPTION
6+
Retrieves audit log searches for a tenant, processes them, and stores the results in a cache table.
7+
Also tracks performance metrics for download and processing times.
8+
.PARAMETER TenantFilter
9+
The tenant to filter on.
10+
#>
11+
param (
12+
[Parameter(Mandatory = $true)]
13+
[string]$TenantFilter,
14+
[string]$SearchId
15+
)
16+
17+
try {
18+
Write-Information "Starting audit log cache process for tenant: $TenantFilter"
19+
$CacheWebhooksTable = Get-CippTable -TableName 'CacheWebhooks'
20+
$CacheWebhookStatsTable = Get-CippTable -TableName 'CacheWebhookStats'
21+
# Start tracking download time
22+
$downloadStartTime = Get-Date
23+
# Process each search and store results in cache
24+
try {
25+
Write-Information "Processing search ID: $($SearchId) for tenant: $TenantFilter"
26+
# Get the search results
27+
#check if we haven't already downloaded this search by checking the cache table, if there are items with the same search id and tenant, we skip this search
28+
$searchEntity = Get-CIPPAzDataTableEntity @CacheWebhooksTable -Filter "PartitionKey eq '$TenantFilter' and SearchId eq '$SearchId'"
29+
if ($searchEntity) {
30+
Write-Information "Search ID: $SearchId already cached for tenant: $TenantFilter"
31+
exit 0
32+
}
33+
$searchResults = Get-CippAuditLogSearchResults -TenantFilter $TenantFilter -QueryId $SearchId
34+
# Store the results in the cache table
35+
foreach ($searchResult in $searchResults) {
36+
$cacheEntity = @{
37+
RowKey = $searchResult.id
38+
PartitionKey = $TenantFilter
39+
SearchId = $SearchId
40+
JSON = [string]($searchResult | ConvertTo-Json -Depth 10)
41+
}
42+
Add-CIPPAzDataTableEntity @CacheWebhooksTable -Entity $cacheEntity -Force
43+
}
44+
Write-Information "Successfully cached search ID: $($item.id) for tenant: $TenantFilter"
45+
} catch {
46+
throw $_
47+
}
48+
49+
# Calculate download time
50+
$downloadEndTime = Get-Date
51+
$downloadSeconds = ($downloadEndTime - $downloadStartTime).TotalSeconds
52+
53+
# Store performance metrics
54+
$statsEntity = @{
55+
RowKey = $TenantFilter
56+
PartitionKey = 'Stats'
57+
DownloadSecs = [string]$downloadSeconds
58+
SearchCount = [string]$logSearches.Count
59+
}
60+
61+
Add-CIPPAzDataTableEntity @CacheWebhookStatsTable -Entity $statsEntity -Force
62+
63+
Write-Information "Completed audit log cache process for tenant: $TenantFilter. Download time: $downloadSeconds seconds"
64+
65+
return $logSearches.Count
66+
} catch {
67+
Write-Information "Error in New-CIPPAuditLogSearchResultsCache for tenant: $TenantFilter. Error: $($_.Exception.Message)"
68+
throw $_
69+
}
70+
}

Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Webhooks/Push-AuditLogTenant.ps1 renamed to Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Webhooks/Push-AuditLogTenantDownload.ps1

Lines changed: 11 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
1-
function Push-AuditLogTenant {
1+
function Push-AuditLogTenantDownload {
22
Param($Item)
33
$ConfigTable = Get-CippTable -TableName 'WebhookRules'
44
$TenantFilter = $Item.TenantFilter
55

66
try {
7-
Write-Information "Audit Logs: Processing $($TenantFilter)"
7+
Write-Information "Audit Logs: Downloading $($TenantFilter)"
88
# Get CIPP Url, cleanup legacy tasks
99
$SchedulerConfig = Get-CippTable -TableName 'SchedulerConfig'
1010
$LegacyWebhookTasks = Get-CIPPAzDataTableEntity @SchedulerConfig -Filter "PartitionKey eq 'webhookcreation'"
@@ -45,20 +45,15 @@ function Push-AuditLogTenant {
4545
if ($Configuration) {
4646
try {
4747
$LogSearches = Get-CippAuditLogSearches -TenantFilter $TenantFilter -ReadyToProcess | Select-Object -First 10
48-
Write-Information ('Audit Logs: Found {0} searches, begin processing' -f $LogSearches.Count)
48+
Write-Information ('Audit Logs: Found {0} searches, begin downloading' -f $LogSearches.Count)
4949
foreach ($Search in $LogSearches) {
5050
$SearchEntity = Get-CIPPAzDataTableEntity @LogSearchesTable -Filter "Tenant eq '$($TenantFilter)' and RowKey eq '$($Search.id)'"
5151
$SearchEntity.CippStatus = 'Processing'
5252
Add-CIPPAzDataTableEntity @LogSearchesTable -Entity $SearchEntity -Force
5353
try {
54-
# Test the audit log rules against the search results
55-
$AuditLogTest = Test-CIPPAuditLogRules -TenantFilter $TenantFilter -SearchId $Search.id
56-
57-
$SearchEntity.CippStatus = 'Completed'
58-
$MatchedRules = [string](ConvertTo-Json -Compress -InputObject $AuditLogTest.MatchedRules)
59-
$SearchEntity | Add-Member -MemberType NoteProperty -Name MatchedRules -Value $MatchedRules -Force
60-
$SearchEntity | Add-Member -MemberType NoteProperty -Name MatchedLogs -Value $AuditLogTest.MatchedLogs -Force
61-
$SearchEntity | Add-Member -MemberType NoteProperty -Name TotalLogs -Value $AuditLogTest.TotalLogs -Force
54+
Write-Information "Audit Log search: Processing search ID: $($Search.id) for tenant: $TenantFilter"
55+
$Downloads = New-CIPPAuditLogSearchResultsCache -TenantFilter $TenantFilter -searchId $Search.id
56+
$SearchEntity.CippStatus = 'Downloaded'
6257
} catch {
6358
if ($_.Exception.Message -match 'Request rate is large. More Request Units may be needed, so no changes were made. Please retry this request later.') {
6459
$SearchEntity.CippStatus = 'Pending'
@@ -74,34 +69,17 @@ function Push-AuditLogTenant {
7469
$SearchEntity.CippStatus = 'Failed'
7570
Write-Information "Error processing audit log rules: $($_.Exception.Message)"
7671
}
77-
$AuditLogTest = [PSCustomObject]@{
78-
DataToProcess = @()
79-
}
72+
8073
}
8174
Add-CIPPAzDataTableEntity @LogSearchesTable -Entity $SearchEntity -Force
82-
$DataToProcess = ($AuditLogTest).DataToProcess
83-
Write-Information "Audit Logs: Data to process found: $($DataToProcess.count) items"
84-
if ($DataToProcess) {
85-
foreach ($AuditLog in $DataToProcess) {
86-
Write-Information "Processing $($AuditLog.operation)"
87-
$Webhook = @{
88-
Data = $AuditLog
89-
CIPPURL = [string]$CIPPURL
90-
TenantFilter = $TenantFilter
91-
}
92-
try {
93-
Invoke-CippWebhookProcessing @Webhook
94-
} catch {
95-
Write-Information "Error processing webhook: $($_.Exception.Message)"
96-
}
97-
}
98-
}
9975
}
10076
} catch {
101-
Write-Information ( 'Audit Log search: Error {0} line {1} - {2}' -f $_.InvocationInfo.ScriptName, $_.InvocationInfo.ScriptLineNumber, $_.Exception.Message)
77+
Write-Information ('Audit Log search: Error {0} line {1} - {2}' -f $_.InvocationInfo.ScriptName, $_.InvocationInfo.ScriptLineNumber, $_.Exception.Message)
78+
exit 0
10279
}
10380
}
10481
} catch {
105-
Write-Information ( 'Push-AuditLogTenant: Error {0} line {1} - {2}' -f $_.InvocationInfo.ScriptName, $_.InvocationInfo.ScriptLineNumber, $_.Exception.Message)
82+
Write-Information ('Push-AuditLogTenant: Error {0} line {1} - {2}' -f $_.InvocationInfo.ScriptName, $_.InvocationInfo.ScriptLineNumber, $_.Exception.Message)
83+
exit 0
10684
}
10785
}

‎Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Webhooks/Push-AuditLogTenantProcess.ps1‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
function Push-AuditLogTenantProcess {
2+
Param($Item)
3+
$TenantFilter = $Item.TenantFilter
4+
$RowIds = $Item.RowIds
5+
6+
try {
7+
Write-Information "Audit Logs: Processing $($TenantFilter) with $($RowIds.Count) row IDs. We're processing id $($RowIds[0]) to $($RowIds[-1])"
8+
9+
# Get the CacheWebhooks table
10+
$CacheWebhooksTable = Get-CippTable -TableName 'CacheWebhooks'
11+
# we do it this way because the rows can grow extremely large, if we get them all it might just hang for minutes at a time.
12+
$Rows = foreach ($RowId in $RowIds) {
13+
$CacheEntity = Get-CIPPAzDataTableEntity @CacheWebhooksTable -Filter "PartitionKey eq '$TenantFilter' and RowKey eq '$RowId'"
14+
if ($CacheEntity) {
15+
$AuditData = $CacheEntity.JSON | ConvertFrom-Json -ErrorAction SilentlyContinue
16+
$AuditData
17+
}
18+
}
19+
20+
if ($Rows.Count -gt 0) {
21+
Write-Information "Retrieved $($Rows.Count) rows from cache for processing"
22+
Test-CIPPAuditLogRules -TenantFilter $TenantFilter -Rows $Rows
23+
} else {
24+
Write-Information 'No rows found in cache for the provided row IDs'
25+
}
26+
} catch {
27+
Write-Information ('Push-AuditLogTenant: Error {0} line {1} - {2}' -f $_.InvocationInfo.ScriptName, $_.InvocationInfo.ScriptLineNumber, $_.Exception.Message)
28+
}
29+
}

‎Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-AuditLogOrchestrator.ps1‎

Lines changed: 36 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,17 +19,48 @@ function Start-AuditLogOrchestrator {
1919
} elseif (($WebhookRules | Measure-Object).Count -eq 0) {
2020
Write-Information 'No webhook rules defined'
2121
} else {
22-
Write-Information "Audit Logs: Processing $($AuditLogSearches.Count) searches"
22+
Write-Information "Audit Logs: Downloading $($AuditLogSearches.Count) searches"
2323
if ($PSCmdlet.ShouldProcess('Start-AuditLogOrchestrator', 'Starting Audit Log Polling')) {
24-
$Queue = New-CippQueueEntry -Name 'Audit Log Collection' -Reference 'AuditLogCollection' -TotalTasks ($AuditLogSearches).Count
25-
$Batch = $AuditLogSearches | Sort-Object -Property Tenant -Unique | Select-Object @{Name = 'TenantFilter'; Expression = { $_.Tenant } }, @{Name = 'QueueId'; Expression = { $Queue.RowKey } }, @{Name = 'FunctionName'; Expression = { 'AuditLogTenant' } }
26-
24+
$Queue = New-CippQueueEntry -Name 'Audit Logs Download' -Reference 'AuditLogsDownload' -TotalTasks ($AuditLogSearches).Count
25+
$Batch = $AuditLogSearches | Sort-Object -Property Tenant -Unique | Select-Object @{Name = 'TenantFilter'; Expression = { $_.Tenant } }, @{Name = 'QueueId'; Expression = { $Queue.RowKey } }, @{Name = 'FunctionName'; Expression = { 'AuditLogTenantDownload' } }
2726
$InputObject = [PSCustomObject]@{
28-
OrchestratorName = 'AuditLogs'
27+
OrchestratorName = 'AuditLogsDownload'
2928
Batch = @($Batch)
3029
SkipLog = $true
3130
}
3231
Start-NewOrchestration -FunctionName 'CIPPOrchestrator' -InputObject ($InputObject | ConvertTo-Json -Depth 5 -Compress)
32+
Write-Information 'Starting audit log processing in batches of 1000, per tenant'
33+
$WebhookCacheTable = Get-CippTable -TableName 'CacheWebhooks'
34+
$WebhookCache = Get-CIPPAzDataTableEntity @WebhookCacheTable
35+
$TenantGroups = $WebhookCache | Group-Object -Property PartitionKey
36+
37+
if ($TenantGroups.Count -gt 0) {
38+
Write-Information "Processing webhook cache for $($TenantGroups.Count) tenants"
39+
$ProcessQueue = New-CippQueueEntry -Name 'Audit Logs Process' -Reference 'AuditLogsProcess' -TotalTasks ($TenantGroups | Measure-Object -Property Count -Sum).Sum
40+
$ProcessBatch = foreach ($TenantGroup in $TenantGroups) {
41+
$TenantFilter = $TenantGroup.Name
42+
$RowIds = $TenantGroup.Group.RowKey
43+
for ($i = 0; $i -lt $RowIds.Count; $i += 1000) {
44+
$BatchRowIds = $RowIds[$i..([Math]::Min($i + 999, $RowIds.Count - 1))]
45+
46+
[PSCustomObject]@{
47+
TenantFilter = $TenantFilter
48+
RowIds = $BatchRowIds
49+
QueueId = $ProcessQueue.RowKey
50+
FunctionName = 'AuditLogTenantProcess'
51+
}
52+
}
53+
}
54+
if ($ProcessBatch.Count -gt 0) {
55+
$ProcessInputObject = [PSCustomObject]@{
56+
OrchestratorName = 'AuditLogsProcess'
57+
Batch = @($ProcessBatch)
58+
SkipLog = $true
59+
}
60+
Start-NewOrchestration -FunctionName 'CIPPOrchestrator' -InputObject ($ProcessInputObject | ConvertTo-Json -Depth 5 -Compress)
61+
Write-Information "Started audit log processing orchestration with $($ProcessBatch.Count) batches"
62+
}
63+
}
3364
}
3465
}
3566
} catch {

‎Modules/CIPPCore/Public/Webhooks/Test-CIPPAuditLogRules.ps1‎

Lines changed: 48 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ function Test-CIPPAuditLogRules {
44
[Parameter(Mandatory = $true)]
55
$TenantFilter,
66
[Parameter(Mandatory = $true)]
7-
$SearchId
7+
$Rows
88
)
99

1010
$Results = [PSCustomObject]@{
@@ -14,6 +14,9 @@ function Test-CIPPAuditLogRules {
1414
DataToProcess = @()
1515
}
1616

17+
# Get the CacheWebhooks table for removing processed rows
18+
$CacheWebhooksTable = Get-CippTable -TableName 'CacheWebhooks'
19+
1720
$ExtendedPropertiesIgnoreList = @(
1821
'OAuth2:Authorize'
1922
'OAuth2:Token'
@@ -35,23 +38,24 @@ function Test-CIPPAuditLogRules {
3538
LogType = $_.Type
3639
}
3740
}
38-
#write-warning 'Getting audit records from Graph API'
41+
3942
try {
40-
$LogCount = Get-CippAuditLogSearchResults -TenantFilter $TenantFilter -QueryId $SearchId -CountOnly
43+
$LogCount = $Rows.count
4144
$RunGuid = (New-Guid).Guid
42-
Write-Warning "Logs to process: $LogCount - SearchId: $SearchId - RunGuid: $($RunGuid) - $($TenantFilter)"
45+
Write-Warning "Logs to process: $LogCount - RunGuid: $($RunGuid) - $($TenantFilter)"
4346
$Results.TotalLogs = $LogCount
44-
Write-Information "RunGuid: $RunGud - Collecting logs"
45-
$SearchResults = Get-CippAuditLogSearchResults -TenantFilter $TenantFilter -QueryId $SearchId
47+
Write-Information "RunGuid: $RunGuid - Collecting logs"
48+
$SearchResults = $Rows
4649
} catch {
4750
Write-Warning "Error getting audit logs: $($_.Exception.Message)"
48-
Write-LogMessage -API 'Webhooks' -message "Error getting audit logs for search $($SearchId)" -LogData (Get-CippException -Exception $_) -sev Error -tenant $TenantFilter
51+
Write-LogMessage -API 'Webhooks' -message 'Error Processing Audit logs' -LogData (Get-CippException -Exception $_) -sev Error -tenant $TenantFilter
4952
throw $_
5053
}
5154

5255
if ($LogCount -gt 0) {
5356
$LocationTable = Get-CIPPTable -TableName 'knownlocationdb'
5457
$ProcessedData = foreach ($AuditRecord in $SearchResults) {
58+
Write-Host "Auditlogs: The record is $($AuditRecord.operation) - $($TenantFilter)"
5559
$RootProperties = $AuditRecord | Select-Object * -ExcludeProperty auditData
5660
$Data = $AuditRecord.auditData | Select-Object *, CIPPAction, CIPPClause, CIPPGeoLocation, CIPPBadRepIP, CIPPHostedIP, CIPPIPDetected, CIPPLocationInfo, CIPPExtendedProperties, CIPPDeviceProperties, CIPPParameters, CIPPModifiedProperties, AuditRecord -ErrorAction SilentlyContinue
5761
try {
@@ -179,10 +183,10 @@ function Test-CIPPAuditLogRules {
179183

180184
$MatchedRules = [System.Collections.Generic.List[string]]::new()
181185
$DataToProcess = foreach ($clause in $Where) {
182-
#write-warning "Webhook: Processing clause: $($clause.clause)"
186+
Write-Warning "Webhook: Processing clause: $($clause.clause)"
183187
$ReturnedData = $ProcessedData | Where-Object { Invoke-Expression $clause.clause }
184188
if ($ReturnedData) {
185-
#write-warning "Webhook: There is matching data: $(($ReturnedData.operation | Select-Object -Unique) -join ', ')"
189+
Write-Warning "Webhook: There is matching data: $(($ReturnedData.operation | Select-Object -Unique) -join ', ')"
186190
$ReturnedData = foreach ($item in $ReturnedData) {
187191
$item.CIPPAction = $clause.expectedAction
188192
$item.CIPPClause = $clause.CIPPClause -join ' and '
@@ -196,6 +200,39 @@ function Test-CIPPAuditLogRules {
196200
$Results.MatchedLogs = ($DataToProcess | Measure-Object).Count
197201
$Results.DataToProcess = $DataToProcess
198202
}
199-
Write-Warning "Finished - RunGuid: $($RunGuid) - $($TenantFilter)"
200-
$Results
203+
204+
if ($DataToProcess) {
205+
$CippConfigTable = Get-CippTable -tablename Config
206+
$CippConfig = Get-CIPPAzDataTableEntity @CippConfigTable -Filter "PartitionKey eq 'InstanceProperties' and RowKey eq 'CIPPURL'"
207+
$CIPPURL = 'https://{0}' -f $CippConfig.Value
208+
foreach ($AuditLog in $DataToProcess) {
209+
Write-Information "Processing $($AuditLog.operation)"
210+
$Webhook = @{
211+
Data = $AuditLog
212+
CIPPURL = [string]$CIPPURL
213+
TenantFilter = $TenantFilter
214+
}
215+
try {
216+
Invoke-CippWebhookProcessing @Webhook
217+
} catch {
218+
Write-Information "Error sending final step of auditlog processing: $($_.Exception.Message)"
219+
}
220+
}
221+
}
222+
223+
# Remove processed rows from the cache table
224+
try {
225+
Write-Information 'Removing processed rows from cache'
226+
foreach ($Row in $Rows) {
227+
if ($Row.id) {
228+
$RowEntity = Get-CIPPAzDataTableEntity @CacheWebhooksTable -Filter "PartitionKey eq '$TenantFilter' and RowKey eq '$($Row.id)'"
229+
if ($RowEntity) {
230+
Remove-AzDataTableEntity @CacheWebhooksTable -Entity $RowEntity -Force
231+
Write-Information "Removed row $($Row.id) from cache"
232+
}
233+
}
234+
}
235+
} catch {
236+
Write-Information "Error removing rows from cache: $($_.Exception.Message)"
237+
}
201238
}

0 commit comments

Comments
 (0)