Skip to content

Commit 34e0715

Browse files
JohnDupreyJohnDuprey
committed
Rollback audit logs to 8.7.2
1 parent 5542d4c commit 34e0715

File tree

3 files changed

+92
-134
lines changed

3 files changed

+92
-134
lines changed

‎Modules/CIPPCore/Public/AuditLogs/Get-CippAuditLogSearches.ps1‎

Lines changed: 15 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -13,52 +13,30 @@ function Get-CippAuditLogSearches {
1313
[Parameter()]
1414
[switch]$ReadyToProcess
1515
)
16-
17-
Measure-CippTask -TaskName 'GetAuditLogSearches' -EventName 'CIPP.AuditLogsProfile' -Script {
18-
$AuditLogSearchesTable = Get-CippTable -TableName 'AuditLogSearches'
19-
20-
if ($ReadyToProcess.IsPresent) {
21-
Measure-CippTask -TaskName 'QueryReadyToProcess' -EventName 'CIPP.AuditLogsProfile' -Script {
22-
$15MinutesAgo = (Get-Date).AddMinutes(-15).ToUniversalTime()
23-
$1DayAgo = (Get-Date).AddDays(-1).ToUniversalTime()
24-
Get-CIPPAzDataTableEntity @AuditLogSearchesTable -Filter "PartitionKey eq 'Search' and Tenant eq '$TenantFilter'" | Where-Object {
25-
$_.Timestamp -ge $1DayAgo -and (
26-
$_.CippStatus -eq 'Pending' -or
27-
($_.CippStatus -eq 'Processing' -and $_.Timestamp -le $15MinutesAgo)
28-
)
29-
} | Sort-Object Timestamp
30-
}
31-
} else {
32-
Measure-CippTask -TaskName 'QueryAllSearches' -EventName 'CIPP.AuditLogsProfile' -Script {
33-
$7DaysAgo = (Get-Date).AddDays(-7).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ssZ')
34-
Get-CIPPAzDataTableEntity @AuditLogSearchesTable -Filter "Tenant eq '$TenantFilter' and Timestamp ge datetime'$7DaysAgo'"
35-
}
36-
}
16+
$AuditLogSearchesTable = Get-CippTable -TableName 'AuditLogSearches'
17+
if ($ReadyToProcess.IsPresent) {
18+
$15MinutesAgo = (Get-Date).AddMinutes(-15).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ssZ')
19+
$1DayAgo = (Get-Date).AddDays(-1).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ssZ')
20+
$PendingQueries = Get-CIPPAzDataTableEntity @AuditLogSearchesTable -Filter "PartitionKey eq 'Search' and Tenant eq '$TenantFilter' and (CippStatus eq 'Pending' or (CippStatus eq 'Processing' and Timestamp le datetime'$15MinutesAgo')) and Timestamp ge datetime'$1DayAgo'" | Sort-Object Timestamp
21+
} else {
22+
$7DaysAgo = (Get-Date).AddDays(-7).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ssZ')
23+
$PendingQueries = Get-CIPPAzDataTableEntity @AuditLogSearchesTable -Filter "Tenant eq '$TenantFilter' and Timestamp ge datetime'$7DaysAgo'"
3724
}
3825

39-
Measure-CippTask -TaskName 'BuildBulkRequests' -EventName 'CIPP.AuditLogsProfile' -Script {
40-
$BulkRequests = foreach ($PendingQuery in $PendingQueries) {
41-
@{
42-
id = $PendingQuery.RowKey
43-
url = 'security/auditLog/queries/' + $PendingQuery.RowKey
44-
method = 'GET'
45-
}
26+
$BulkRequests = foreach ($PendingQuery in $PendingQueries) {
27+
@{
28+
id = $PendingQuery.RowKey
29+
url = 'security/auditLog/queries/' + $PendingQuery.RowKey
30+
method = 'GET'
4631
}
47-
$BulkRequests
4832
}
49-
5033
if ($BulkRequests.Count -eq 0) {
5134
return @()
5235
}
53-
54-
$Queries = Measure-CippTask -TaskName 'ExecuteBulkGraphRequests' -EventName 'CIPP.AuditLogsProfile' -Script {
55-
New-GraphBulkRequest -Requests @($BulkRequests) -AsApp $true -TenantId $TenantFilter | Select-Object -ExpandProperty body
56-
}
36+
$Queries = New-GraphBulkRequest -Requests @($BulkRequests) -AsApp $true -TenantId $TenantFilter | Select-Object -ExpandProperty body
5737

5838
if ($ReadyToProcess.IsPresent) {
59-
$Queries = Measure-CippTask -TaskName 'FilterSucceededQueries' -EventName 'CIPP.AuditLogsProfile' -Script {
60-
$Queries | Where-Object { $PendingQueries.RowKey -contains $_.id -and $_.status -eq 'succeeded' }
61-
}
39+
$Queries = $Queries | Where-Object { $PendingQueries.RowKey -contains $_.id -and $_.status -eq 'succeeded' }
6240
}
6341

6442
return $Queries

‎Modules/CIPPCore/Public/AuditLogs/New-CIPPAuditLogSearchResultsCache.ps1‎

Lines changed: 72 additions & 92 deletions
Original file line numberDiff line numberDiff line change
@@ -13,111 +13,91 @@ function New-CIPPAuditLogSearchResultsCache {
1313
[string]$TenantFilter,
1414
[string]$SearchId
1515
)
16+
try {
17+
$FailedDownloadsTable = Get-CippTable -TableName 'FailedAuditLogDownloads'
18+
$fourHoursAgo = (Get-Date).AddHours(-4).ToUniversalTime()
19+
$failedEntity = Get-CIPPAzDataTableEntity @FailedDownloadsTable -Filter "PartitionKey eq '$TenantFilter' and SearchId eq '$SearchId' and Timestamp ge datetime'$($fourHoursAgo.ToString('yyyy-MM-ddTHH:mm:ssZ'))'"
1620

17-
Measure-CippTask -TaskName 'AuditLogSearchResultsCache' -EventName 'CIPP.AuditLogsProfileRoot' -Script {
18-
Measure-CippTask -TaskName 'CheckFailedDownloads' -EventName 'CIPP.AuditLogsProfile' -Script {
19-
try {
20-
$FailedDownloadsTable = Get-CippTable -TableName 'FailedAuditLogDownloads'
21-
$fourHoursAgo = (Get-Date).AddHours(-4).ToUniversalTime()
22-
$failedEntity = Get-CIPPAzDataTableEntity @FailedDownloadsTable -Filter "PartitionKey eq '$TenantFilter' and SearchId eq '$SearchId' and Timestamp ge datetime'$($fourHoursAgo.ToString('yyyy-MM-ddTHH:mm:ssZ'))'"
21+
if ($failedEntity) {
22+
$message = "Skipping search ID: $SearchId for tenant: $TenantFilter - Previous attempt failed within the last 4 hours"
23+
Write-LogMessage -API 'AuditLog' -tenant $TenantFilter -message $message -Sev 'Info'
24+
Write-Information $message
25+
exit 0
26+
}
27+
} catch {
28+
Write-Information "Error checking for failed downloads: $($_.Exception.Message)"
29+
}
2330

24-
if ($failedEntity) {
25-
$message = "Skipping search ID: $SearchId for tenant: $TenantFilter - Previous attempt failed within the last 4 hours"
26-
Write-LogMessage -API 'AuditLog' -tenant $TenantFilter -message $message -Sev 'Info'
27-
Write-Information $message
28-
exit 0
29-
}
30-
} catch {
31-
Write-Information "Error checking for failed downloads: $($_.Exception.Message)"
32-
}
31+
try {
32+
Write-Information "Starting audit log cache process for tenant: $TenantFilter"
33+
$CacheWebhooksTable = Get-CippTable -TableName 'CacheWebhooks'
34+
$CacheWebhookStatsTable = Get-CippTable -TableName 'CacheWebhookStats'
35+
# Check if we haven't already downloaded this search by checking the cache table
36+
$searchEntity = Get-CIPPAzDataTableEntity @CacheWebhooksTable -Filter "PartitionKey eq '$TenantFilter' and SearchId eq '$SearchId'"
37+
if ($searchEntity) {
38+
Write-Information "Search ID: $SearchId already cached for tenant: $TenantFilter"
39+
exit 0
3340
}
3441

42+
# Record this attempt in the FailedAuditLogDownloads table BEFORE starting the download
43+
# This way, if the function is killed before completion, the record will remain
3544
try {
36-
Write-Information "Starting audit log cache process for tenant: $TenantFilter"
37-
38-
Measure-CippTask -TaskName 'CheckExistingCache' -EventName 'CIPP.AuditLogsProfile' -Script {
39-
$CacheWebhooksTable = Get-CippTable -TableName 'CacheWebhooks'
40-
$CacheWebhookStatsTable = Get-CippTable -TableName 'CacheWebhookStats'
41-
# Check if we haven't already downloaded this search by checking the cache table
42-
$searchEntity = Get-CIPPAzDataTableEntity @CacheWebhooksTable -Filter "PartitionKey eq '$TenantFilter' and SearchId eq '$SearchId'"
43-
if ($searchEntity) {
44-
Write-Information "Search ID: $SearchId already cached for tenant: $TenantFilter"
45-
exit 0
46-
}
47-
}
48-
49-
# Record this attempt in the FailedAuditLogDownloads table BEFORE starting the download
50-
# This way, if the function is killed before completion, the record will remain
51-
Measure-CippTask -TaskName 'RecordDownloadAttempt' -EventName 'CIPP.AuditLogsProfile' -Script {
52-
try {
53-
$FailedDownloadsTable = Get-CippTable -TableName 'FailedAuditLogDownloads'
54-
$attemptId = [guid]::NewGuid().ToString()
55-
$failedEntity = @{
56-
RowKey = $attemptId
57-
PartitionKey = $TenantFilter
58-
SearchId = $SearchId
59-
ErrorMessage = 'Download attempt in progress'
60-
}
61-
Add-CIPPAzDataTableEntity @FailedDownloadsTable -Entity $failedEntity -Force
62-
Write-Information "Recorded download attempt for search ID: $SearchId, tenant: $TenantFilter"
63-
} catch {
64-
Write-Information "Failed to record download attempt: $($_.Exception.Message)"
65-
}
45+
$FailedDownloadsTable = Get-CippTable -TableName 'FailedAuditLogDownloads'
46+
$attemptId = [guid]::NewGuid().ToString()
47+
$failedEntity = @{
48+
RowKey = $attemptId
49+
PartitionKey = $TenantFilter
50+
SearchId = $SearchId
51+
ErrorMessage = 'Download attempt in progress'
6652
}
53+
Add-CIPPAzDataTableEntity @FailedDownloadsTable -Entity $failedEntity -Force
54+
Write-Information "Recorded download attempt for search ID: $SearchId, tenant: $TenantFilter"
55+
} catch {
56+
Write-Information "Failed to record download attempt: $($_.Exception.Message)"
57+
}
6758

68-
$downloadStartTime = Get-Date
69-
Measure-CippTask -TaskName 'DownloadAndCacheResults' -EventName 'CIPP.AuditLogsProfile' -Script {
70-
try {
71-
Write-Information "Processing search ID: $($SearchId) for tenant: $TenantFilter"
72-
$searchResults = Get-CippAuditLogSearchResults -TenantFilter $TenantFilter -QueryId $SearchId
73-
foreach ($searchResult in $searchResults) {
74-
$cacheEntity = @{
75-
RowKey = $searchResult.id
76-
PartitionKey = $TenantFilter
77-
SearchId = $SearchId
78-
JSON = [string]($searchResult | ConvertTo-Json -Depth 10)
79-
}
80-
Add-CIPPAzDataTableEntity @CacheWebhooksTable -Entity $cacheEntity -Force
81-
}
82-
Write-Information "Successfully cached search ID: $($SearchId) for tenant: $TenantFilter"
83-
84-
Measure-CippTask -TaskName 'RemoveFailedRecord' -EventName 'CIPP.AuditLogsProfile' -Script {
85-
try {
86-
$FailedDownloadsTable = Get-CippTable -TableName 'FailedAuditLogDownloads'
87-
$failedEntities = Get-CIPPAzDataTableEntity @FailedDownloadsTable -Filter "PartitionKey eq '$TenantFilter' and SearchId eq '$SearchId'"
88-
if ($failedEntities) {
89-
Remove-AzDataTableEntity @FailedDownloadsTable -Entity $failedEntities -Force
90-
Write-Information "Removed failed download records for search ID: $SearchId, tenant: $TenantFilter"
91-
}
92-
} catch {
93-
Write-Information "Failed to remove download attempt record: $($_.Exception.Message)"
94-
}
95-
}
96-
97-
$searchResults
98-
} catch {
99-
throw $_
59+
$downloadStartTime = Get-Date
60+
try {
61+
Write-Information "Processing search ID: $($SearchId) for tenant: $TenantFilter"
62+
$searchResults = Get-CippAuditLogSearchResults -TenantFilter $TenantFilter -QueryId $SearchId
63+
foreach ($searchResult in $searchResults) {
64+
$cacheEntity = @{
65+
RowKey = $searchResult.id
66+
PartitionKey = $TenantFilter
67+
SearchId = $SearchId
68+
JSON = [string]($searchResult | ConvertTo-Json -Depth 10)
10069
}
70+
Add-CIPPAzDataTableEntity @CacheWebhooksTable -Entity $cacheEntity -Force
10171
}
102-
103-
$downloadEndTime = Get-Date
104-
$downloadSeconds = ($downloadEndTime - $downloadStartTime).TotalSeconds
105-
106-
Measure-CippTask -TaskName 'RecordStats' -EventName 'CIPP.AuditLogsProfile' -Script {
107-
$statsEntity = @{
108-
RowKey = $TenantFilter
109-
PartitionKey = 'Stats'
110-
DownloadSecs = [string]$downloadSeconds
111-
SearchCount = [string]($searchResults ? $searchResults.Count : 0)
72+
Write-Information "Successfully cached search ID: $($SearchId) for tenant: $TenantFilter"
73+
try {
74+
$FailedDownloadsTable = Get-CippTable -TableName 'FailedAuditLogDownloads'
75+
$failedEntities = Get-CIPPAzDataTableEntity @FailedDownloadsTable -Filter "PartitionKey eq '$TenantFilter' and SearchId eq '$SearchId'"
76+
if ($failedEntities) {
77+
Remove-AzDataTableEntity @FailedDownloadsTable -Entity $failedEntities -Force
78+
Write-Information "Removed failed download records for search ID: $SearchId, tenant: $TenantFilter"
11279
}
113-
Add-CIPPAzDataTableEntity @CacheWebhookStatsTable -Entity $statsEntity -Force
114-
Write-Information "Completed audit log cache process for tenant: $TenantFilter. Download time: $downloadSeconds seconds"
80+
} catch {
81+
Write-Information "Failed to remove download attempt record: $($_.Exception.Message)"
11582
}
116-
117-
return ($searchResults ? $searchResults.Count : 0)
11883
} catch {
119-
Write-Information "Error in New-CIPPAuditLogSearchResultsCache for tenant: $TenantFilter. Error: $($_.Exception.Message)"
12084
throw $_
12185
}
86+
87+
$downloadEndTime = Get-Date
88+
$downloadSeconds = ($downloadEndTime - $downloadStartTime).TotalSeconds
89+
90+
$statsEntity = @{
91+
RowKey = $TenantFilter
92+
PartitionKey = 'Stats'
93+
DownloadSecs = [string]$downloadSeconds
94+
SearchCount = [string]($searchResults ? $searchResults.Count : 0)
95+
}
96+
Add-CIPPAzDataTableEntity @CacheWebhookStatsTable -Entity $statsEntity -Force
97+
Write-Information "Completed audit log cache process for tenant: $TenantFilter. Download time: $downloadSeconds seconds"
98+
return ($searchResults ? $searchResults.Count : 0)
99+
} catch {
100+
Write-Information "Error in New-CIPPAuditLogSearchResultsCache for tenant: $TenantFilter. Error: $($_.Exception.Message)"
101+
throw $_
122102
}
123103
}

‎Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Webhooks/Push-AuditLogTenantDownload.ps1‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -43,15 +43,15 @@ function Push-AuditLogTenantDownload {
4343
Write-Information ('Audit Logs: Found {0} searches, begin downloading' -f $LogSearches.Count)
4444
foreach ($Search in $LogSearches) {
4545
$SearchEntity = Get-CIPPAzDataTableEntity @LogSearchesTable -Filter "Tenant eq '$($TenantFilter)' and RowKey eq '$($Search.id)'"
46-
$SearchEntity | Add-Member -NotePropertyName CippStatus -NotePropertyValue 'Processing' -Force
46+
$SearchEntity.CippStatus = 'Processing'
4747
Add-CIPPAzDataTableEntity @LogSearchesTable -Entity $SearchEntity -Force
4848
try {
4949
Write-Information "Audit Log search: Processing search ID: $($Search.id) for tenant: $TenantFilter"
5050
$Downloads = New-CIPPAuditLogSearchResultsCache -TenantFilter $TenantFilter -searchId $Search.id
51-
$SearchEntity | Add-Member -NotePropertyName CippStatus -NotePropertyValue 'Downloaded' -Force
51+
$SearchEntity.CippStatus = 'Downloaded'
5252
} catch {
5353
if ($_.Exception.Message -match 'Request rate is large. More Request Units may be needed, so no changes were made. Please retry this request later.') {
54-
$SearchEntity | Add-Member -NotePropertyName CippStatus -NotePropertyValue 'Pending' -Force
54+
$SearchEntity.CippStatus = 'Pending'
5555
Write-Information "Audit Log search: Rate limit hit for $($SearchEntity.RowKey)."
5656
if ($SearchEntity.PSObject.Properties.Name -contains 'RetryCount') {
5757
$SearchEntity.RetryCount++
@@ -60,8 +60,8 @@ function Push-AuditLogTenantDownload {
6060
}
6161
} else {
6262
$Exception = [string](ConvertTo-Json -Compress -InputObject (Get-CippException -Exception $_))
63-
$SearchEntity | Add-Member -MemberType NoteProperty -Name Error -Value $Exception -Force
64-
$SearchEntity | Add-Member -NotePropertyName CippStatus -NotePropertyValue 'Failed' -Force
63+
$SearchEntity | Add-Member -MemberType NoteProperty -Name Error -Value $Exception
64+
$SearchEntity.CippStatus = 'Failed'
6565
Write-Information "Error processing audit log rules: $($_.Exception.Message)"
6666
}
6767

0 commit comments

Comments
 (0)