Merge pull request #238 from KelvinTegelaar/dev

[pull] dev from KelvinTegelaar:dev

Author: pull[bot]

Cache_SAMSetup/SAMManifest.json

@@ -106,7 +106,6 @@
         { "id": "bf394140-e372-4bf9-a898-299cfc7564e5", "type": "Role" },
         { "id": "741f803b-c850-494e-b5df-cde7c675a1ca", "type": "Role" },
         { "id": "230c1aed-a721-4c5d-9cb4-a90514e508ef", "type": "Role" },
-        { "id": "b633e1c5-b582-4048-a93e-9f11b44c7e96", "type": "Role" },
         { "id": "5b567255-7703-4780-807c-7be8301ae99b", "type": "Role" },
         { "id": "62a82d76-70ea-41e2-9197-370581804d09", "type": "Role" },
         { "id": "7ab1d382-f21e-4acd-a863-ba3e13f7da61", "type": "Role" },

Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-AuditLogProcessingOrchestrator.ps1

@@ -7,36 +7,52 @@ function Start-AuditLogProcessingOrchestrator {
     param()
     Write-Information 'Starting audit log processing in batches of 1000, per tenant'
     $WebhookCacheTable = Get-CippTable -TableName 'CacheWebhooks'
-    $WebhookCache = Get-CIPPAzDataTableEntity @WebhookCacheTable
-    $TenantGroups = $WebhookCache | Group-Object -Property PartitionKey
 
-    if ($TenantGroups) {
-        Write-Information "Processing webhook cache for $($TenantGroups.Count) tenants"
-        #Write-Warning "AuditLogJobs are: $($TenantGroups.Count) tenants. Tenants: $($TenantGroups.name | ConvertTo-Json -Compress) "
-        #Write-Warning "Here are the groups: $($TenantGroups | ConvertTo-Json -Compress)"
-        $ProcessQueue = New-CippQueueEntry -Name 'Audit Logs Process' -Reference 'AuditLogsProcess' -TotalTasks ($TenantGroups | Measure-Object -Property Count -Sum).Sum
-        $ProcessBatch = foreach ($TenantGroup in $TenantGroups) {
-            $TenantFilter = $TenantGroup.Name
-            $RowIds = @($TenantGroup.Group.RowKey)
-            for ($i = 0; $i -lt $RowIds.Count; $i += 1000) {
-                Write-Host "Processing $TenantFilter with $($RowIds.Count) row IDs. We're processing id $($RowIds[$i]) to $($RowIds[[Math]::Min($i + 999, $RowIds.Count - 1)])"
-                $BatchRowIds = $RowIds[$i..([Math]::Min($i + 999, $RowIds.Count - 1))]
-                [PSCustomObject]@{
-                    TenantFilter = $TenantFilter
-                    RowIds       = $BatchRowIds
-                    QueueId      = $ProcessQueue.RowKey
-                    FunctionName = 'AuditLogTenantProcess'
+    $DataTableQuery = @{
+        First = 20000
+        Skip  = 0
+    }
+
+    do {
+        $WebhookCache = Get-CIPPAzDataTableEntity @WebhookCacheTable @DataTableQuery
+        $TenantGroups = $WebhookCache | Group-Object -Property PartitionKey
+
+        if ($TenantGroups) {
+            Write-Information "Processing webhook cache for $($TenantGroups.Count) tenants"
+            #Write-Warning "AuditLogJobs are: $($TenantGroups.Count) tenants. Tenants: $($TenantGroups.name | ConvertTo-Json -Compress) "
+            #Write-Warning "Here are the groups: $($TenantGroups | ConvertTo-Json -Compress)"
+            $ProcessQueue = New-CippQueueEntry -Name 'Audit Logs Process' -Reference 'AuditLogsProcess' -TotalTasks ($TenantGroups | Measure-Object -Property Count -Sum).Sum
+            $ProcessBatch = foreach ($TenantGroup in $TenantGroups) {
+                $TenantFilter = $TenantGroup.Name
+                $RowIds = @($TenantGroup.Group.RowKey)
+                for ($i = 0; $i -lt $RowIds.Count; $i += 1000) {
+                    Write-Host "Processing $TenantFilter with $($RowIds.Count) row IDs. We're processing id $($RowIds[$i]) to $($RowIds[[Math]::Min($i + 999, $RowIds.Count - 1)])"
+                    $BatchRowIds = $RowIds[$i..([Math]::Min($i + 999, $RowIds.Count - 1))]
+                    [PSCustomObject]@{
+                        TenantFilter = $TenantFilter
+                        RowIds       = $BatchRowIds
+                        QueueId      = $ProcessQueue.RowKey
+                        FunctionName = 'AuditLogTenantProcess'
+                    }
                 }
             }
-        }
-        if ($ProcessBatch) {
-            $ProcessInputObject = [PSCustomObject]@{
-                OrchestratorName = 'AuditLogTenantProcess'
-                Batch            = @($ProcessBatch)
-                SkipLog          = $true
+            if ($ProcessBatch) {
+                $ProcessInputObject = [PSCustomObject]@{
+                    OrchestratorName = 'AuditLogTenantProcess'
+                    Batch            = @($ProcessBatch)
+                    SkipLog          = $true
+                }
+                Start-NewOrchestration -FunctionName 'CIPPOrchestrator' -InputObject ($ProcessInputObject | ConvertTo-Json -Depth 5 -Compress)
+                Write-Information "Started audit log processing orchestration with $($ProcessBatch.Count) batches"
             }
-            Start-NewOrchestration -FunctionName 'CIPPOrchestrator' -InputObject ($ProcessInputObject | ConvertTo-Json -Depth 5 -Compress)
-            Write-Information "Started audit log processing orchestration with $($ProcessBatch.Count) batches"
         }
-    }
+
+        if ($WebhookCache.Count -lt 20000) {
+            Write-Information 'No more rows to process'
+            break
+        }
+        Write-Information "Processed $($WebhookCache.Count) rows"
+        $DataTableQuery.Skip += 20000
+        Write-Information "Getting next batch of $($DataTableQuery.First) rows"
+    } while ($WebhookCache.Count -eq 20000)
 }

Modules/CIPPCore/Public/SAMManifest.json

@@ -135,10 +135,6 @@
           "id": "19da66cb-0fb0-4390-b071-ebc76a349482",
           "type": "Role"
         },
-        {
-          "id": "b633e1c5-b582-4048-a93e-9f11b44c7e96",
-          "type": "Role"
-        },
         {
           "id": "6931bccd-447a-43d1-b442-00a195474933",
           "type": "Role"
@@ -635,4 +631,4 @@
       ]
     }
   ]
-}
+}