Update Start-DurableCleanup.ps1

Author: JohnDuprey

Modules/CIPPCore/Public/Entrypoints/Timer Functions/Start-DurableCleanup.ps1

@@ -17,5 +17,80 @@ function Start-DurableCleanup {
     param(
         [int]$MaxDuration = 86400
     )
-    Write-Information "This cleanup is no longer required."
+
+    $WarningPreference = 'SilentlyContinue'
+    $TargetTime = (Get-Date).ToUniversalTime().AddSeconds(-$MaxDuration)
+    $Context = New-AzDataTableContext -ConnectionString $env:AzureWebJobsStorage
+    $InstancesTables = Get-AzDataTable -Context $Context | Where-Object { $_ -match 'Instances' }
+
+    $CleanupCount = 0
+    $QueueCount = 0
+
+    $FunctionsWithLongRunningOrchestrators = [System.Collections.Generic.List[object]]::new()
+    $NonDeterministicOrchestrators = [System.Collections.Generic.List[object]]::new()
+
+    foreach ($Table in $InstancesTables) {
+        $Table = Get-CippTable -TableName $Table
+        $FunctionName = $Table.TableName -replace 'Instances', ''
+        $Orchestrators = Get-CIPPAzDataTableEntity @Table -Filter "RuntimeStatus eq 'Running'" | Select-Object * -ExcludeProperty Input
+        $Queues = Get-AzStorageQueue -Context $StorageContext -Name ('{0}*' -f $FunctionName) | Select-Object -Property Name, ApproximateMessageCount, QueueClient
+        $LongRunningOrchestrators = $Orchestrators | Where-Object { $_.CreatedTime.DateTime -lt $TargetTime }
+
+        if ($LongRunningOrchestrators.Count -gt 0) {
+            $FunctionsWithLongRunningOrchestrators.Add(@{'FunctionName' = $FunctionName })
+            foreach ($Orchestrator in $LongRunningOrchestrators) {
+                $CreatedTime = [DateTime]::SpecifyKind($Orchestrator.CreatedTime.DateTime, [DateTimeKind]::Utc)
+                $TimeSpan = New-TimeSpan -Start $CreatedTime -End (Get-Date).ToUniversalTime()
+                $RunningDuration = [math]::Round($TimeSpan.TotalMinutes, 2)
+                Write-Information "Orchestrator: $($Orchestrator.PartitionKey), created: $CreatedTime, running for: $RunningDuration minutes"
+                if ($PSCmdlet.ShouldProcess($_.PartitionKey, 'Terminate Orchestrator')) {
+                    $Orchestrator = Get-CIPPAzDataTableEntity @Table -Filter "PartitionKey eq '$($Orchestrator.PartitionKey)'"
+                    $Orchestrator.RuntimeStatus = 'Failed'
+                    if ($Orchestrator.PSObject.Properties.Name -contains 'CustomStatus') {
+                        $Orchestrator.CustomStatus = "Terminated by Durable Cleanup - Exceeded max duration of $MaxDuration seconds"
+                    } else {
+                        $Orchestrator | Add-Member -MemberType NoteProperty -Name CustomStatus -Value "Terminated by Durable Cleanup - Exceeded max duration of $MaxDuration seconds"
+                    }
+                    Update-AzDataTableEntity @Table -Entity $Orchestrator
+                    $CleanupCount++
+                }
+            }
+        }
+
+        $NonDeterministicOrchestrators = $Orchestrators | Where-Object { $_.Output -match 'Non-Deterministic workflow detected' }
+        if ($NonDeterministicOrchestrators.Count -gt 0) {
+            $NonDeterministicOrchestrators.Add(@{'FunctionName' = $FunctionName })
+            foreach ($Orchestrator in $NonDeterministicOrchestrators) {
+                Write-Information "Orchestrator: $($Orchestrator.PartitionKey) is Non-Deterministic"
+                if ($PSCmdlet.ShouldProcess($_.PartitionKey, 'Terminate Orchestrator')) {
+                    $Orchestrator = Get-CIPPAzDataTableEntity @Table -Filter "PartitionKey eq '$($Orchestrator.PartitionKey)'"
+                    $Orchestrator.RuntimeStatus = 'Failed'
+                    if ($Orchestrator.PSObject.Properties.Name -contains 'CustomStatus') {
+                        $Orchestrator.CustomStatus = 'Terminated by Durable Cleanup - Non-Deterministic workflow detected'
+                    } else {
+                        $Orchestrator | Add-Member -MemberType NoteProperty -Name CustomStatus -Value 'Terminated by Durable Cleanup - Non-Deterministic workflow detected'
+                    }
+                    Update-AzDataTableEntity @Table -Entity $Orchestrator
+                    $CleanupCount++
+                }
+            }
+        }
+
+        if (($LongRunningOrchestrators.Count -gt 0 -or $NonDeterministicOrchestrators.Count -gt 0) -and $Queues.ApproximateMessageCount -gt 0) {
+            $RunningQueues = $Queues | Where-Object { $_.ApproximateMessageCount -gt 0 }
+            foreach ($Queue in $RunningQueues) {
+                Write-Information "- Removing queue: $($Queue.Name), message count: $($Queue.ApproximateMessageCount)"
+                if ($PSCmdlet.ShouldProcess($Queue.Name, 'Clear Queue')) {
+                    $Queue.QueueClient.ClearMessagesAsync() | Out-Null
+                }
+                $QueueCount++
+            }
+        }
+    }
+
+    if ($CleanupCount -gt 0 -or $QueueCount -gt 0) {
+        Write-LogMessage -api 'Durable Cleanup' -message "$CleanupCount orchestrators were terminated. $QueueCount queues were cleared." -sev 'Info' -LogData $FunctionsWithLongRunningOrchestrators
+    }
+
+    Write-Information "Durable cleanup complete. $CleanupCount orchestrators were terminated. $QueueCount queues were cleared."
 }