Improve scheduled task handling and rerun protection

JohnDuprey · JohnDuprey · commit 73a43114af53 · 2026-01-01T11:02:58.000-05:00
Adds duplicate RowKey checks to prevent race conditions when creating scheduled tasks. Enhances rerun protection logic in Push-ExecScheduledCommand to avoid duplicate executions within recurrence intervals. Refines orchestrator task state transitions and filtering for stuck tasks. Improves logging and filtering for scheduled item listing, and updates Test-CIPPRerun to support custom intervals and base times for scheduled tasks.
diff --git a/Modules/CIPPCore/Public/Add-CIPPScheduledTask.ps1 b/Modules/CIPPCore/Public/Add-CIPPScheduledTask.ps1
@@ -46,6 +46,20 @@ function Add-CIPPScheduledTask {
                 return "Could not run task: $ErrorMessage"
             }
         } else {
+            # Generate RowKey early to use in duplicate check
+            if (!$Task.RowKey) {
+                $RowKey = (New-Guid).Guid
+            } else {
+                $RowKey = $Task.RowKey
+            }
+
+            # Check for duplicate RowKey (prevents race conditions)
+            $Filter = "PartitionKey eq 'ScheduledTask' and RowKey eq '$RowKey'"
+            $ExistingTaskByKey = (Get-CIPPAzDataTableEntity @Table -Filter $Filter)
+            if ($ExistingTaskByKey) {
+                return "Task with ID $RowKey already exists"
+            }
+
             if ($DisallowDuplicateName) {
                 $Filter = "PartitionKey eq 'ScheduledTask' and Name eq '$($Task.Name)'"
                 $ExistingTask = (Get-CIPPAzDataTableEntity @Table -Filter $Filter)
@@ -110,11 +124,8 @@ function Add-CIPPScheduledTask {
             }
             $AdditionalProperties = ([PSCustomObject]$AdditionalProperties | ConvertTo-Json -Compress)
             if ($Parameters -eq 'null') { $Parameters = '' }
-            if (!$Task.RowKey) {
-                $RowKey = (New-Guid).Guid
-            } else {
-                $RowKey = $Task.RowKey
-            }
+
+            # RowKey already generated during duplicate check above
 
             $Recurrence = if ([string]::IsNullOrEmpty($task.Recurrence.value)) {
                 $task.Recurrence
diff --git a/Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Push-ExecScheduledCommand.ps1 b/Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Push-ExecScheduledCommand.ps1
@@ -35,6 +35,48 @@ function Push-ExecScheduledCommand {
         Remove-Variable -Name ScheduledTaskId -Scope Script -ErrorAction SilentlyContinue
         return
     }
+    # Task should be 'Pending' (queued by orchestrator) or 'Running' (retry/recovery)
+    # We accept both to handle edge cases
+
+    # Check for rerun protection - prevent duplicate executions within the recurrence interval
+    if ($task.Recurrence -and $task.Recurrence -ne '0') {
+        # Calculate interval in seconds from recurrence string
+        $IntervalSeconds = switch -Regex ($task.Recurrence) {
+            '^(\d+)$' { [int64]$matches[1] * 86400 }  # Plain number = days
+            '(\d+)m$' { [int64]$matches[1] * 60 }
+            '(\d+)h$' { [int64]$matches[1] * 3600 }
+            '(\d+)d$' { [int64]$matches[1] * 86400 }
+            default { 0 }
+        }
+
+        if ($IntervalSeconds -gt 0) {
+            # Round down to nearest 15-minute interval (900 seconds) since that's when orchestrator runs
+            # This prevents rerun blocking issues due to slight timing variations
+            $FifteenMinutes = 900
+            $AdjustedInterval = [Math]::Floor($IntervalSeconds / $FifteenMinutes) * $FifteenMinutes
+
+            # Ensure we have at least one 15-minute interval
+            if ($AdjustedInterval -lt $FifteenMinutes) {
+                $AdjustedInterval = $FifteenMinutes
+            }
+            # Use task RowKey as API identifier for rerun cache
+            $RerunParams = @{
+                TenantFilter = $Tenant
+                Type         = 'ScheduledTask'
+                API          = $task.RowKey
+                Interval     = $AdjustedInterval
+                BaseTime     = [int64]$task.ScheduledTime
+                Headers      = $Headers
+            }
+
+            $IsRerun = Test-CIPPRerun @RerunParams
+            if ($IsRerun) {
+                Write-Information "Scheduled task $($task.Name) for tenant $Tenant was recently executed. Skipping to prevent duplicate execution."
+                Remove-Variable -Name ScheduledTaskId -Scope Script -ErrorAction SilentlyContinue
+                return
+            }
+        }
+    }
 
     if ($task.Trigger) {
         # Extract trigger data from the task and process
diff --git a/Modules/CIPPCore/Public/Entrypoints/HTTP Functions/CIPP/Core/Invoke-ExecAppInsightsQuery.ps1 b/Modules/CIPPCore/Public/Entrypoints/HTTP Functions/CIPP/Core/Invoke-ExecAppInsightsQuery.ps1
@@ -24,13 +24,12 @@ function Invoke-ExecAppInsightsQuery {
     try {
         $LogData = Get-ApplicationInsightsQuery -Query $Query
 
-        $Body = @{
+        $Body = ConvertTo-Json -Depth 10 -Compress -InputObject @{
             Results  = @($LogData)
             Metadata = @{
                 Query = $Query
             }
-        } | ConvertTo-Json -Depth 10 -Compress
-
+        }
         return [HttpResponseContext]@{
             StatusCode = [HttpStatusCode]::OK
             Body       = $Body
diff --git a/Modules/CIPPCore/Public/Entrypoints/HTTP Functions/CIPP/Scheduler/Invoke-AddScheduledItem.ps1 b/Modules/CIPPCore/Public/Entrypoints/HTTP Functions/CIPP/Scheduler/Invoke-AddScheduledItem.ps1
@@ -32,7 +32,7 @@ function Invoke-AddScheduledItem {
         $ScheduledTask = @{
             Task                  = $Request.Body
             Headers               = $Request.Headers
-            hidden                = $hidden
+            Hidden                = $hidden
             DisallowDuplicateName = $Request.Query.DisallowDuplicateName
             DesiredStartTime      = $Request.Body.DesiredStartTime
         }
diff --git a/Modules/CIPPCore/Public/Entrypoints/HTTP Functions/CIPP/Scheduler/Invoke-ListScheduledItems.ps1 b/Modules/CIPPCore/Public/Entrypoints/HTTP Functions/CIPP/Scheduler/Invoke-ListScheduledItems.ps1
@@ -22,9 +22,9 @@ function Invoke-ListScheduledItems {
         $SearchTitle = $Request.query.SearchTitle ?? $Request.body.SearchTitle
 
         if ($ShowHidden -eq $true) {
-            $ScheduledItemFilter.Add('Hidden eq true')
+            $ScheduledItemFilter.Add("(Hidden eq true or Hidden eq 'True')")
         } else {
-            $ScheduledItemFilter.Add('Hidden eq false')
+            $ScheduledItemFilter.Add("(Hidden eq false or Hidden eq 'False')")
         }
 
         if ($Name) {
@@ -43,6 +43,7 @@ function Invoke-ListScheduledItems {
         $HiddenTasks = $true
     }
     $Tasks = Get-CIPPAzDataTableEntity @Table -Filter $Filter
+    Write-Information "Retrieved $($Tasks.Count) scheduled tasks from storage."
     if ($Type) {
         $Tasks = $Tasks | Where-Object { $_.command -eq $Type }
     }
@@ -58,8 +59,12 @@ function Invoke-ListScheduledItems {
         $AllowedTenantDomains = $TenantList | Where-Object -Property customerId -In $AllowedTenants | Select-Object -ExpandProperty defaultDomainName
         $Tasks = $Tasks | Where-Object -Property Tenant -In $AllowedTenantDomains
     }
-    $ScheduledTasks = foreach ($Task in $tasks) {
+
+    Write-Information "Found $($Tasks.Count) scheduled tasks after filtering and access check."
+
+    $ScheduledTasks = foreach ($Task in $Tasks) {
         if (!$Task.Tenant -or !$Task.Command) {
+            Write-Information "Skipping invalid scheduled task entry: $($Task.RowKey)"
             continue
         }
 
diff --git a/Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-UserTasksOrchestrator.ps1 b/Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-UserTasksOrchestrator.ps1
@@ -10,8 +10,11 @@ function Start-UserTasksOrchestrator {
     param()
 
     $Table = Get-CippTable -tablename 'ScheduledTasks'
-    $1HourAgo = (Get-Date).AddHours(-1).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ssZ')
-    $Filter = "PartitionKey eq 'ScheduledTask' and (TaskState eq 'Planned' or TaskState eq 'Failed - Planned' or (TaskState eq 'Running' and Timestamp lt datetime'$1HourAgo'))"
+    $30MinutesAgo = (Get-Date).AddMinutes(-30).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ssZ')
+    $4HoursAgo = (Get-Date).AddHours(-4).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ssZ')
+    # Pending = orchestrator queued, Running = actively executing
+    # Pick up: Planned, Failed-Planned, stuck Pending (>30min), or stuck Running (>4hr for large AllTenants tasks)
+    $Filter = "PartitionKey eq 'ScheduledTask' and (TaskState eq 'Planned' or TaskState eq 'Failed - Planned' or (TaskState eq 'Pending' and Timestamp lt datetime'$30MinutesAgo') or (TaskState eq 'Running' and Timestamp lt datetime'$4HoursAgo'))"
     $tasks = Get-CIPPAzDataTableEntity @Table -Filter $Filter
 
     $RateLimitTable = Get-CIPPTable -tablename 'SchedulerRateLimits'
@@ -49,11 +52,14 @@ function Start-UserTasksOrchestrator {
         $currentUnixTime = [int64](([datetime]::UtcNow) - (Get-Date '1/1/1970')).TotalSeconds
         if ($currentUnixTime -ge $task.ScheduledTime) {
             try {
+                # Update task state to 'Pending' immediately to prevent concurrent orchestrator runs from picking it up
+                # 'Pending' = orchestrator has picked it up and is queuing commands
+                # 'Running' = actual execution is happening (set by Push-ExecScheduledCommand)
                 $null = Update-AzDataTableEntity -Force @Table -Entity @{
                     PartitionKey = $task.PartitionKey
                     RowKey       = $task.RowKey
                     ExecutedTime = "$currentUnixTime"
-                    TaskState    = 'Planned'
+                    TaskState    = 'Pending'
                 }
                 $task.Parameters = $task.Parameters | ConvertFrom-Json -AsHashtable
                 $task.AdditionalProperties = $task.AdditionalProperties | ConvertFrom-Json
diff --git a/Modules/CIPPCore/Public/Test-CIPPRerun.ps1 b/Modules/CIPPCore/Public/Test-CIPPRerun.ps1
@@ -7,15 +7,25 @@ function Test-CIPPRerun {
         $Settings,
         $Headers,
         [switch]$Clear,
-        [switch]$ClearAll
+        [switch]$ClearAll,
+        [int64]$Interval = 0,  # Custom interval in seconds (for scheduled tasks)
+        [int64]$BaseTime = 0   # Base time to calculate from (defaults to current time)
     )
     $RerunTable = Get-CIPPTable -tablename 'RerunCache'
-    $EstimatedDifference = switch ($Type) {
-        'Standard' { 9800 } # 2 hours 45 minutes ish.
-        'BPA' { 85000 } # 24 hours ish.
-        default { throw "Unknown type: $Type" }
+    
+    # Use custom interval if provided, otherwise use type-based defaults
+    if ($Interval -gt 0) {
+        $EstimatedDifference = $Interval
+    } else {
+        $EstimatedDifference = switch ($Type) {
+            'Standard' { 9800 } # 2 hours 45 minutes ish.
+            'BPA' { 85000 } # 24 hours ish.
+            default { throw "Unknown type: $Type" }
+        }
     }
-    $CurrentUnixTime = [int][double]::Parse((Get-Date -UFormat %s))
+    
+    # Use BaseTime if provided, otherwise use current time
+    $CurrentUnixTime = if ($BaseTime -gt 0) { $BaseTime } else { [int][double]::Parse((Get-Date -UFormat %s)) }
     $EstimatedNextRun = $CurrentUnixTime + $EstimatedDifference
 
     try {

Original file line number	Diff line number	Diff line change
`@@ -32,7 +32,7 @@ function Invoke-AddScheduledItem {`
`32`	`32`	`$ScheduledTask = @{`
`33`	`33`	`Task = $Request.Body`
`34`	`34`	`Headers = $Request.Headers`
`35`		`- hidden = $hidden`
	`35`	`+ Hidden = $hidden`
`36`	`36`	`DisallowDuplicateName = $Request.Query.DisallowDuplicateName`
`37`	`37`	`DesiredStartTime = $Request.Body.DesiredStartTime`
`38`	`38`	`}`
Original file line number	Diff line number	Diff line change
`@@ -22,9 +22,9 @@ function Invoke-ListScheduledItems {`
`22`	`22`	`$SearchTitle = $Request.query.SearchTitle ?? $Request.body.SearchTitle`
`23`	`23`
`24`	`24`	`if ($ShowHidden -eq $true) {`
`25`		`- $ScheduledItemFilter.Add('Hidden eq true')`
	`25`	`+ $ScheduledItemFilter.Add("(Hidden eq true or Hidden eq 'True')")`
`26`	`26`	`} else {`
`27`		`- $ScheduledItemFilter.Add('Hidden eq false')`
	`27`	`+ $ScheduledItemFilter.Add("(Hidden eq false or Hidden eq 'False')")`
`28`	`28`	`}`
`29`	`29`
`30`	`30`	`if ($Name) {`
`@@ -43,6 +43,7 @@ function Invoke-ListScheduledItems {`
`43`	`43`	`$HiddenTasks = $true`
`44`	`44`	`}`
`45`	`45`	`$Tasks = Get-CIPPAzDataTableEntity @Table -Filter $Filter`
	`46`	`+ Write-Information "Retrieved $($Tasks.Count) scheduled tasks from storage."`
`46`	`47`	`if ($Type) {`
`47`	`48`	`$Tasks = $Tasks \| Where-Object { $_.command -eq $Type }`
`48`	`49`	`}`
`@@ -58,8 +59,12 @@ function Invoke-ListScheduledItems {`
`58`	`59`	`$AllowedTenantDomains = $TenantList \| Where-Object -Property customerId -In $AllowedTenants \| Select-Object -ExpandProperty defaultDomainName`
`59`	`60`	`$Tasks = $Tasks \| Where-Object -Property Tenant -In $AllowedTenantDomains`
`60`	`61`	`}`
`61`		`- $ScheduledTasks = foreach ($Task in $tasks) {`
	`62`	`+`
	`63`	`+ Write-Information "Found $($Tasks.Count) scheduled tasks after filtering and access check."`
	`64`	`+`
	`65`	`+ $ScheduledTasks = foreach ($Task in $Tasks) {`
`62`	`66`	`if (!$Task.Tenant -or !$Task.Command) {`
	`67`	`+ Write-Information "Skipping invalid scheduled task entry: $($Task.RowKey)"`
`63`	`68`	`continue`
`64`	`69`	`}`
`65`	`70`