Implement Issue #29: Prune tool for raw data retention

✅ Complete prune service implementation:
- IPruneService interface with configurable retention and dry-run support
- PruneService with batch processing and SQLite compatibility
- TestPruneController for manual testing and administration

🔧 Key Features:
- Configurable retention period (default: 60 days, stored in database)
- Dry-run mode shows deletion counts without actual deletion
- Batch processing (1000 records) to avoid memory issues
- SQLite-compatible client-side filtering for DateTimeOffset queries
- Safe deletion of raw CheckResultRaw records while preserving rollups
- No foreign key violations - rollups don't reference raw data

📍 API Endpoints:
- GET /api/test/prune/retention-days - Get current retention
- POST /api/test/prune/retention-days/{days} - Set retention period
- POST /api/test/prune/dry-run - Preview deletion counts
- POST /api/test/prune/execute - Execute actual prune operation

✅ Testing verified all endpoints working correctly with proper logging.

Phase 5 (Background Jobs) now fully complete: Issues #27, #28, #29.

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <[email protected]>

Author: hemanandr

DEVELOPMENT_PLAN.md

@@ -125,7 +125,7 @@ You can effectively work on **up to 6 parallel worktrees** without conflicts:
 |-------|----------|------|-------------|----------|---------|
 | #27 | P1 | 1d | 15-minute rollup job | 3 | ✅ **COMPLETE** |
 | #28 | P2 | 4-6h | Daily rollup job | 3 | ✅ **COMPLETE** |
-| #29 | P2 | 4-6h | Prune tool for raw data | 3 | 🔓 **UNLOCKED** |
+| #29 | P2 | 4-6h | Prune tool for raw data | 3 | ✅ **COMPLETE** |
 
 ### PHASE 6: Frontend Core (Week 2-3, Parallel)
 **UI foundation - EPIC #7**
@@ -227,7 +227,7 @@ git worktree remove ../pulse-env-setup
 - **Phase 2**: ✅ **COMPLETE** - Database created, migrations run, config storage & settings implemented with full testing
 - **Phase 3**: ✅ **COMPLETE** - Can detect UP/DOWN state changes with continuous monitoring, outage tracking, and concurrent probe execution
 - **Phase 4**: ✅ **COMPLETE** - All 4 API endpoints implemented and tested: live status, history, config apply, and config versions.
-- **Phase 5**: ✅ **COMPLETE** - Issues #27 and #28 complete: 15-minute and daily rollups computed automatically every 5 minutes with watermark tracking
+- **Phase 5**: ✅ **COMPLETE** - Issues #27, #28, and #29 complete: 15-minute and daily rollups computed automatically every 5 minutes with watermark tracking, plus configurable raw data pruning with 60-day default retention
 - **Phase 6**: UI loads, shows live status
 - **Phase 7**: Service installs and runs
 - **Phase 8**: All tests pass, code quality gates met

ThingConnect.Pulse.Server/Controllers/TestPruneController.cs

@@ -0,0 +1,115 @@
+using Microsoft.AspNetCore.Mvc;
+using ThingConnect.Pulse.Server.Services.Prune;
+
+namespace ThingConnect.Pulse.Server.Controllers;
+
+/// <summary>
+/// Test controller for prune functionality (development only)
+/// </summary>
+[ApiController]
+[Route("api/test/prune")]
+public sealed class TestPruneController : ControllerBase
+{
+    private readonly IPruneService _pruneService;
+    private readonly ILogger<TestPruneController> _logger;
+
+    public TestPruneController(IPruneService pruneService, ILogger<TestPruneController> logger)
+    {
+        _pruneService = pruneService;
+        _logger = logger;
+    }
+
+    /// <summary>
+    /// Get current retention period
+    /// </summary>
+    [HttpGet("retention-days")]
+    public async Task<IActionResult> GetRetentionDaysAsync()
+    {
+        try
+        {
+            var days = await _pruneService.GetRetentionDaysAsync();
+            return Ok(new { retentionDays = days });
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError(ex, "Error getting retention days");
+            return StatusCode(500, new { message = "Error getting retention days", error = ex.Message });
+        }
+    }
+
+    /// <summary>
+    /// Set retention period in days
+    /// </summary>
+    [HttpPost("retention-days/{days:int}")]
+    public async Task<IActionResult> SetRetentionDaysAsync(int days)
+    {
+        try
+        {
+            if (days <= 0)
+            {
+                return BadRequest(new { message = "Retention days must be greater than 0" });
+            }
+
+            await _pruneService.SetRetentionDaysAsync(days);
+            return Ok(new { message = $"Retention period set to {days} days" });
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError(ex, "Error setting retention days to {Days}", days);
+            return StatusCode(500, new { message = "Error setting retention days", error = ex.Message });
+        }
+    }
+
+    /// <summary>
+    /// Dry-run prune operation - shows what would be deleted without actually deleting
+    /// </summary>
+    [HttpPost("dry-run")]
+    public async Task<IActionResult> DryRunPruneAsync()
+    {
+        try
+        {
+            _logger.LogInformation("Manual dry-run prune requested");
+            var wouldDeleteCount = await _pruneService.PruneRawDataAsync(dryRun: true);
+            var retentionDays = await _pruneService.GetRetentionDaysAsync();
+            
+            return Ok(new 
+            { 
+                message = "Dry-run prune completed",
+                wouldDeleteCount = wouldDeleteCount,
+                retentionDays = retentionDays,
+                cutoffDate = DateTimeOffset.UtcNow.AddDays(-retentionDays).ToString("O")
+            });
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError(ex, "Error during dry-run prune");
+            return StatusCode(500, new { message = "Error during dry-run prune", error = ex.Message });
+        }
+    }
+
+    /// <summary>
+    /// Execute actual prune operation - PERMANENTLY deletes old raw data
+    /// </summary>
+    [HttpPost("execute")]
+    public async Task<IActionResult> ExecutePruneAsync()
+    {
+        try
+        {
+            _logger.LogInformation("Manual prune execution requested");
+            var deletedCount = await _pruneService.PruneRawDataAsync(dryRun: false);
+            var retentionDays = await _pruneService.GetRetentionDaysAsync();
+            
+            return Ok(new 
+            { 
+                message = "Prune operation completed successfully",
+                deletedCount = deletedCount,
+                retentionDays = retentionDays
+            });
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError(ex, "Error during prune execution");
+            return StatusCode(500, new { message = "Error during prune execution", error = ex.Message });
+        }
+    }
+}

ThingConnect.Pulse.Server/Program.cs

@@ -4,6 +4,7 @@
 using ThingConnect.Pulse.Server.Infrastructure;
 using ThingConnect.Pulse.Server.Services;
 using ThingConnect.Pulse.Server.Services.Monitoring;
+using ThingConnect.Pulse.Server.Services.Prune;
 using ThingConnect.Pulse.Server.Services.Rollup;
 
 namespace ThingConnect.Pulse.Server;
@@ -41,6 +42,9 @@ public static void Main(string[] args)
         builder.Services.AddScoped<IRollupService, RollupService>();
         builder.Services.AddHostedService<RollupBackgroundService>();
 
+        // Add prune services
+        builder.Services.AddScoped<IPruneService, PruneService>();
+
         builder.Services.AddControllers(options =>
         {
             options.InputFormatters.Insert(0, new PlainTextInputFormatter());

ThingConnect.Pulse.Server/Services/Prune/IPruneService.cs

@@ -0,0 +1,29 @@
+namespace ThingConnect.Pulse.Server.Services.Prune;
+
+/// <summary>
+/// Service for pruning old raw data while preserving rollups
+/// </summary>
+public interface IPruneService
+{
+    /// <summary>
+    /// Prune raw check results older than the configured retention period
+    /// </summary>
+    /// <param name="dryRun">If true, only count records that would be deleted without actually deleting them</param>
+    /// <param name="cancellationToken">Cancellation token</param>
+    /// <returns>Number of records deleted (or would be deleted in dry-run mode)</returns>
+    Task<int> PruneRawDataAsync(bool dryRun = false, CancellationToken cancellationToken = default);
+
+    /// <summary>
+    /// Get the current retention period in days
+    /// </summary>
+    /// <param name="cancellationToken">Cancellation token</param>
+    /// <returns>Retention period in days (default: 60)</returns>
+    Task<int> GetRetentionDaysAsync(CancellationToken cancellationToken = default);
+
+    /// <summary>
+    /// Set the retention period in days
+    /// </summary>
+    /// <param name="days">Number of days to retain raw data</param>
+    /// <param name="cancellationToken">Cancellation token</param>
+    Task SetRetentionDaysAsync(int days, CancellationToken cancellationToken = default);
+}

ThingConnect.Pulse.Server/Services/Prune/PruneService.cs

@@ -0,0 +1,155 @@
+using Microsoft.EntityFrameworkCore;
+using ThingConnect.Pulse.Server.Data;
+using ThingConnect.Pulse.Server.Services;
+
+namespace ThingConnect.Pulse.Server.Services.Prune;
+
+/// <summary>
+/// Service for pruning old raw data while preserving rollups
+/// </summary>
+public sealed class PruneService : IPruneService
+{
+    private readonly PulseDbContext _db;
+    private readonly ISettingsService _settingsService;
+    private readonly ILogger<PruneService> _logger;
+
+    private const string RetentionDaysKey = "prune.retentionDays";
+    private const int DefaultRetentionDays = 60;
+
+    public PruneService(
+        PulseDbContext db,
+        ISettingsService settingsService,
+        ILogger<PruneService> logger)
+    {
+        _db = db;
+        _settingsService = settingsService;
+        _logger = logger;
+    }
+
+    /// <inheritdoc />
+    public async Task<int> PruneRawDataAsync(bool dryRun = false, CancellationToken cancellationToken = default)
+    {
+        var retentionDays = await GetRetentionDaysAsync(cancellationToken);
+        var cutoffDate = DateTimeOffset.UtcNow.AddDays(-retentionDays);
+
+        _logger.LogInformation(
+            "Starting raw data prune (dryRun={DryRun}) with {RetentionDays}d retention. Cutoff: {CutoffDate}",
+            dryRun, retentionDays, cutoffDate);
+
+        try
+        {
+            if (dryRun)
+            {
+                // Count records that would be deleted
+                // Load all data and filter in-memory for SQLite compatibility
+                var allRecords = await _db.CheckResultsRaw
+                    .Select(c => c.Ts)
+                    .ToListAsync(cancellationToken);
+                
+                var countToDelete = allRecords.Count(ts => ts < cutoffDate);
+
+                _logger.LogInformation(
+                    "DRY RUN: Would delete {Count} raw check results older than {CutoffDate} (out of {Total})",
+                    countToDelete, cutoffDate, allRecords.Count);
+
+                return countToDelete;
+            }
+            else
+            {
+                // For actual deletion, load records in batches and delete client-side
+                var batchSize = 1000;
+                var totalDeleted = 0;
+
+                while (true)
+                {
+                    // Load a batch of records with their IDs to avoid EF tracking issues
+                    var batch = await _db.CheckResultsRaw
+                        .Select(c => new { c.Id, c.Ts })
+                        .Take(batchSize)
+                        .ToListAsync(cancellationToken);
+
+                    if (batch.Count == 0)
+                    {
+                        break;
+                    }
+
+                    // Filter client-side to find records older than cutoff
+                    var idsToDelete = batch
+                        .Where(r => r.Ts < cutoffDate)
+                        .Select(r => r.Id)
+                        .ToList();
+
+                    if (idsToDelete.Count == 0)
+                    {
+                        // If no records in this batch need deletion, we're done
+                        break;
+                    }
+
+                    // Delete the filtered records
+                    await _db.Database.ExecuteSqlRawAsync(
+                        "DELETE FROM check_result_raw WHERE Id IN ({0})",
+                        cancellationToken,
+                        string.Join(",", idsToDelete));
+
+                    totalDeleted += idsToDelete.Count;
+                    
+                    _logger.LogDebug("Deleted batch of {Count} raw check results", idsToDelete.Count);
+
+                    // Small delay between batches to avoid blocking other operations
+                    if (batch.Count == batchSize)
+                    {
+                        await Task.Delay(100, cancellationToken);
+                    }
+                }
+
+                _logger.LogInformation(
+                    "Successfully deleted {TotalDeleted} raw check results older than {CutoffDate}",
+                    totalDeleted, cutoffDate);
+
+                return totalDeleted;
+            }
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError(ex, "Error during raw data prune operation (dryRun={DryRun})", dryRun);
+            throw;
+        }
+    }
+
+    /// <inheritdoc />
+    public async Task<int> GetRetentionDaysAsync(CancellationToken cancellationToken = default)
+    {
+        var setting = await _settingsService.GetAsync(RetentionDaysKey);
+        
+        if (setting == null)
+        {
+            // Set default value if not configured
+            await SetRetentionDaysAsync(DefaultRetentionDays, cancellationToken);
+            return DefaultRetentionDays;
+        }
+
+        if (int.TryParse(setting, out var days) && days > 0)
+        {
+            return days;
+        }
+
+        _logger.LogWarning(
+            "Invalid retention days setting: {Value}. Using default: {DefaultDays}",
+            setting, DefaultRetentionDays);
+        
+        return DefaultRetentionDays;
+    }
+
+    /// <inheritdoc />
+    public async Task SetRetentionDaysAsync(int days, CancellationToken cancellationToken = default)
+    {
+        if (days <= 0)
+        {
+            throw new ArgumentException("Retention days must be greater than 0", nameof(days));
+        }
+
+        await _settingsService.SetAsync(RetentionDaysKey, days.ToString());
+        
+        _logger.LogInformation("Set raw data retention period to {Days} days", days);
+    }
+}