perf: optimize dashboard performance for large datasets

SwaggyMacro · SwaggyMacro · commit fb798becadf2 · 2025-12-15T16:37:34.000+08:00
- Add database indexes on SensorData (Timestamp, SensorName+Timestamp)
- Move statistics calculation from memory to database-level aggregation
- Implement time-based sampling for chart data on large datasets
- Fix MaintenanceService to delete old data instead of clearing all
- Add VACUUM after cleanup to reclaim SQLite space
diff --git a/FanX/Components/Pages/Dashboard.razor b/FanX/Components/Pages/Dashboard.razor
@@ -299,61 +299,79 @@
         var weekAgo = now.AddDays(-7);
         var monthAgo = now.AddDays(-30);
 
-        // Load statistics data with optimized queries
-        var allDbData = await db.Queryable<SensorData>()
-            .Where(s => s.Timestamp >= monthAgo)
-            .OrderBy(s => s.Timestamp)
+        // Use database-level aggregation instead of loading all data into memory
+        // Get distinct sensor names first
+        var tempSensorNames = await db.Queryable<SensorData>()
+            .Where(s => s.SensorType == "Temperature" && s.Timestamp >= monthAgo)
+            .Select(s => s.SensorName)
+            .Distinct()
             .ToListAsync();
-        
-        // Calculate averages
-        var tempSensors = allDbData.Where(d => d.SensorType == "Temperature").GroupBy(d => d.SensorName);
-        foreach (var group in tempSensors)
-        {
-            if (group.Key != null)
-            _tempAverages[group.Key] = (
-                group.Where(d => d.Timestamp >= hourAgo).DefaultIfEmpty().Average(d => d?.Reading ?? 0),
-                group.Where(d => d.Timestamp >= dayAgo).DefaultIfEmpty().Average(d => d?.Reading ?? 0),
-                group.Where(d => d.Timestamp >= weekAgo).DefaultIfEmpty().Average(d => d?.Reading ?? 0),
-                group.DefaultIfEmpty().Average(d => d?.Reading ?? 0),
-                group.DefaultIfEmpty().Min(d => d?.Reading ?? 0),
-                group.DefaultIfEmpty().Max(d => d?.Reading ?? 0)
-            );
-        }
 
-        var powerData = allDbData.Where(d => d.SensorName == "Pwr Consumption").ToList();
-        if (powerData.Any())
+        foreach (var sensorName in tempSensorNames.Where(n => n != null))
         {
-            _powerAverages = (
-                powerData.Where(d => d.Timestamp >= hourAgo).DefaultIfEmpty().Average(d => d?.Reading ?? 0),
-                powerData.Where(d => d.Timestamp >= dayAgo).DefaultIfEmpty().Average(d => d?.Reading ?? 0),
-                powerData.Where(d => d.Timestamp >= weekAgo).DefaultIfEmpty().Average(d => d?.Reading ?? 0),
-                powerData.DefaultIfEmpty().Average(d => d?.Reading ?? 0),
-                powerData.DefaultIfEmpty().Min(d => d?.Reading ?? 0),
-                powerData.DefaultIfEmpty().Max(d => d?.Reading ?? 0)
-            );
+            var stats = await CalculateSensorStats(db, sensorName!, hourAgo, dayAgo, weekAgo, monthAgo);
+            _tempAverages[sensorName!] = stats;
         }
-        
-        var fanSensors = allDbData.Where(d => d.SensorType == "Fan").GroupBy(d => d.SensorName);
-        foreach (var group in fanSensors)
+
+        // Power statistics
+        var powerStats = await CalculateSensorStats(db, "Pwr Consumption", hourAgo, dayAgo, weekAgo, monthAgo);
+        _powerAverages = powerStats;
+
+        // Fan statistics
+        var fanSensorNames = await db.Queryable<SensorData>()
+            .Where(s => s.SensorType == "Fan" && s.Timestamp >= monthAgo)
+            .Select(s => s.SensorName)
+            .Distinct()
+            .ToListAsync();
+
+        foreach (var sensorName in fanSensorNames.Where(n => n != null))
         {
-            if (group.Key != null)
-            _fanAverages[group.Key] = (
-                group.Where(d => d.Timestamp >= hourAgo).DefaultIfEmpty().Average(d => d?.Reading ?? 0),
-                group.Where(d => d.Timestamp >= dayAgo).DefaultIfEmpty().Average(d => d?.Reading ?? 0),
-                group.Where(d => d.Timestamp >= weekAgo).DefaultIfEmpty().Average(d => d?.Reading ?? 0),
-                group.DefaultIfEmpty().Average(d => d?.Reading ?? 0),
-                group.DefaultIfEmpty().Min(d => d?.Reading ?? 0),
-                group.DefaultIfEmpty().Max(d => d?.Reading ?? 0)
-            );
+            var stats = await CalculateSensorStats(db, sensorName!, hourAgo, dayAgo, weekAgo, monthAgo);
+            _fanAverages[sensorName!] = stats;
         }
     }
 
+    private async Task<(double Hour, double Day, double Week, double Month, double Min, double Max)> CalculateSensorStats(
+        SqlSugar.SqlSugarScope db, string sensorName, DateTime hourAgo, DateTime dayAgo, DateTime weekAgo, DateTime monthAgo)
+    {
+        // Use database aggregation - much faster than loading all data
+        var monthStats = await db.Queryable<SensorData>()
+            .Where(s => s.SensorName == sensorName && s.Timestamp >= monthAgo)
+            .Select(s => new { 
+                Avg = SqlSugar.SqlFunc.AggregateAvg(s.Reading),
+                Min = SqlSugar.SqlFunc.AggregateMin(s.Reading),
+                Max = SqlSugar.SqlFunc.AggregateMax(s.Reading)
+            })
+            .FirstAsync();
+
+        var weekAvg = await db.Queryable<SensorData>()
+            .Where(s => s.SensorName == sensorName && s.Timestamp >= weekAgo)
+            .AvgAsync(s => s.Reading);
+
+        var dayAvg = await db.Queryable<SensorData>()
+            .Where(s => s.SensorName == sensorName && s.Timestamp >= dayAgo)
+            .AvgAsync(s => s.Reading);
+
+        var hourAvg = await db.Queryable<SensorData>()
+            .Where(s => s.SensorName == sensorName && s.Timestamp >= hourAgo)
+            .AvgAsync(s => s.Reading);
+
+        return (
+            hourAvg,
+            dayAvg,
+            weekAvg,
+            monthStats?.Avg ?? 0,
+            monthStats?.Min ?? 0,
+            monthStats?.Max ?? 0
+        );
+    }
+
     private async Task LoadChartData()
     {
         // Check cache first
-        if (_chartDataCache.ContainsKey(_selectedTimeRange))
+        if (_chartDataCache.TryGetValue(_selectedTimeRange, out var cachedData))
         {
-            _chartData = _chartDataCache[_selectedTimeRange];
+            _chartData = cachedData;
             await UpdateCharts();
             return;
         }
@@ -364,14 +382,61 @@
         var now = DateTime.Now;
         var startTime = now.AddHours(-_selectedTimeRange);
 
-        // Load chart data with sampling for performance
-        var chartDataQuery = db.Queryable<SensorData>()
+        // Get total count first to determine sampling strategy
+        var totalCount = await db.Queryable<SensorData>()
             .Where(s => s.Timestamp >= startTime)
-            .OrderBy(s => s.Timestamp);
+            .CountAsync();
 
-        var rawData = await chartDataQuery.ToListAsync();
+        List<SensorData> rawData;
+        
+        if (totalCount <= MaxDataPoints * 10) // If data is manageable, load directly
+        {
+            rawData = await db.Queryable<SensorData>()
+                .Where(s => s.Timestamp >= startTime)
+                .OrderBy(s => s.Timestamp)
+                .ToListAsync();
+        }
+        else
+        {
+            // Use database-level sampling for large datasets
+            // Calculate sampling interval based on time range
+            var intervalMinutes = _selectedTimeRange switch
+            {
+                1 => 1,      // 1 hour: every minute
+                6 => 2,      // 6 hours: every 2 minutes
+                24 => 5,     // 24 hours: every 5 minutes
+                168 => 30,   // 7 days: every 30 minutes
+                720 => 120,  // 30 days: every 2 hours
+                _ => 5
+            };
+
+            // Get distinct timestamps at intervals, then fetch data for those timestamps
+            var sampledTimestamps = await db.Queryable<SensorData>()
+                .Where(s => s.Timestamp >= startTime)
+                .GroupBy(s => new { 
+                    Year = s.Timestamp.Year,
+                    Month = s.Timestamp.Month,
+                    Day = s.Timestamp.Day,
+                    Hour = s.Timestamp.Hour,
+                    MinuteGroup = s.Timestamp.Minute / intervalMinutes
+                })
+                .Select(g => SqlSugar.SqlFunc.AggregateMin(g.Timestamp))
+                .ToListAsync();
+
+            if (sampledTimestamps.Any())
+            {
+                rawData = await db.Queryable<SensorData>()
+                    .Where(s => sampledTimestamps.Contains(s.Timestamp))
+                    .OrderBy(s => s.Timestamp)
+                    .ToListAsync();
+            }
+            else
+            {
+                rawData = new List<SensorData>();
+            }
+        }
 
-        // Normalize timestamps and sample data properly
+        // Normalize timestamps and sample data if still too large
         _chartData = NormalizeAndSampleData(rawData, MaxDataPoints);
 
         // Cache the result
diff --git a/FanX/Models/SensorData.cs b/FanX/Models/SensorData.cs
@@ -3,6 +3,8 @@
 namespace FanX.Models
 {
     [SugarTable("SensorData")]
+    [SugarIndex("idx_sensordata_timestamp", nameof(Timestamp), OrderByType.Desc)]
+    [SugarIndex("idx_sensordata_sensorname_timestamp", nameof(SensorName), OrderByType.Asc, nameof(Timestamp), OrderByType.Desc)]
     public class SensorData
     {
         [SugarColumn(IsPrimaryKey = true, IsIdentity = true)]
diff --git a/FanX/Services/MaintenanceService.cs b/FanX/Services/MaintenanceService.cs
@@ -9,6 +9,7 @@ public class MaintenanceService : IHostedService, IDisposable
         private Timer? _timer;
         private DateTime _lastLogCleanup = DateTime.MinValue;
         private DateTime _lastSensorCleanup = DateTime.MinValue;
+        private const int DefaultSensorRetentionDays = 30; // Default: keep 30 days of data
 
         public MaintenanceService(IServiceProvider serviceProvider)
         {
@@ -17,8 +18,8 @@ public MaintenanceService(IServiceProvider serviceProvider)
 
         public Task StartAsync(CancellationToken cancellationToken)
         {
-            // Run every second to support debug intervals
-            _timer = new Timer(async void (_) => await DoWork(), null, TimeSpan.Zero, TimeSpan.FromSeconds(1));
+            // Run every hour for maintenance tasks (no need to run every second)
+            _timer = new Timer(async void (_) => await DoWork(), null, TimeSpan.FromMinutes(1), TimeSpan.FromHours(1));
             return Task.CompletedTask;
         }
 
@@ -28,54 +29,98 @@ private async Task DoWork()
             LoggerService.Debug($"MaintenanceService executing at {nowUtc} UTC.");
             if (!_initialized)
             {
-                // Skip cleanup on initial startup
                 _initialized = true;
                 _lastLogCleanup = nowUtc;
                 _lastSensorCleanup = nowUtc;
                 LoggerService.Debug("MaintenanceService initialization complete, skipping first cleanup.");
                 return;
             }
+            
             using var scope = _serviceProvider.CreateScope();
             var db = scope.ServiceProvider.GetRequiredService<DatabaseService>().Db;
             var settings = await db.Queryable<AppSetting>().ToListAsync();
 
-            // Periodic full log cleanup by days
-            if (settings.Any(s => s.Key == "LogRetentionDays") && int.TryParse(settings.First(s => s.Key == "LogRetentionDays").Value, out var days) && days > 0)
+            // Clean old logs daily
+            if (nowUtc - _lastLogCleanup >= TimeSpan.FromDays(1))
             {
-                if (nowUtc - _lastLogCleanup >= TimeSpan.FromDays(days))
+                var logRetentionDays = 7; // Default 7 days
+                if (settings.Any(s => s.Key == "LogRetentionDays") && 
+                    int.TryParse(settings.First(s => s.Key == "LogRetentionDays").Value, out var days) && days > 0)
                 {
-                    LoggerService.Info($"Clearing all logs (daily interval: {days} days)");
-                    ClearAllLogs();
-                    _lastLogCleanup = nowUtc;
+                    logRetentionDays = days;
                 }
+                LoggerService.Info($"Cleaning logs older than {logRetentionDays} days");
+                CleanOldLogs(logRetentionDays);
+                _lastLogCleanup = nowUtc;
             }
 
-            // Periodic full sensor data cleanup by days
-            if (settings.Any(s => s.Key == "SensorDataRetentionDays") && int.TryParse(settings.First(s => s.Key == "SensorDataRetentionDays").Value, out var sdDays) && sdDays > 0)
+            // Clean old sensor data daily - DELETE data older than retention period
+            if (nowUtc - _lastSensorCleanup >= TimeSpan.FromDays(1))
             {
-                if (nowUtc - _lastSensorCleanup >= TimeSpan.FromDays(sdDays))
+                var sensorRetentionDays = DefaultSensorRetentionDays;
+                if (settings.Any(s => s.Key == "SensorDataRetentionDays") && 
+                    int.TryParse(settings.First(s => s.Key == "SensorDataRetentionDays").Value, out var sdDays) && sdDays > 0)
+                {
+                    sensorRetentionDays = sdDays;
+                }
+                
+                var cutoffDate = DateTime.Now.AddDays(-sensorRetentionDays);
+                LoggerService.Info($"Cleaning sensor data older than {sensorRetentionDays} days (before {cutoffDate})");
+                
+                // Delete old data in batches to avoid locking the database
+                var deletedCount = await db.Deleteable<SensorData>()
+                    .Where(s => s.Timestamp < cutoffDate)
+                    .ExecuteCommandAsync();
+                    
+                if (deletedCount > 0)
                 {
-                    LoggerService.Info($"Clearing all sensor data (daily interval: {sdDays} days)");
-                    var deletedCount = await db.Deleteable<SensorData>().ExecuteCommandAsync();
-                    LoggerService.Info($"Deleted {deletedCount} sensor data entries (full clear).");
-                    _lastSensorCleanup = nowUtc;
+                    LoggerService.Info($"Deleted {deletedCount} old sensor data entries.");
+                    // Run VACUUM to reclaim space (SQLite specific)
+                    try
+                    {
+                        await db.Ado.ExecuteCommandAsync("VACUUM;");
+                        LoggerService.Info("Database vacuumed successfully.");
+                    }
+                    catch (Exception ex)
+                    {
+                        LoggerService.Warn($"Failed to vacuum database: {ex.Message}");
+                    }
                 }
+                
+                _lastSensorCleanup = nowUtc;
             }
         }
 
-        private void ClearAllLogs()
+        private void CleanOldLogs(int retentionDays)
         {
             var logDir = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Logs");
             if (!Directory.Exists(logDir)) return;
+            
+            var cutoffDate = DateTime.Now.AddDays(-retentionDays);
             var files = Directory.GetFiles(logDir, "*.log", SearchOption.AllDirectories);
+            var deletedCount = 0;
+            
             foreach (var file in files)
             {
-                try { File.Delete(file); }
+                try
+                {
+                    var fileInfo = new FileInfo(file);
+                    if (fileInfo.LastWriteTime < cutoffDate)
+                    {
+                        File.Delete(file);
+                        deletedCount++;
+                    }
+                }
                 catch
                 {
                     // ignored
                 }
             }
+            
+            if (deletedCount > 0)
+            {
+                LoggerService.Info($"Deleted {deletedCount} old log files.");
+            }
         }
 
         public Task StopAsync(CancellationToken cancellationToken)

Original file line number	Diff line number	Diff line change
`@@ -3,6 +3,8 @@`
`3`	`3`	`namespace FanX.Models`
`4`	`4`	`{`
`5`	`5`	`[SugarTable("SensorData")]`
	`6`	`+ [SugarIndex("idx_sensordata_timestamp", nameof(Timestamp), OrderByType.Desc)]`
	`7`	`+ [SugarIndex("idx_sensordata_sensorname_timestamp", nameof(SensorName), OrderByType.Asc, nameof(Timestamp), OrderByType.Desc)]`
`6`	`8`	`public class SensorData`
`7`	`9`	`{`
`8`	`10`	`[SugarColumn(IsPrimaryKey = true, IsIdentity = true)]`