Skip to content

Commit db0f8c7

Browse files
houdaslassihoudaslassi
authored
Merge pull request #38 from houdaslassi/refactor/code-improvements
Fix memory exhaustion issue with large datasets (40k+ jobs) by processing records in chunks instead of loading all into memory.
2 parents f8e3dcb + 2733c9b commit db0f8c7

File tree

4 files changed

+112
-98
lines changed

4 files changed

+112
-98
lines changed

‎src/Http/Controllers/QueueMonitorController.php‎

Lines changed: 107 additions & 95 deletions
Original file line numberDiff line numberDiff line change
@@ -113,33 +113,8 @@ public function index(Request $request)
113113
->limit(5)
114114
->get();
115115

116-
// Top tags - only select needed columns
117-
$topTags = VantageJob::select(['job_tags', 'status', 'job_class'])
118-
->where('created_at', '>', $since)
119-
->whereNotNull('job_tags')
120-
->get()
121-
->flatMap(function ($job) {
122-
return collect($job->job_tags)->map(function ($tag) use ($job) {
123-
return [
124-
'tag' => $tag,
125-
'status' => $job->status,
126-
'job_class' => $job->job_class,
127-
];
128-
});
129-
})
130-
->groupBy('tag')
131-
->map(function ($jobs, $tag) {
132-
return [
133-
'tag' => $tag,
134-
'total' => $jobs->count(),
135-
'failed' => $jobs->where('status', 'failed')->count(),
136-
'processed' => $jobs->where('status', 'processed')->count(),
137-
'processing' => $jobs->where('status', 'processing')->count(),
138-
];
139-
})
140-
->sortByDesc('total')
141-
->take(10)
142-
->values();
116+
// Top tags - use chunking to avoid memory issues with large datasets
117+
$topTags = $this->getTopTags($since, 10);
143118

144119
// Recent batches (if batch table exists)
145120
$recentBatches = collect();
@@ -360,30 +335,9 @@ public function jobs(Request $request)
360335

361336
$jobClasses = VantageJob::distinct()->pluck('job_class')->map(fn($c) => class_basename($c))->filter();
362337

363-
// Get all available tags with counts - only select needed columns
364-
$allTags = VantageJob::select(['job_tags', 'status'])
365-
->whereNotNull('job_tags')
366-
->get()
367-
->flatMap(function ($job) {
368-
return collect($job->job_tags)->map(function ($tag) use ($job) {
369-
return [
370-
'tag' => $tag,
371-
'status' => $job->status,
372-
];
373-
});
374-
})
375-
->groupBy('tag')
376-
->map(function ($jobs, $tag) {
377-
return [
378-
'tag' => $tag,
379-
'total' => $jobs->count(),
380-
'processed' => $jobs->where('status', 'processed')->count(),
381-
'failed' => $jobs->where('status', 'failed')->count(),
382-
'processing' => $jobs->where('status', 'processing')->count(),
383-
];
384-
})
385-
->sortByDesc('total')
386-
->take(50); // Limit to top 50 tags
338+
// Get all available tags with counts - use chunking to avoid memory issues
339+
// Only look at last 30 days to limit memory usage
340+
$allTags = $this->getTopTags(now()->subDays(30), 50);
387341

388342
return view('vantage::jobs', compact('jobs', 'queues', 'jobClasses', 'allTags'));
389343
}
@@ -412,50 +366,8 @@ public function tags(Request $request)
412366
$period = $request->get('period', '7d');
413367
$since = $this->getSinceDate($period);
414368

415-
// Get all jobs with tags - only select needed columns
416-
$jobs = VantageJob::select(['job_tags', 'status', 'duration_ms'])
417-
->whereNotNull('job_tags')
418-
->where('created_at', '>', $since)
419-
->get();
420-
421-
// Calculate tag statistics
422-
$tagStats = [];
423-
foreach ($jobs as $job) {
424-
foreach ($job->job_tags ?? [] as $tag) {
425-
if (!isset($tagStats[$tag])) {
426-
$tagStats[$tag] = [
427-
'total' => 0,
428-
'processed' => 0,
429-
'failed' => 0,
430-
'processing' => 0,
431-
'durations' => [],
432-
];
433-
}
434-
435-
$tagStats[$tag]['total']++;
436-
$tagStats[$tag][$job->status]++;
437-
438-
if ($job->duration_ms) {
439-
$tagStats[$tag]['durations'][] = $job->duration_ms;
440-
}
441-
}
442-
}
443-
444-
// Calculate averages and success rates
445-
foreach ($tagStats as $tag => &$stats) {
446-
$stats['avg_duration'] = !empty($stats['durations'])
447-
? round(array_sum($stats['durations']) / count($stats['durations']), 2)
448-
: 0;
449-
450-
$stats['success_rate'] = $stats['total'] > 0
451-
? round(($stats['processed'] / $stats['total']) * 100, 1)
452-
: 0;
453-
454-
unset($stats['durations']);
455-
}
456-
457-
// Sort by total count
458-
uasort($tagStats, fn($a, $b) => $b['total'] <=> $a['total']);
369+
// Use chunking to avoid memory issues with large datasets
370+
$tagStats = $this->getTagStats($since);
459371

460372
return view('vantage::tags', compact('tagStats', 'period'));
461373
}
@@ -586,6 +498,106 @@ protected function getRetryChain($job)
586498
return array_reverse($chain);
587499
}
588500

501+
/**
502+
* Get top tags using chunking to avoid memory issues
503+
*
504+
* @param \Carbon\Carbon $since
505+
* @param int $limit
506+
* @return \Illuminate\Support\Collection
507+
*/
508+
protected function getTopTags($since, int $limit = 10)
509+
{
510+
$tagStats = [];
511+
512+
VantageJob::select(['job_tags', 'status'])
513+
->where('created_at', '>', $since)
514+
->whereNotNull('job_tags')
515+
->chunk(1000, function ($jobs) use (&$tagStats) {
516+
foreach ($jobs as $job) {
517+
foreach ($job->job_tags ?? [] as $tag) {
518+
if (!isset($tagStats[$tag])) {
519+
$tagStats[$tag] = [
520+
'tag' => $tag,
521+
'total' => 0,
522+
'failed' => 0,
523+
'processed' => 0,
524+
'processing' => 0,
525+
];
526+
}
527+
$tagStats[$tag]['total']++;
528+
if (isset($tagStats[$tag][$job->status])) {
529+
$tagStats[$tag][$job->status]++;
530+
}
531+
}
532+
}
533+
});
534+
535+
return collect($tagStats)
536+
->sortByDesc('total')
537+
->take($limit)
538+
->values();
539+
}
540+
541+
/**
542+
* Get tag statistics using chunking to avoid memory issues
543+
*
544+
* @param \Carbon\Carbon $since
545+
* @return array
546+
*/
547+
protected function getTagStats($since): array
548+
{
549+
$tagStats = [];
550+
551+
VantageJob::select(['job_tags', 'status', 'duration_ms'])
552+
->whereNotNull('job_tags')
553+
->where('created_at', '>', $since)
554+
->chunk(1000, function ($jobs) use (&$tagStats) {
555+
foreach ($jobs as $job) {
556+
foreach ($job->job_tags ?? [] as $tag) {
557+
if (!isset($tagStats[$tag])) {
558+
$tagStats[$tag] = [
559+
'total' => 0,
560+
'processed' => 0,
561+
'failed' => 0,
562+
'processing' => 0,
563+
'duration_sum' => 0,
564+
'duration_count' => 0,
565+
];
566+
}
567+
568+
$tagStats[$tag]['total']++;
569+
if (isset($tagStats[$tag][$job->status])) {
570+
$tagStats[$tag][$job->status]++;
571+
}
572+
573+
if ($job->duration_ms) {
574+
$tagStats[$tag]['duration_sum'] += $job->duration_ms;
575+
$tagStats[$tag]['duration_count']++;
576+
}
577+
}
578+
}
579+
});
580+
581+
// Calculate averages and success rates
582+
foreach ($tagStats as $tag => &$stats) {
583+
$stats['avg_duration'] = $stats['duration_count'] > 0
584+
? round($stats['duration_sum'] / $stats['duration_count'], 2)
585+
: 0;
586+
587+
$stats['success_rate'] = $stats['total'] > 0
588+
? round(($stats['processed'] / $stats['total']) * 100, 1)
589+
: 0;
590+
591+
// Remove intermediate calculation fields
592+
unset($stats['duration_sum'], $stats['duration_count']);
593+
}
594+
595+
// Sort by total count
596+
uasort($tagStats, fn($a, $b) => $b['total'] <=> $a['total']);
597+
598+
return $tagStats;
599+
}
600+
589601
/**
590602
* Get since date from period string
591603
*/

‎src/Listeners/RecordJobStart.php‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
use HoudaSlassi\Vantage\Support\PayloadExtractor;
88
use HoudaSlassi\Vantage\Support\JobPerformanceContext;
99
use Illuminate\Queue\Events\JobProcessing;
10+
use Illuminate\Support\Str;
1011
use HoudaSlassi\Vantage\Models\VantageJob;
1112

1213
class RecordJobStart
@@ -93,7 +94,7 @@ protected function bestUuid(JobProcessing $event): string
9394
}
9495

9596
// Last resort: generate new UUID
96-
return (string) \Illuminate\Support\Str::uuid();
97+
return (string) Str::uuid();
9798
}
9899

99100
/**

‎src/Listeners/RecordJobSuccess.php‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -153,7 +153,7 @@ protected function bestUuid(JobProcessed $event): string
153153
}
154154

155155
// Last resort: generate new UUID
156-
return (string) \Illuminate\Support\Str::uuid();
156+
return (string) Str::uuid();
157157
}
158158

159159

‎src/Support/Traits/ExtractsRetryOf.php‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
namespace HoudaSlassi\Vantage\Support\Traits;
44

55
use HoudaSlassi\Vantage\Support\VantageLogger;
6+
use Illuminate\Support\Str;
67

78
trait ExtractsRetryOf
89
{
@@ -58,6 +59,6 @@ protected function getBestUuid($event): string
5859
return (string) $event->job->getJobId();
5960
}
6061
// Otherwise we'll generate a UUID
61-
return (string) \Illuminate\Support\Str::uuid();
62+
return (string) Str::uuid();
6263
}
6364
}

0 commit comments

Comments
 (0)