Skip to content

Commit a6660fd

Browse files
WirewraithWirewraith
committed
Convert events logs to a hypertable
1 parent e1a0d72 commit a6660fd

File tree

4 files changed

+619
-2
lines changed

4 files changed

+619
-2
lines changed

‎app/Console/Commands/EventLogsMaintenance.php‎

Lines changed: 217 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,217 @@
1+
<?php
2+
3+
namespace App\Console\Commands;
4+
5+
use Illuminate\Console\Command;
6+
use Illuminate\Support\Facades\DB;
7+
8+
/**
9+
* Maintenance command for the events_logs TimescaleDB hypertable.
10+
*
11+
* This command provides visibility into:
12+
* - Chunk status and sizes
13+
* - Compression statistics
14+
* - Data retention information
15+
*
16+
* It can also manually trigger:
17+
* - Compression of eligible chunks
18+
*/
19+
class EventLogsMaintenance extends Command
20+
{
21+
protected $signature = 'gh:events-logs-maintenance
22+
{--compress : Manually compress all eligible chunks}
23+
{--stats : Show detailed statistics}';
24+
25+
protected $description = 'Maintain the events_logs TimescaleDB hypertable';
26+
27+
public function handle(): int
28+
{
29+
$this->info('Events Logs Hypertable Maintenance');
30+
$this->newLine();
31+
32+
if ($this->option('stats')) {
33+
$this->showDetailedStats();
34+
} else {
35+
$this->showBasicStats();
36+
}
37+
38+
if ($this->option('compress')) {
39+
$this->compressChunks();
40+
}
41+
42+
// if ($this->option('retention')) {
43+
// $this->runRetention();
44+
// }
45+
46+
return self::SUCCESS;
47+
}
48+
49+
private function showBasicStats(): void
50+
{
51+
// Get hypertable info
52+
$hypertable = DB::selectOne("
53+
SELECT
54+
hypertable_name,
55+
num_chunks,
56+
pg_size_pretty(hypertable_size('events_logs')) as total_size
57+
FROM timescaledb_information.hypertables
58+
WHERE hypertable_name = 'events_logs'
59+
");
60+
61+
if (! $hypertable) {
62+
$this->error('events_logs is not a TimescaleDB hypertable');
63+
64+
return;
65+
}
66+
67+
$this->table(
68+
['Hypertable', 'Chunks', 'Total Size'],
69+
[[$hypertable->hypertable_name, $hypertable->num_chunks, $hypertable->total_size]]
70+
);
71+
72+
// Get compression stats using hypertable_compression_stats function (TimescaleDB 2.x compatible)
73+
$compression = DB::selectOne("
74+
SELECT
75+
COUNT(*) FILTER (WHERE is_compressed = true) as compressed_chunks,
76+
COUNT(*) FILTER (WHERE is_compressed = false) as uncompressed_chunks
77+
FROM timescaledb_information.chunks
78+
WHERE hypertable_name = 'events_logs'
79+
");
80+
81+
$compressionStats = DB::selectOne("
82+
SELECT
83+
pg_size_pretty(before_compression_total_bytes) as before_compression,
84+
pg_size_pretty(after_compression_total_bytes) as after_compression,
85+
CASE
86+
WHEN before_compression_total_bytes > 0
87+
THEN ROUND((1 - after_compression_total_bytes::numeric / before_compression_total_bytes::numeric) * 100, 1)
88+
ELSE 0
89+
END as compression_ratio
90+
FROM hypertable_compression_stats('events_logs')
91+
");
92+
93+
$this->newLine();
94+
$this->info('Compression Status:');
95+
$this->table(
96+
['Compressed Chunks', 'Uncompressed Chunks', 'Before Compression', 'After Compression', 'Saved'],
97+
[[
98+
$compression->compressed_chunks ?? 0,
99+
$compression->uncompressed_chunks ?? 0,
100+
$compressionStats->before_compression ?? 'N/A',
101+
$compressionStats->after_compression ?? 'N/A',
102+
$compressionStats ? ($compressionStats->compression_ratio.'%') : 'N/A',
103+
]]
104+
);
105+
106+
// Get compression and retention policies from config JSON
107+
$policy = DB::selectOne("
108+
SELECT
109+
config->>'compress_after' as compress_after
110+
FROM timescaledb_information.jobs
111+
WHERE hypertable_name = 'events_logs'
112+
AND proc_name = 'policy_compression'
113+
");
114+
115+
$retention = DB::selectOne("
116+
SELECT
117+
config->>'drop_after' as drop_after
118+
FROM timescaledb_information.jobs
119+
WHERE hypertable_name = 'events_logs'
120+
AND proc_name = 'policy_retention'
121+
");
122+
123+
$this->newLine();
124+
$this->info('Policies:');
125+
$this->line(' Compression: '.($policy->compress_after ?? 'Not configured'));
126+
$this->line(' Retention: '.($retention->drop_after ?? 'Not configured'));
127+
}
128+
129+
private function showDetailedStats(): void
130+
{
131+
$this->showBasicStats();
132+
133+
$this->newLine();
134+
$this->info('Chunk Details:');
135+
136+
// Get chunk details
137+
$chunks = DB::select("
138+
SELECT
139+
chunk_name,
140+
range_start,
141+
range_end,
142+
is_compressed
143+
FROM timescaledb_information.chunks
144+
WHERE hypertable_name = 'events_logs'
145+
ORDER BY range_start DESC
146+
LIMIT 20
147+
");
148+
149+
$this->table(
150+
['Chunk', 'Start', 'End', 'Compressed'],
151+
collect($chunks)->map(fn ($c) => [
152+
$c->chunk_name,
153+
$c->range_start,
154+
$c->range_end,
155+
$c->is_compressed ? 'Yes' : 'No',
156+
])->toArray()
157+
);
158+
159+
// Row counts
160+
$counts = DB::selectOne('
161+
SELECT
162+
COUNT(*) as total_rows,
163+
MIN(created_at) as oldest_record,
164+
MAX(created_at) as newest_record
165+
FROM events_logs
166+
');
167+
168+
$this->newLine();
169+
$this->info('Data Summary:');
170+
$this->line(' Total Rows: '.number_format($counts->total_rows));
171+
$this->line(' Oldest Record: '.($counts->oldest_record ?? 'N/A'));
172+
$this->line(' Newest Record: '.($counts->newest_record ?? 'N/A'));
173+
}
174+
175+
private function compressChunks(): void
176+
{
177+
$this->newLine();
178+
$this->info('Compressing eligible chunks...');
179+
180+
// Find uncompressed chunks that are old enough
181+
$result = DB::selectOne("
182+
SELECT compress_chunk(c.chunk_name::regclass)
183+
FROM timescaledb_information.chunks c
184+
WHERE c.hypertable_name = 'events_logs'
185+
AND c.is_compressed = false
186+
AND c.range_end < NOW() - INTERVAL '7 days'
187+
");
188+
189+
if ($result) {
190+
$this->info('Compression complete.');
191+
} else {
192+
$this->line('No chunks eligible for compression.');
193+
}
194+
}
195+
196+
// private function runRetention(): void
197+
// {
198+
// $this->newLine();
199+
// $this->info('Running retention policy...');
200+
201+
// $before = DB::selectOne("
202+
// SELECT COUNT(*) as count FROM timescaledb_information.chunks
203+
// WHERE hypertable_name = 'events_logs'
204+
// ");
205+
206+
// // Drop chunks older than retention period
207+
// DB::statement("SELECT drop_chunks('events_logs', older_than => INTERVAL '3 months')");
208+
209+
// $after = DB::selectOne("
210+
// SELECT COUNT(*) as count FROM timescaledb_information.chunks
211+
// WHERE hypertable_name = 'events_logs'
212+
// ");
213+
214+
// $dropped = $before->count - $after->count;
215+
// $this->info("Dropped {$dropped} old chunk(s).");
216+
// }
217+
}

‎app/ModelFilters/EventLogFilter.php‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,4 +21,64 @@ public function round($val)
2121
{
2222
return $this->where('round_id', $val);
2323
}
24+
25+
/**
26+
* Filter by server_id via the gameRound relationship
27+
*
28+
* @param string $val
29+
*/
30+
public function server($val)
31+
{
32+
return $this->whereHas('gameRound', function ($q) use ($val) {
33+
$q->where('server_id', $val);
34+
});
35+
}
36+
37+
/**
38+
* Filter by log type(s)
39+
*
40+
* @param string|array<string> $val
41+
*/
42+
public function type($val)
43+
{
44+
if (is_array($val)) {
45+
return $this->whereIn('type', $val);
46+
}
47+
48+
return $this->where('type', $val);
49+
}
50+
51+
/**
52+
* Full-text search using TimescaleDB GIN-indexed tsvector column
53+
*
54+
* @param string $val
55+
*/
56+
public function search($val)
57+
{
58+
if (empty(trim($val))) {
59+
return $this;
60+
}
61+
62+
return $this->whereRaw(
63+
"search_vector @@ plainto_tsquery('english', ?)",
64+
[$val]
65+
);
66+
}
67+
68+
/**
69+
* Exact phrase search using TimescaleDB GIN-indexed tsvector column
70+
*
71+
* @param string $val
72+
*/
73+
public function searchPhrase($val)
74+
{
75+
if (empty(trim($val))) {
76+
return $this;
77+
}
78+
79+
return $this->whereRaw(
80+
"search_vector @@ phraseto_tsquery('english', ?)",
81+
[$val]
82+
);
83+
}
2484
}

‎app/Models/Events/EventLog.php‎

Lines changed: 66 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
namespace App\Models\Events;
44

55
use App\Models\GameRound;
6+
use Illuminate\Database\Eloquent\Builder;
67
use Illuminate\Database\Eloquent\Factories\HasFactory;
78

89
/**
@@ -11,14 +12,15 @@
1112
* @property string|null $type
1213
* @property string|null $source
1314
* @property string|null $message
15+
* @property string|null $search_vector
1416
* @property \Illuminate\Support\Carbon|null $created_at
1517
* @property \Illuminate\Support\Carbon|null $updated_at
1618
* @property-read \Illuminate\Database\Eloquent\Collection<int, \App\Models\Audit> $audits
1719
* @property-read int|null $audits_count
1820
* @property-read \App\Models\GameRound $gameRound
1921
*
2022
* @method static \Illuminate\Database\Eloquent\Builder<static>|\App\Models\Events\EventLog filter(array $input = [], $filter = null)
21-
* @method static \Illuminate\Database\Eloquent\Builder<static>|\App\Models\Events\EventLog indexFilter(\EloquentFilter\ModelFilter|string|null $filter = null, array $default = [], string $sortBy = 'id', string $order = 'desc', int $limit = 15)
23+
* @method static \Illuminate\Database\Eloquent\Builder<static>|\App\Models\Events\EventLog indexFilter(\EloquentFilter\ModelFilter|string|null $filter = null, array $default = [], string $sortBy = 'id', string $order = 'desc')
2224
* @method static \Illuminate\Pagination\LengthAwarePaginator indexFilterPaginate(\Illuminate\Database\Eloquent\Builder $query, \EloquentFilter\ModelFilter|string|null $filter = null, array $default = [], string $sortBy = 'id', string $order = 'desc', int $perPage = 15, bool $simple = false)
2325
* @method static \Illuminate\Database\Eloquent\Builder<static>|\App\Models\Events\EventLog newModelQuery()
2426
* @method static \Illuminate\Database\Eloquent\Builder<static>|\App\Models\Events\EventLog newQuery()
@@ -35,6 +37,8 @@
3537
* @method static \Illuminate\Database\Eloquent\Builder<static>|\App\Models\Events\EventLog whereSource($value)
3638
* @method static \Illuminate\Database\Eloquent\Builder<static>|\App\Models\Events\EventLog whereType($value)
3739
* @method static \Illuminate\Database\Eloquent\Builder<static>|\App\Models\Events\EventLog whereUpdatedAt($value)
40+
* @method static \Illuminate\Database\Eloquent\Builder<static>|\App\Models\Events\EventLog search(string $term)
41+
* @method static \Illuminate\Database\Eloquent\Builder<static>|\App\Models\Events\EventLog searchPhrase(string $phrase)
3842
*
3943
* @mixin \Eloquent
4044
*/
@@ -45,10 +49,70 @@ class EventLog extends BaseEventModel
4549
protected $table = 'events_logs';
4650

4751
/**
48-
* @return \Illuminate\Database\Eloquent\Relations\BelongsTo
52+
* The attributes that should be hidden for serialization.
53+
* Hide the search_vector column from JSON responses.
54+
*
55+
* @var list<string>
56+
*/
57+
protected $hidden = ['search_vector'];
58+
59+
/**
60+
* @return \Illuminate\Database\Eloquent\Relations\BelongsTo<GameRound, $this>
4961
*/
5062
public function gameRound()
5163
{
5264
return $this->belongsTo(GameRound::class, 'round_id');
5365
}
66+
67+
/**
68+
* Scope for PostgreSQL full-text search using the GIN-indexed tsvector column.
69+
*
70+
* Uses plainto_tsquery for simple word-based searches.
71+
* Words are stemmed and stop words are removed automatically.
72+
*
73+
* @example EventLog::search('player explosion')->get()
74+
* @example EventLog::where('round_id', 123)->search('admin ban')->get()
75+
*
76+
* @param Builder<EventLog> $query
77+
* @return Builder<EventLog>
78+
*/
79+
public function scopeSearch(Builder $query, string $term): Builder
80+
{
81+
return $query->whereRaw(
82+
"search_vector @@ plainto_tsquery('english', ?)",
83+
[$term]
84+
);
85+
}
86+
87+
/**
88+
* Scope for exact phrase search using PostgreSQL full-text search.
89+
*
90+
* Uses phraseto_tsquery for phrase searches where word order matters.
91+
*
92+
* @example EventLog::searchPhrase('killed by explosion')->get()
93+
*
94+
* @param Builder<EventLog> $query
95+
* @return Builder<EventLog>
96+
*/
97+
public function scopeSearchPhrase(Builder $query, string $phrase): Builder
98+
{
99+
return $query->whereRaw(
100+
"search_vector @@ phraseto_tsquery('english', ?)",
101+
[$phrase]
102+
);
103+
}
104+
105+
/**
106+
* Scope to order results by search relevance.
107+
*
108+
* @param Builder<EventLog> $query
109+
* @return Builder<EventLog>
110+
*/
111+
public function scopeOrderByRelevance(Builder $query, string $term): Builder
112+
{
113+
return $query->orderByRaw(
114+
"ts_rank(search_vector, plainto_tsquery('english', ?)) DESC",
115+
[$term]
116+
);
117+
}
54118
}

0 commit comments

Comments
 (0)