-
-
Notifications
You must be signed in to change notification settings - Fork 362
Add query logging #3934
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
+360
−1
Merged
Add query logging #3934
Changes from all commits
Commits
Show all changes
3 commits
Select commit
Hold shift + click to select a range
File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Some comments aren't visible on the classic Files Changed page.
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,60 @@ | ||
| <?php | ||
|
|
||
| /** | ||
| * SPDX-License-Identifier: MIT | ||
| * Copyright (c) 2017-2018 Tobias Reich | ||
| * Copyright (c) 2018-2026 LycheeOrg. | ||
| */ | ||
|
|
||
| namespace App\Listeners; | ||
|
|
||
| use Illuminate\Database\Events\QueryExecuted; | ||
| use Illuminate\Support\Facades\Log; | ||
|
|
||
| /** | ||
| * Tracks long-running SQL queries that may timeout. | ||
| * | ||
| * This listener works by storing query start information in cache | ||
| * and checking execution time after completion. | ||
| */ | ||
| class LogQueryTimeout | ||
| { | ||
| /** | ||
| * Handle the event when a query is executed. | ||
| * | ||
| * @param QueryExecuted $event | ||
| * | ||
| * @return void | ||
| */ | ||
| public function handle(QueryExecuted $event): void | ||
| { | ||
| $max_execution_time = config('octane.max_execution_time', 30) * 1000; // convert to ms | ||
| $critical_threshold = $max_execution_time * 0.9; // 90% of timeout | ||
| $warning_threshold = $max_execution_time * 0.7; // 70% of timeout | ||
|
|
||
| // Log if query is dangerously slow | ||
| if ($event->time >= $critical_threshold) { | ||
| Log::error('🚨 CRITICAL: Query approaching timeout', [ | ||
| 'execution_time_ms' => $event->time, | ||
| 'execution_time_s' => round($event->time / 1000, 2), | ||
| 'timeout_limit_s' => $max_execution_time / 1000, | ||
| 'percentage' => round(($event->time / $max_execution_time) * 100, 1) . '%', | ||
| 'sql' => $event->sql, | ||
| 'bindings' => $event->bindings, | ||
| 'connection' => $event->connectionName, | ||
| 'url' => request()?->fullUrl() ?? 'N/A', | ||
| ]); | ||
| } elseif ($event->time >= $warning_threshold) { | ||
| Log::warning('⚠️ WARNING: Slow query detected', [ | ||
| 'execution_time_ms' => $event->time, | ||
| 'execution_time_s' => round($event->time / 1000, 2), | ||
| 'timeout_limit_s' => $max_execution_time / 1000, | ||
| 'percentage' => round(($event->time / $max_execution_time) * 100, 1) . '%', | ||
| 'sql' => $event->sql, | ||
| 'bindings' => $event->bindings, | ||
| 'connection' => $event->connectionName, | ||
| 'url' => request()?->fullUrl() ?? 'N/A', | ||
| ]); | ||
| } | ||
| } | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,140 @@ | ||
| # SQL Timeout Logging | ||
|
|
||
| This document explains how to log SQL requests that are timing out or approaching timeout limits in Lychee. | ||
|
|
||
| ## Overview | ||
|
|
||
| When running with Laravel Octane (FrankenPHP/Swoole/RoadRunner), requests have a maximum execution time (default: 30 seconds as configured in `config/octane.php`). SQL queries that take too long can cause the entire request to timeout. | ||
|
|
||
| The logging system tracks: | ||
| 1. **Slow queries** - Queries taking longer than configured threshold | ||
| 2. **Queries approaching timeout** - Queries using >70% of max execution time | ||
| 3. **Critical queries** - Queries using >90% of max execution time | ||
| 4. **PHP timeouts** - When the entire request times out | ||
|
|
||
| ## Configuration | ||
|
|
||
| ### Enable SQL Logging | ||
|
|
||
| Set the following in your `.env` file: | ||
|
|
||
| ```env | ||
| # Enable SQL query logging | ||
| DB_LOG_SQL=true | ||
| # Optional: Minimum execution time to log (in milliseconds, default: 100) | ||
| DB_LOG_SQL_MIN_TIME=100 | ||
| # Optional: Enable EXPLAIN for MySQL SELECT queries | ||
| DB_LOG_SQL_EXPLAIN=true | ||
| ``` | ||
|
|
||
| ### Timeout Settings | ||
|
|
||
| The max execution time is configured in `config/octane.php`: | ||
|
|
||
| ```php | ||
| 'max_execution_time' => 30, // seconds | ||
| ``` | ||
|
|
||
| ## How It Works | ||
|
|
||
| ### 1. Query Execution Logging | ||
|
|
||
| **Location**: [app/Providers/AppServiceProvider.php:238-300](app/Providers/AppServiceProvider.php#L238-L300) | ||
|
|
||
| The `logSQL()` method logs queries after they complete with severity based on execution time: | ||
| - **Debug**: Normal slow queries (>100ms) | ||
| - **Warning**: Queries taking >1 second | ||
| - **Error**: Queries approaching timeout (>90% of max_execution_time) | ||
|
|
||
| ### 2. Timeout Detection Listener | ||
|
|
||
| **Location**: [app/Listeners/LogQueryTimeout.php](app/Listeners/LogQueryTimeout.php) | ||
|
|
||
| Registered in [app/Providers/EventServiceProvider.php:97-99](app/Providers/EventServiceProvider.php#L97-L99) | ||
|
|
||
| This listener provides detailed logging for queries that exceed warning/critical thresholds: | ||
| - **70% threshold**: WARNING level log | ||
| - **90% threshold**: CRITICAL/ERROR level log | ||
|
|
||
| ### 3. PHP Timeout Handler | ||
|
|
||
| **Location**: [app/Providers/AppServiceProvider.php:200-216](app/Providers/AppServiceProvider.php#L200-L216) | ||
|
|
||
| A shutdown function that catches when PHP times out entirely, logging: | ||
| - Error message | ||
| - File and line where timeout occurred | ||
| - Request URL and method | ||
|
|
||
| ## Log Files | ||
|
|
||
| Logs are written to different files based on severity: | ||
|
|
||
| - `storage/logs/errors.log` - Critical/slow queries (>90% timeout) | ||
| - `storage/logs/warning.log` - Slow queries (>80% timeout or >1s) | ||
| - `storage/logs/daily.log` - All SQL queries (when DB_LOG_SQL=true) | ||
ildyria marked this conversation as resolved.
Show resolved
Hide resolved
|
||
|
|
||
| ## Example Log Entries | ||
|
|
||
| ### Slow Query Warning | ||
| ``` | ||
| [2026-01-04 10:15:32] warning.WARNING: ⚠️ WARNING: Slow query detected {"execution_time_ms":21000,"execution_time_s":21,"timeout_limit_s":30,"percentage":"70%","sql":"SELECT * FROM photos WHERE album_id = ?","bindings":[123],"connection":"mysql","url":"https://lychee.local/api/albums/123/photos"} | ||
| ``` | ||
|
|
||
| ### Critical Query Near Timeout | ||
| ``` | ||
| [2026-01-04 10:20:45] error.ERROR: 🚨 CRITICAL: Query approaching timeout {"execution_time_ms":27500,"execution_time_s":27.5,"timeout_limit_s":30,"percentage":"91.7%","sql":"SELECT * FROM photos WHERE...","bindings":[...],"connection":"mysql","url":"https://lychee.local/api/..."} | ||
| ``` | ||
|
|
||
| ### PHP Timeout Detected | ||
| ``` | ||
| [2026-01-04 10:25:12] error.ERROR: 🔥 PHP TIMEOUT DETECTED {"error":"Maximum execution time of 30 seconds exceeded","file":"/app/vendor/laravel/framework/src/Illuminate/Database/Connection.php","line":742,"url":"https://lychee.local/api/albums/delete","method":"DELETE"} | ||
| ``` | ||
|
|
||
| ## Troubleshooting Timeouts | ||
|
|
||
| When you see timeout logs: | ||
|
|
||
| 1. **Check the SQL query** - Look for missing indexes, inefficient joins, or full table scans | ||
| 2. **Use EXPLAIN** - Enable `DB_LOG_SQL_EXPLAIN=true` to see query execution plans | ||
| 3. **Add indexes** - Common fixes involve adding database indexes | ||
| 4. **Optimize queries** - Rewrite queries to be more efficient | ||
| 5. **Increase timeout** - As a last resort, increase `max_execution_time` in `config/octane.php` | ||
| 6. **Use queues** - Move long-running operations to background jobs | ||
|
|
||
| ## Performance Impact | ||
|
|
||
| SQL logging has minimal performance impact when disabled. When enabled: | ||
| - Each query execution triggers event listeners | ||
| - EXPLAIN queries add overhead for SELECT statements (MySQL only) | ||
| - Logs are written asynchronously via Monolog | ||
|
|
||
| **Recommendation**: Only enable in development or temporarily in production for debugging. | ||
|
|
||
| ## Related Configuration | ||
|
|
||
| ### Database Connection Timeouts | ||
|
|
||
| MySQL connection settings in [config/database.php:111-113](config/database.php#L111-L113): | ||
|
|
||
| ```php | ||
| PDO::ATTR_TIMEOUT => 5, // Connection timeout (5 seconds) | ||
| PDO::MYSQL_ATTR_INIT_COMMAND => 'SET SESSION wait_timeout=28800', // 8 hours | ||
| ``` | ||
|
|
||
| ### Octane Database Ping | ||
|
|
||
| The AppServiceProvider pings database connections every 30 seconds to prevent timeouts: [app/Providers/AppServiceProvider.php:340-341](app/Providers/AppServiceProvider.php#L340-L341) | ||
|
|
||
| ## Viewing Logs | ||
|
|
||
| Logs can be viewed via: | ||
| 1. **Log Viewer** - Built-in at `/log-viewer` (requires admin access) | ||
| 2. **Command line**: `tail -f storage/logs/errors.log` | ||
| 3. **Docker**: `docker logs <container_name>` | ||
|
|
||
| ## Additional Notes | ||
|
|
||
| - Timeout detection works best with FrankenPHP, Swoole, or RoadRunner | ||
| - Traditional PHP-FPM may not trigger all timeout handlers consistently | ||
ildyria marked this conversation as resolved.
Show resolved
Hide resolved
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,82 @@ | ||
| <?php | ||
|
|
||
| /** | ||
| * SPDX-License-Identifier: MIT | ||
| * Copyright (c) 2017-2018 Tobias Reich | ||
| * Copyright (c) 2018-2026 LycheeOrg. | ||
| */ | ||
|
|
||
| /** | ||
| * We don't care for unhandled exceptions in tests. | ||
| * It is the nature of a test to throw an exception. | ||
| * Without this suppression we had 100+ Linter warning in this file which | ||
| * don't help anything. | ||
| * | ||
| * @noinspection PhpDocMissingThrowsInspection | ||
| * @noinspection PhpUnhandledExceptionInspection | ||
| */ | ||
|
|
||
| namespace Tests\Unit\Listeners; | ||
|
|
||
| use App\Listeners\LogQueryTimeout; | ||
| use Illuminate\Database\Events\QueryExecuted; | ||
| use Illuminate\Support\Facades\Config; | ||
| use Illuminate\Support\Facades\Log; | ||
| use Tests\AbstractTestCase; | ||
|
|
||
| class LogQueryTimeoutTest extends AbstractTestCase | ||
| { | ||
| public function testQueryBelowWarningThreshold(): void | ||
| { | ||
| Log::shouldReceive('error')->never(); | ||
| Log::shouldReceive('warning')->never(); | ||
|
|
||
| Config::set('octane.max_execution_time', 30); | ||
|
|
||
| $listener = new LogQueryTimeout(); | ||
| $event = new QueryExecuted( | ||
| 'SELECT * FROM users', | ||
| [], | ||
| 10000, // 10 seconds (below 70% of 30s = 21s) | ||
| new \Illuminate\Database\Connection(new \PDO('sqlite::memory:')) | ||
| ); | ||
|
|
||
| $listener->handle($event); | ||
| } | ||
|
|
||
| public function testQueryAtWarningThreshold(): void | ||
| { | ||
| Log::shouldReceive('error')->never(); | ||
| Log::shouldReceive('warning')->once()->with('⚠️ WARNING: Slow query detected', \Mockery::type('array')); | ||
|
|
||
| Config::set('octane.max_execution_time', 30); | ||
|
|
||
| $listener = new LogQueryTimeout(); | ||
| $event = new QueryExecuted( | ||
| 'SELECT * FROM large_table', | ||
| ['param1'], | ||
| 22000, // 22 seconds (above 70% of 30s = 21s, below 90% = 27s) | ||
| new \Illuminate\Database\Connection(new \PDO('sqlite::memory:')) | ||
| ); | ||
|
|
||
| $listener->handle($event); | ||
| } | ||
|
|
||
| public function testQueryAtCriticalThreshold(): void | ||
| { | ||
| Log::shouldReceive('warning')->never(); | ||
| Log::shouldReceive('error')->once()->with('🚨 CRITICAL: Query approaching timeout', \Mockery::type('array')); | ||
|
|
||
| Config::set('octane.max_execution_time', 30); | ||
|
|
||
| $listener = new LogQueryTimeout(); | ||
| $event = new QueryExecuted( | ||
| 'SELECT * FROM very_large_table', | ||
| ['param1', 'param2'], | ||
| 28000, // 28 seconds (above 90% of 30s = 27s) | ||
| new \Illuminate\Database\Connection(new \PDO('sqlite::memory:')) | ||
| ); | ||
|
|
||
| $listener->handle($event); | ||
| } | ||
| } |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.