diff --git a/app/Console/Commands/QueueUpdates.php b/app/Console/Commands/QueueUpdates.php index 57b433d..267bfeb 100644 --- a/app/Console/Commands/QueueUpdates.php +++ b/app/Console/Commands/QueueUpdates.php @@ -63,7 +63,7 @@ function (Collection $chunk) use ($targetVersion) { $this->totalPushed += $chunk->count(); // Push each site check to queue - $chunk->each(fn ($site) => UpdateSite::dispatch($site, $targetVersion)); + $chunk->each(fn ($site) => UpdateSite::dispatch($site, $targetVersion)->onQueue('updates')); } ); diff --git a/app/Jobs/CheckSiteHealth.php b/app/Jobs/CheckSiteHealth.php index 8fc188e..a522bb5 100644 --- a/app/Jobs/CheckSiteHealth.php +++ b/app/Jobs/CheckSiteHealth.php @@ -8,14 +8,17 @@ use App\RemoteSite\Connection; use App\TUF\TufFetcher; use Carbon\Carbon; +use Illuminate\Contracts\Queue\ShouldBeUnique; use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Queue\Queueable; use Illuminate\Support\Facades\App; -class CheckSiteHealth implements ShouldQueue +class CheckSiteHealth implements ShouldQueue, ShouldBeUnique { use Queueable; + public int $uniqueFor = 120; + /** * Create a new job instance. */ @@ -23,6 +26,14 @@ public function __construct(protected readonly Site $site) { } + /** + * Get the unique ID for the job. + */ + public function uniqueId(): string + { + return (string) $this->site->id; + } + /** * Execute the job. */ @@ -66,6 +77,6 @@ public function handle(): void UpdateSite::dispatch( $this->site, $latestVersion - ); + )->onQueue('updates'); } } diff --git a/app/Jobs/UpdateSite.php b/app/Jobs/UpdateSite.php index 4a1970a..c2bfb7e 100644 --- a/app/Jobs/UpdateSite.php +++ b/app/Jobs/UpdateSite.php @@ -9,15 +9,17 @@ use App\RemoteSite\Connection; use App\RemoteSite\Responses\PrepareUpdate; use GuzzleHttp\Exception\RequestException; +use Illuminate\Contracts\Queue\ShouldBeUnique; use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Queue\Queueable; use Illuminate\Support\Facades\App; use Illuminate\Support\Facades\Log; -class UpdateSite implements ShouldQueue +class UpdateSite implements ShouldQueue, ShouldBeUnique { use Queueable; protected ?int $preUpdateCode = null; + public int $uniqueFor = 3600; /** * Create a new job instance. @@ -26,6 +28,14 @@ public function __construct(protected readonly Site $site, protected string $tar { } + /** + * Get the unique ID for the job. + */ + public function uniqueId(): string + { + return (string) $this->site->id; + } + /** * Execute the job. */ diff --git a/app/Providers/AppServiceProvider.php b/app/Providers/AppServiceProvider.php index d25fa7b..7027757 100644 --- a/app/Providers/AppServiceProvider.php +++ b/app/Providers/AppServiceProvider.php @@ -2,6 +2,7 @@ namespace App\Providers; +use App\Network\DNSLookup; use Illuminate\Cache\RateLimiting\Limit; use Illuminate\Http\Request; use Illuminate\Support\Facades\RateLimiter; @@ -23,10 +24,32 @@ public function register(): void public function boot(): void { RateLimiter::for('site', function (Request $request) { - return Limit::perMinute(10)->by( - // @phpstan-ignore-next-line - parse_url((string) $request->input('url'), PHP_URL_HOST) - ); + $siteHost = 'default'; + + if (is_string($request->input('url'))) { + $siteHost = (string) parse_url($request->input('url'), PHP_URL_HOST); + } + + // Define a rate limit per target IP + $siteIpLimits = []; + + if ($siteHost !== 'default') { + $siteIps = (new DNSLookup())->getIPs($siteHost); + + foreach ($siteIps as $siteIp) { + $siteIpLimits[] = Limit::perMinute(5)->by("siteip-" . $siteIp); + } + } + + if (!count($siteIpLimits)) { + $siteIpLimits = [Limit::perMinute(5)->by("siteip-default")]; + } + + return [ + Limit::perMinute(5)->by("sitehost-" . $siteHost), + Limit::perMinute(50)->by("requestip-" . $request->ip()), + ...$siteIpLimits + ]; }); } } diff --git a/app/Providers/HttpclientServiceProvider.php b/app/Providers/HttpclientServiceProvider.php index 0fa2547..8450c08 100644 --- a/app/Providers/HttpclientServiceProvider.php +++ b/app/Providers/HttpclientServiceProvider.php @@ -3,12 +3,6 @@ namespace App\Providers; use GuzzleHttp\Client; -use GuzzleHttp\Exception\ConnectException; -use GuzzleHttp\Handler\CurlHandler; -use GuzzleHttp\HandlerStack; -use GuzzleHttp\Middleware; -use GuzzleHttp\Psr7\Request; -use GuzzleHttp\Psr7\Response; use Illuminate\Support\ServiceProvider; class HttpclientServiceProvider extends ServiceProvider @@ -21,42 +15,7 @@ class HttpclientServiceProvider extends ServiceProvider public function register() { $this->app->singleton(Client::class, function ($app) { - $handlerStack = HandlerStack::create(new CurlHandler()); - $handlerStack->push( - Middleware::retry( - function ( - $retries, - Request $request, - Response $response = null, - \Throwable $exception = null - ) { - // Limit the number of retries to 3 - if ($retries >= 3) { - return false; - } - - // Retry connection exceptions - if ($exception instanceof ConnectException) { - return true; - } - - if ($response) { - // Retry on server errors - if ($response->getStatusCode() >= 500) { - return true; - } - } - - return false; - }, - function ($numberOfRetries) { - return 1000 * $numberOfRetries; - } - ) - ); - return new Client([ - 'handler' => $handlerStack, 'allow_redirects' => [ 'max' => 5, 'strict' => true, // "strict" redirects - that's key as a redirected POST stays a POST diff --git a/app/RemoteSite/Connection.php b/app/RemoteSite/Connection.php index 3f32adf..87f2e83 100644 --- a/app/RemoteSite/Connection.php +++ b/app/RemoteSite/Connection.php @@ -94,7 +94,9 @@ public function performWebserviceRequest( "headers" => [ "Content-Type" => "application/json", "Accept" => "application/vnd.api+json" - ] + ], + 'timeout' => 60.0, + 'connect_timeout' => 5.0, ] ); diff --git a/config/horizon.php b/config/horizon.php index dfac10b..26b2c2d 100644 --- a/config/horizon.php +++ b/config/horizon.php @@ -180,7 +180,7 @@ */ 'defaults' => [ - 'supervisor-1' => [ + 'supervisor-default' => [ 'connection' => 'redis', 'queue' => ['default'], 'balance' => 'auto', @@ -190,22 +190,43 @@ 'maxJobs' => 0, 'memory' => 128, 'tries' => 1, - 'timeout' => 60, + 'timeout' => 15, + 'nice' => 0, + ], + 'supervisor-updates' => [ + 'connection' => 'redis', + 'queue' => ['updates'], + 'balance' => 'auto', + 'autoScalingStrategy' => 'time', + 'maxProcesses' => 1, + 'maxTime' => 0, + 'maxJobs' => 0, + 'memory' => 128, + 'tries' => 1, + 'timeout' => 600, 'nice' => 0, ], ], 'environments' => [ 'production' => [ - 'supervisor-1' => [ - 'maxProcesses' => 10, - 'balanceMaxShift' => 1, - 'balanceCooldown' => 3, + 'supervisor-default' => [ + 'maxProcesses' => 250, + 'balanceMaxShift' => 10, + 'balanceCooldown' => 25, + ], + 'supervisor-updates' => [ + 'maxProcesses' => 500, + 'balanceMaxShift' => 10, + 'balanceCooldown' => 25, ], ], 'local' => [ - 'supervisor-1' => [ + 'supervisor-default' => [ + 'maxProcesses' => 3, + ], + 'supervisor-updates' => [ 'maxProcesses' => 3, ], ],