implement various tweaks to rate limits and queue

Author: SniperSister

app/Console/Commands/QueueHealthChecks.php

@@ -48,7 +48,7 @@ function (Collection $chunk) {
                     $this->totalPushed += $chunk->count();
 
                     // Push each site check to queue
-                    $chunk->each(fn ($site) => CheckSiteHealth::dispatch($site));
+                    $chunk->each(fn ($site) => CheckSiteHealth::dispatch($site)->onQueue('cron'));
                 }
             );
 

app/Console/Commands/QueueUpdates.php

@@ -63,7 +63,7 @@ function (Collection $chunk) use ($targetVersion) {
                 $this->totalPushed += $chunk->count();
 
                 // Push each site check to queue
-                $chunk->each(fn ($site) => UpdateSite::dispatch($site, $targetVersion));
+                $chunk->each(fn ($site) => UpdateSite::dispatch($site, $targetVersion)->onQueue('updates'));
             }
         );
 

app/Jobs/CheckSiteHealth.php

@@ -8,21 +8,32 @@
 use App\RemoteSite\Connection;
 use App\TUF\TufFetcher;
 use Carbon\Carbon;
+use Illuminate\Contracts\Queue\ShouldBeUnique;
 use Illuminate\Contracts\Queue\ShouldQueue;
 use Illuminate\Foundation\Queue\Queueable;
 use Illuminate\Support\Facades\App;
 
-class CheckSiteHealth implements ShouldQueue
+class CheckSiteHealth implements ShouldQueue, ShouldBeUnique
 {
     use Queueable;
 
+    public int $uniqueFor = 120;
+
     /**
      * Create a new job instance.
      */
     public function __construct(protected readonly Site $site)
     {
     }
 
+    /**
+     * Get the unique ID for the job.
+     */
+    public function uniqueId(): string
+    {
+        return $this->site->id;
+    }
+
     /**
      * Execute the job.
      */
@@ -66,6 +77,6 @@ public function handle(): void
         UpdateSite::dispatch(
             $this->site,
             $latestVersion
-        );
+        )->onQueue('updates');
     }
 }

app/Jobs/UpdateSite.php

@@ -9,15 +9,17 @@
 use App\RemoteSite\Connection;
 use App\RemoteSite\Responses\PrepareUpdate;
 use GuzzleHttp\Exception\RequestException;
+use Illuminate\Contracts\Queue\ShouldBeUnique;
 use Illuminate\Contracts\Queue\ShouldQueue;
 use Illuminate\Foundation\Queue\Queueable;
 use Illuminate\Support\Facades\App;
 use Illuminate\Support\Facades\Log;
 
-class UpdateSite implements ShouldQueue
+class UpdateSite implements ShouldQueue, ShouldBeUnique
 {
     use Queueable;
     protected ?int $preUpdateCode = null;
+    public int $uniqueFor = 3600;
 
     /**
      * Create a new job instance.
@@ -26,6 +28,14 @@ public function __construct(protected readonly Site $site, protected string $tar
     {
     }
 
+    /**
+     * Get the unique ID for the job.
+     */
+    public function uniqueId(): string
+    {
+        return $this->site->id;
+    }
+
     /**
      * Execute the job.
      */

app/Providers/AppServiceProvider.php

@@ -23,10 +23,22 @@ public function register(): void
     public function boot(): void
     {
         RateLimiter::for('site', function (Request $request) {
-            return Limit::perMinute(10)->by(
-                // @phpstan-ignore-next-line
-                parse_url((string) $request->input('url'), PHP_URL_HOST)
-            );
+            $siteHost = 'default';
+            $siteIp = 'default';
+
+            if (is_string($request->input('url'))) {
+                $siteHost = parse_url($request->input('url'), PHP_URL_HOST);
+            }
+
+            if ($siteHost !== 'default' && $dnsResult = dns_get_record($siteHost, DNS_A)) {
+                $siteIp = $dnsResult[0]['ip'];
+            }
+
+            return [
+                Limit::perMinute(5)->by("sitehost-" . $siteHost),
+                Limit::perMinute(10)->by("siteip-" . $siteIp),
+                Limit::perMinute(50)->by("ip-" . $request->ip())
+            ];
         });
     }
 }

app/Providers/HttpclientServiceProvider.php

@@ -21,42 +21,7 @@ class HttpclientServiceProvider extends ServiceProvider
     public function register()
     {
         $this->app->singleton(Client::class, function ($app) {
-            $handlerStack = HandlerStack::create(new CurlHandler());
-            $handlerStack->push(
-                Middleware::retry(
-                    function (
-                        $retries,
-                        Request $request,
-                        Response $response = null,
-                        \Throwable $exception = null
-                    ) {
-                        // Limit the number of retries to 3
-                        if ($retries >= 3) {
-                            return false;
-                        }
-
-                        // Retry connection exceptions
-                        if ($exception instanceof ConnectException) {
-                            return true;
-                        }
-
-                        if ($response) {
-                            // Retry on server errors
-                            if ($response->getStatusCode() >= 500) {
-                                return true;
-                            }
-                        }
-
-                        return false;
-                    },
-                    function ($numberOfRetries) {
-                        return 1000 * $numberOfRetries;
-                    }
-                )
-            );
-
             return new Client([
-                'handler'  => $handlerStack,
                 'allow_redirects' => [
                     'max'             => 5,
                     'strict'          => true,  // "strict" redirects - that's key as a redirected POST stays a POST

app/RemoteSite/Connection.php

@@ -94,7 +94,9 @@ public function performWebserviceRequest(
                 "headers" => [
                     "Content-Type" => "application/json",
                     "Accept" => "application/vnd.api+json"
-                ]
+                ],
+                'timeout' => 60.0,
+                'connect_timeout' => 5.0,
             ]
         );
 

config/horizon.php

@@ -180,32 +180,53 @@
     */
 
     'defaults' => [
-        'supervisor-1' => [
+        'supervisor-cron'  => [
             'connection' => 'redis',
-            'queue' => ['default'],
+            'queue' => ['default', 'cron'],
             'balance' => 'auto',
             'autoScalingStrategy' => 'time',
             'maxProcesses' => 1,
             'maxTime' => 0,
             'maxJobs' => 0,
             'memory' => 128,
             'tries' => 1,
-            'timeout' => 60,
+            'timeout' => 15,
+            'nice' => 0,
+        ],
+        'supervisor-updates' => [
+            'connection' => 'redis',
+            'queue' => ['updates'],
+            'balance' => 'auto',
+            'autoScalingStrategy' => 'time',
+            'maxProcesses' => 1,
+            'maxTime' => 0,
+            'maxJobs' => 0,
+            'memory' => 128,
+            'tries' => 1,
+            'timeout' => 600,
             'nice' => 0,
         ],
     ],
 
     'environments' => [
         'production' => [
-            'supervisor-1' => [
-                'maxProcesses' => 10,
-                'balanceMaxShift' => 1,
-                'balanceCooldown' => 3,
+            'supervisor-cron' => [
+                'maxProcesses' => 250,
+                'balanceMaxShift' => 10,
+                'balanceCooldown' => 25,
+            ],
+            'supervisor-updates' => [
+                'maxProcesses' => 500,
+                'balanceMaxShift' => 10,
+                'balanceCooldown' => 25,
             ],
         ],
 
         'local' => [
-            'supervisor-1' => [
+            'supervisor-cron' => [
+                'maxProcesses' => 3,
+            ],
+            'supervisor-updates' => [
                 'maxProcesses' => 3,
             ],
         ],