Rate-Limit and Queue-Tweaks

app/Console/Commands/QueueUpdates.php

@@ -63,7 +63,7 @@ function (Collection $chunk) use ($targetVersion) {
                 $this->totalPushed += $chunk->count();
 
                 // Push each site check to queue
-                $chunk->each(fn ($site) => UpdateSite::dispatch($site, $targetVersion));
+                $chunk->each(fn ($site) => UpdateSite::dispatch($site, $targetVersion)->onQueue('updates'));
             }
         );
 

app/Jobs/CheckSiteHealth.php

@@ -8,21 +8,32 @@
 use App\RemoteSite\Connection;
 use App\TUF\TufFetcher;
 use Carbon\Carbon;
+use Illuminate\Contracts\Queue\ShouldBeUnique;
 use Illuminate\Contracts\Queue\ShouldQueue;
 use Illuminate\Foundation\Queue\Queueable;
 use Illuminate\Support\Facades\App;
 
-class CheckSiteHealth implements ShouldQueue
+class CheckSiteHealth implements ShouldQueue, ShouldBeUnique
 {
     use Queueable;
 
+    public int $uniqueFor = 120;
+
     /**
      * Create a new job instance.
      */
     public function __construct(protected readonly Site $site)
     {
     }
 
+    /**
+     * Get the unique ID for the job.
+     */
+    public function uniqueId(): string
+    {
+        return (string) $this->site->id;
+    }
+
     /**
      * Execute the job.
      */
@@ -66,6 +77,6 @@ public function handle(): void
         UpdateSite::dispatch(
             $this->site,
             $latestVersion
-        );
+        )->onQueue('updates');
     }
 }

app/Jobs/UpdateSite.php

@@ -9,15 +9,17 @@
 use App\RemoteSite\Connection;
 use App\RemoteSite\Responses\PrepareUpdate;
 use GuzzleHttp\Exception\RequestException;
+use Illuminate\Contracts\Queue\ShouldBeUnique;
 use Illuminate\Contracts\Queue\ShouldQueue;
 use Illuminate\Foundation\Queue\Queueable;
 use Illuminate\Support\Facades\App;
 use Illuminate\Support\Facades\Log;
 
-class UpdateSite implements ShouldQueue
+class UpdateSite implements ShouldQueue, ShouldBeUnique
 {
     use Queueable;
     protected ?int $preUpdateCode = null;
+    public int $uniqueFor = 3600;
 
     /**
      * Create a new job instance.
@@ -26,6 +28,14 @@ public function __construct(protected readonly Site $site, protected string $tar
     {
     }
 
+    /**
+     * Get the unique ID for the job.
+     */
+    public function uniqueId(): string
+    {
+        return (string) $this->site->id;
+    }
+
     /**
      * Execute the job.
      */

app/Providers/AppServiceProvider.php

@@ -2,6 +2,7 @@
 
 namespace App\Providers;
 
+use App\Network\DNSLookup;
 use Illuminate\Cache\RateLimiting\Limit;
 use Illuminate\Http\Request;
 use Illuminate\Support\Facades\RateLimiter;
@@ -23,10 +24,32 @@ public function register(): void
     public function boot(): void
     {
         RateLimiter::for('site', function (Request $request) {
-            return Limit::perMinute(10)->by(
-                // @phpstan-ignore-next-line
-                parse_url((string) $request->input('url'), PHP_URL_HOST)
-            );
+            $siteHost = 'default';
+
+            if (is_string($request->input('url'))) {
+                $siteHost = (string) parse_url($request->input('url'), PHP_URL_HOST);
+            }
+
+            // Define a rate limit per target IP
+            $siteIpLimits = [];
+
+            if ($siteHost !== 'default') {
+                $siteIps = (new DNSLookup())->getIPs($siteHost);
+
+                foreach ($siteIps as $siteIp) {
+                    $siteIpLimits[] = Limit::perMinute(5)->by("siteip-" . $siteIp);
+                }
+            }
+
+            if (!count($siteIpLimits)) {
+                $siteIpLimits = [Limit::perMinute(5)->by("siteip-default")];
+            }
+
+            return [
+                Limit::perMinute(5)->by("sitehost-" . $siteHost),
+                Limit::perMinute(50)->by("requestip-" . $request->ip()),
+                ...$siteIpLimits
+            ];
         });
     }
 }

app/Providers/HttpclientServiceProvider.php

@@ -3,12 +3,6 @@
 namespace App\Providers;
 
 use GuzzleHttp\Client;
-use GuzzleHttp\Exception\ConnectException;
-use GuzzleHttp\Handler\CurlHandler;
-use GuzzleHttp\HandlerStack;
-use GuzzleHttp\Middleware;
-use GuzzleHttp\Psr7\Request;
-use GuzzleHttp\Psr7\Response;
 use Illuminate\Support\ServiceProvider;
 
 class HttpclientServiceProvider extends ServiceProvider
@@ -21,42 +15,7 @@ class HttpclientServiceProvider extends ServiceProvider
     public function register()
     {
         $this->app->singleton(Client::class, function ($app) {
-            $handlerStack = HandlerStack::create(new CurlHandler());
-            $handlerStack->push(
-                Middleware::retry(
-                    function (
-                        $retries,
-                        Request $request,
-                        Response $response = null,
-                        \Throwable $exception = null
-                    ) {
-                        // Limit the number of retries to 3
-                        if ($retries >= 3) {
-                            return false;
-                        }
-
-                        // Retry connection exceptions
-                        if ($exception instanceof ConnectException) {
-                            return true;
-                        }
-
-                        if ($response) {
-                            // Retry on server errors
-                            if ($response->getStatusCode() >= 500) {
-                                return true;
-                            }
-                        }
-
-                        return false;
-                    },
-                    function ($numberOfRetries) {
-                        return 1000 * $numberOfRetries;
-                    }
-                )
-            );
-
             return new Client([
-                'handler'  => $handlerStack,
                 'allow_redirects' => [
                     'max'             => 5,
                     'strict'          => true,  // "strict" redirects - that's key as a redirected POST stays a POST

app/RemoteSite/Connection.php

@@ -94,7 +94,9 @@ public function performWebserviceRequest(
                 "headers" => [
                     "Content-Type" => "application/json",
                     "Accept" => "application/vnd.api+json"
-                ]
+                ],
+                'timeout' => 60.0,
+                'connect_timeout' => 5.0,
             ]
         );
 

config/horizon.php

@@ -180,7 +180,7 @@
     */
 
     'defaults' => [
-        'supervisor-1' => [
+        'supervisor-default'  => [
             'connection' => 'redis',
             'queue' => ['default'],
             'balance' => 'auto',
@@ -190,22 +190,43 @@
             'maxJobs' => 0,
             'memory' => 128,
             'tries' => 1,
-            'timeout' => 60,
+            'timeout' => 15,
+            'nice' => 0,
+        ],
+        'supervisor-updates' => [
+            'connection' => 'redis',
+            'queue' => ['updates'],
+            'balance' => 'auto',
+            'autoScalingStrategy' => 'time',
+            'maxProcesses' => 1,
+            'maxTime' => 0,
+            'maxJobs' => 0,
+            'memory' => 128,
+            'tries' => 1,
+            'timeout' => 600,
             'nice' => 0,
         ],
     ],
 
     'environments' => [
         'production' => [
-            'supervisor-1' => [
-                'maxProcesses' => 10,
-                'balanceMaxShift' => 1,
-                'balanceCooldown' => 3,
+            'supervisor-default' => [
+                'maxProcesses' => 250,
+                'balanceMaxShift' => 10,
+                'balanceCooldown' => 25,
+            ],
+            'supervisor-updates' => [
+                'maxProcesses' => 500,
+                'balanceMaxShift' => 10,
+                'balanceCooldown' => 25,
             ],
         ],
 
         'local' => [
-            'supervisor-1' => [
+            'supervisor-default' => [
+                'maxProcesses' => 3,
+            ],
+            'supervisor-updates' => [
                 'maxProcesses' => 3,
             ],
         ],