feature symfony#53328 [Messenger] Add jitter parameter to MultiplierRetryStrategy (rmikalkenas)

This PR was merged into the 7.1 branch.

Discussion
----------

[Messenger] Add jitter parameter to MultiplierRetryStrategy

| Q             | A
| ------------- | ---
| Branch?       | 7.1
| Bug fix?      | no
| New feature?  | yes
| Deprecations? | no
| Issues        | -
| License       | MIT

I'm seeing an interesting case, when multiple queue consumers causes failures in a downstream systems a.k.a. [thundering herd](https://en.wikipedia.org/wiki/Thundering_herd_problem) effect.
Many jobs fail, they all get enqueued to try again in a static interval and bring down the downstream system yet again. This repeats until the retry limit is exhausted.
Introduced delay randomness prevents thundering herd effect. The randomness can be controlled via `$jitter` parameter.

~~**Open question:** I added `$jitter` parameter with a default value of `0.1` (as in [http client's generic retry](https://github.com/symfony/symfony/blob/7.1/src/Symfony/Component/HttpClient/Retry/GenericRetryStrategy.php)), but maybe it should be set as `0` to keep code in a BC fashion?~~

Commits
-------

9949684 [Messenger] Add jitter parameter to MultiplierRetryStrategy

Author: fabpot

src/Symfony/Bundle/FrameworkBundle/DependencyInjection/Configuration.php

@@ -1592,6 +1592,7 @@ function ($a) {
                                             ->integerNode('delay')->defaultValue(1000)->min(0)->info('Time in ms to delay (or the initial value when multiplier is used)')->end()
                                             ->floatNode('multiplier')->defaultValue(2)->min(1)->info('If greater than 1, delay will grow exponentially for each retry: this delay = (delay * (multiple ^ retries))')->end()
                                             ->integerNode('max_delay')->defaultValue(0)->min(0)->info('Max time in ms that a retry should ever be delayed (0 = infinite)')->end()
+                                            ->floatNode('jitter')->defaultValue(0.1)->min(0)->max(1)->info('Randomness to apply to the delay (between 0 and 1)')->end()
                                         ->end()
                                     ->end()
                                     ->scalarNode('rate_limiter')

src/Symfony/Bundle/FrameworkBundle/DependencyInjection/FrameworkExtension.php

@@ -2195,7 +2195,8 @@ private function registerMessengerConfiguration(array $config, ContainerBuilder
                     ->replaceArgument(0, $transport['retry_strategy']['max_retries'])
                     ->replaceArgument(1, $transport['retry_strategy']['delay'])
                     ->replaceArgument(2, $transport['retry_strategy']['multiplier'])
-                    ->replaceArgument(3, $transport['retry_strategy']['max_delay']);
+                    ->replaceArgument(3, $transport['retry_strategy']['max_delay'])
+                    ->replaceArgument(4, $transport['retry_strategy']['jitter']);
                 $container->setDefinition($retryServiceId, $retryDefinition);
 
                 $transportRetryReferences[$name] = new Reference($retryServiceId);

src/Symfony/Bundle/FrameworkBundle/Resources/config/messenger.php

@@ -160,6 +160,7 @@
                 abstract_arg('delay ms'),
                 abstract_arg('multiplier'),
                 abstract_arg('max delay ms'),
+                abstract_arg('jitter'),
             ])
 
         // rate limiter

src/Symfony/Component/Messenger/CHANGELOG.md

@@ -7,6 +7,7 @@ CHANGELOG
  * Add option `redis_sentinel` as an alias for `sentinel_master`
  * Add `--all` option to the `messenger:consume` command
  * Make `#[AsMessageHandler]` final
+ * Add parameter `$jitter` to `MultiplierRetryStrategy` in order to randomize delay and prevent the thundering herd effect
 
 7.0
 ---

src/Symfony/Component/Messenger/Retry/MultiplierRetryStrategy.php

@@ -36,14 +36,16 @@ class MultiplierRetryStrategy implements RetryStrategyInterface
     private int $delayMilliseconds;
     private float $multiplier;
     private int $maxDelayMilliseconds;
+    private float $jitter;
 
     /**
      * @param int   $maxRetries           The maximum number of times to retry
      * @param int   $delayMilliseconds    Amount of time to delay (or the initial value when multiplier is used)
      * @param float $multiplier           Multiplier to apply to the delay each time a retry occurs
      * @param int   $maxDelayMilliseconds Maximum delay to allow (0 means no maximum)
+     * @param float $jitter               Randomness to apply to the delay (between 0 and 1)
      */
-    public function __construct(int $maxRetries = 3, int $delayMilliseconds = 1000, float $multiplier = 1, int $maxDelayMilliseconds = 0)
+    public function __construct(int $maxRetries = 3, int $delayMilliseconds = 1000, float $multiplier = 1, int $maxDelayMilliseconds = 0, float $jitter = 0.1)
     {
         $this->maxRetries = $maxRetries;
 
@@ -61,6 +63,11 @@ public function __construct(int $maxRetries = 3, int $delayMilliseconds = 1000,
             throw new InvalidArgumentException(sprintf('Max delay must be greater than or equal to zero: "%s" given.', $maxDelayMilliseconds));
         }
         $this->maxDelayMilliseconds = $maxDelayMilliseconds;
+
+        if ($jitter < 0 || $jitter > 1) {
+            throw new InvalidArgumentException(sprintf('Jitter must be between 0 and 1: "%s" given.', $jitter));
+        }
+        $this->jitter = $jitter;
     }
 
     /**
@@ -82,6 +89,11 @@ public function getWaitingTime(Envelope $message, \Throwable $throwable = null):
 
         $delay = $this->delayMilliseconds * $this->multiplier ** $retries;
 
+        if ($this->jitter > 0) {
+            $randomness = (int) ($delay * $this->jitter);
+            $delay += random_int(-$randomness, +$randomness);
+        }
+
         if ($delay > $this->maxDelayMilliseconds && 0 !== $this->maxDelayMilliseconds) {
             return $this->maxDelayMilliseconds;
         }

src/Symfony/Component/Messenger/Tests/Retry/MultiplierRetryStrategyTest.php

@@ -54,7 +54,7 @@ public function testIsRetryableWithNoStamp()
      */
     public function testGetWaitTime(int $delay, float $multiplier, int $maxDelay, int $previousRetries, int $expectedDelay)
     {
-        $strategy = new MultiplierRetryStrategy(10, $delay, $multiplier, $maxDelay);
+        $strategy = new MultiplierRetryStrategy(10, $delay, $multiplier, $maxDelay, 0);
         $envelope = new Envelope(new \stdClass(), [new RedeliveryStamp($previousRetries)]);
 
         $this->assertSame($expectedDelay, $strategy->getWaitingTime($envelope));
@@ -89,4 +89,39 @@ public static function getWaitTimeTests(): iterable
         yield [1000, 1.5555, 5000, 1, 1556];
         yield [1000, 1.5555, 5000, 2, 2420];
     }
+
+    /**
+     * @dataProvider getJitterTest
+     */
+    public function testJitter(float $jitter, int $maxMin, int $maxMax)
+    {
+        $strategy = new MultiplierRetryStrategy(3, 1000, 1, 0, $jitter);
+        $envelope = new Envelope(new \stdClass());
+
+        $min = 1000;
+        $max = 1000;
+        for ($i = 0; $i < 100; ++$i) {
+            $delay = $strategy->getWaitingTime($envelope);
+            $min = min($min, $delay);
+            $max = max($max, $delay);
+        }
+
+        $this->assertGreaterThanOrEqual($maxMin, $min);
+        $this->assertLessThanOrEqual($maxMax, $max);
+    }
+
+    public static function getJitterTest(): iterable
+    {
+        yield [1.0, 0, 2000];
+        yield [0.9, 100, 1900];
+        yield [0.8, 200, 1800];
+        yield [0.7, 300, 1700];
+        yield [0.6, 400, 1600];
+        yield [0.5, 500, 1500];
+        yield [0.4, 600, 1400];
+        yield [0.3, 700, 1300];
+        yield [0.2, 800, 1200];
+        yield [0.1, 900, 1100];
+        yield [0.0, 1000, 1000];
+    }
 }