Implement random scheduling times per APP_KEY to prevents thundering herd problems (librenms#18246)

* Implement random scheduling times per host to prevents thundering herd problems

The randomization is stable per host, meaning it will always return the same "random" value per input.
This is needed so the scheduler doesn't run the same job multiple times

* Apply fixes from StyleCI

Author: Jellyfrog

LibreNMS/Util/Time.php

@@ -29,6 +29,7 @@
 use Carbon\CarbonInterface;
 use Carbon\Exceptions\InvalidFormatException;
 use Illuminate\Support\Carbon;
+use Illuminate\Support\Facades\Config;
 
 class Time
 {
@@ -173,4 +174,33 @@ public static function durationToSeconds(string $duration): int
 
         return $duration === '' ? 0 : 300;
     }
+
+    /**
+     * Return a random time between the two given times.
+     */
+    public static function randomBetween(string|int $min, string|int $max): Carbon
+    {
+        $time = new Carbon($min);
+
+        $time->addSeconds(mt_rand(0, (int) $time->diffInSeconds(new Carbon($max), true)));
+
+        return $time;
+    }
+
+    /**
+     * Return a psedudo random time between the two given times.
+     * The same time will always be returned for a given APP_KEY
+     */
+    public static function pseudoRandomBetween(string|int $min, string|int $max, string $format = 'H:i'): string
+    {
+        // Seed the random number generator to get consistent results for a given APP_KEY
+        mt_srand(crc32(Config::get('app.key') . $min . $max));
+
+        $time = self::randomBetween($min, $max);
+
+        // Need to restore the seed after
+        mt_srand();
+
+        return $time->format($format);
+    }
 }

app/Console/Commands/MaintenanceFetchOuis.php

@@ -20,15 +20,13 @@ class MaintenanceFetchOuis extends LnmsCommand
 
     protected string $mac_oui_url = 'https://www.wireshark.org/download/automated/data/manuf';
     protected int $min_refresh_days = 6;
-    protected int $max_wait_seconds = 900;
     protected int $upsert_chunk_size = 1000;
 
     public function __construct()
     {
         parent::__construct();
 
         $this->addOption('force', null, InputOption::VALUE_NONE);
-        $this->addOption('wait', null, InputOption::VALUE_NONE);
     }
 
     /**
@@ -56,14 +54,6 @@ public function handle(): int
             return 0;
         }
 
-        // wait for 0-15 minutes to prevent stampeding herd
-        if ($this->option('wait')) {
-            $seconds = random_int(1, $this->max_wait_seconds);
-            $minutes = (int) round($seconds / 60);
-            $this->info(trans_choice('commands.maintenance:fetch-ouis.waiting', $minutes, ['minutes' => $minutes]));
-            sleep($seconds);
-        }
-
         $this->line(trans('commands.maintenance:fetch-ouis.starting'));
 
         try {

routes/console.php

@@ -5,6 +5,7 @@
 use App\Jobs\PingCheck;
 use Illuminate\Support\Facades\Artisan;
 use Illuminate\Support\Facades\Schedule;
+use LibreNMS\Util\Time;
 use Symfony\Component\Process\Process;
 
 /*
@@ -199,12 +200,12 @@
 
 // schedule maintenance, should be after all others
 $maintenance_log_file = Config::get('log_dir') . '/maintenance.log';
-Schedule::command(MaintenanceFetchOuis::class, ['--wait'])
-    ->weeklyOn(0, '1:00')
+Schedule::command(MaintenanceFetchOuis::class)
+    ->weeklyOn(0, Time::pseudoRandomBetween('01:00', '01:59'))
     ->onOneServer()
     ->appendOutputTo($maintenance_log_file);
 
-Schedule::command(MaintenanceCleanupNetworks::class, [])
-    ->weeklyOn(0, '2:00')
+Schedule::command(MaintenanceCleanupNetworks::class)
+    ->weeklyOn(0, Time::pseudoRandomBetween('02:00', '02:59'))
     ->onOneServer()
     ->appendOutputTo($maintenance_log_file);

tests/Unit/TimeUtilityTest.php

@@ -84,4 +84,23 @@ public function testParseInput(): void
 
         $this->assertNull(Time::parseInput('not a date'));
     }
+
+    public function testRandomTimeBetween(): void
+    {
+        $start = time();
+        $end = time() + 3600;
+        $randomTime = Time::randomBetween($start, $end)->format('U');
+        $this->assertGreaterThanOrEqual($start, $randomTime);
+        $this->assertLessThanOrEqual($end, $randomTime);
+
+        // test with pesudo random
+        $randomTime = Time::pseudoRandomBetween($start, $end, 'U');
+        $this->assertGreaterThanOrEqual($start, $randomTime);
+        $this->assertLessThanOrEqual($end, $randomTime);
+
+        $this->assertEquals(
+            Time::pseudoRandomBetween($start, $end),
+            Time::pseudoRandomBetween($start, $end),
+        );
+    }
 }