From cb237e2715f8073fb338ccf6266802e5111f48a8 Mon Sep 17 00:00:00 2001 From: vvzvlad Date: Wed, 8 Oct 2025 21:55:10 +0300 Subject: [PATCH 1/7] feat: add jitter to next_check_at scheduling to avoid thundering herd --- internal/model/feed.go | 27 ++++++++++++++++++++++----- internal/model/feed_test.go | 15 +++++++++------ 2 files changed, 31 insertions(+), 11 deletions(-) diff --git a/internal/model/feed.go b/internal/model/feed.go index 5e8a7aaae95..6c752fd217b 100644 --- a/internal/model/feed.go +++ b/internal/model/feed.go @@ -4,11 +4,12 @@ package model // import "miniflux.app/v2/internal/model" import ( - "fmt" - "io" - "time" + "fmt" + "io" + "math/rand" + "time" - "miniflux.app/v2/internal/config" + "miniflux.app/v2/internal/config" ) // List of supported schedulers. @@ -135,7 +136,7 @@ func (f *Feed) ScheduleNextCheck(weeklyCount int, refreshDelay time.Duration) ti // Use the RSS TTL field, Retry-After, Cache-Control or Expires HTTP headers if defined. interval = max(interval, refreshDelay) - // Limit the max interval value for misconfigured feeds. + // Limit the max interval value for misconfigured feeds. switch config.Opts.PollingScheduler() { case SchedulerRoundRobin: interval = min(interval, config.Opts.SchedulerRoundRobinMaxInterval()) @@ -143,6 +144,22 @@ func (f *Feed) ScheduleNextCheck(weeklyCount int, refreshDelay time.Duration) ti interval = min(interval, config.Opts.SchedulerEntryFrequencyMaxInterval()) } + // Apply a small random jitter to spread next checks and reduce thundering herds. + // Jitter range: [0, 10 minutes]. + jitterMax := 10 * time.Minute + if jitterMax > 0 { + jitter := time.Duration(rand.Int63n(int64(jitterMax + 1))) + interval += jitter + + // Re-apply max clamping after jitter to avoid exceeding configured caps. + switch config.Opts.PollingScheduler() { + case SchedulerRoundRobin: + interval = min(interval, config.Opts.SchedulerRoundRobinMaxInterval()) + case SchedulerEntryFrequency: + interval = min(interval, config.Opts.SchedulerEntryFrequencyMaxInterval()) + } + } + f.NextCheckAt = time.Now().Add(interval) return interval } diff --git a/internal/model/feed_test.go b/internal/model/feed_test.go index b5094870e6e..a8851db1ab3 100644 --- a/internal/model/feed_test.go +++ b/internal/model/feed_test.go @@ -68,12 +68,15 @@ func TestFeedCheckedNow(t *testing.T) { } func checkTargetInterval(t *testing.T, feed *Feed, targetInterval time.Duration, timeBefore time.Time, message string) { - if feed.NextCheckAt.Before(timeBefore.Add(targetInterval)) { - t.Errorf(`The next_check_at should be after timeBefore + %s`, message) - } - if feed.NextCheckAt.After(time.Now().Add(targetInterval)) { - t.Errorf(`The next_check_at should be before now + %s`, message) - } + // Allow a positive jitter up to 10 minutes added by the scheduler. + jitterMax := 10 * time.Minute + + if feed.NextCheckAt.Before(timeBefore.Add(targetInterval)) { + t.Errorf(`The next_check_at should be after timeBefore + %s`, message) + } + if feed.NextCheckAt.After(time.Now().Add(targetInterval + jitterMax)) { + t.Errorf(`The next_check_at should be before now + %s (with jitter)`, message) + } } func TestFeedScheduleNextCheckRoundRobinDefault(t *testing.T) { From b49fb41d1cffd5f349f8fb38778869a6398e882f Mon Sep 17 00:00:00 2001 From: Gabriele Date: Wed, 8 Oct 2025 22:50:28 +0200 Subject: [PATCH 2/7] feat: add config variable for polling jitter --- internal/config/options.go | 12 ++++++++++++ internal/model/feed.go | 40 ++++++++++++++++++-------------------- 2 files changed, 31 insertions(+), 21 deletions(-) diff --git a/internal/config/options.go b/internal/config/options.go index be8768a603e..d2431c33280 100644 --- a/internal/config/options.go +++ b/internal/config/options.go @@ -504,6 +504,14 @@ func NewConfigOptions() *configOptions { return validateChoices(rawValue, []string{"round_robin", "entry_frequency"}) }, }, + "POLLING_JITTER": { + ParsedDuration: 10 * time.Minute, + RawValue: "10", + ValueType: minuteType, + Validator: func(rawValue string) error { + return validateGreaterOrEqualThan(rawValue, 1) + }, + }, "PORT": { ParsedStringValue: "", RawValue: "", @@ -902,6 +910,10 @@ func (c *configOptions) PollingScheduler() string { return c.options["POLLING_SCHEDULER"].ParsedStringValue } +func (c *configOptions) PollingJitter() time.Duration { + return c.options["POLLING_JITTER"].ParsedDuration +} + func (c *configOptions) Port() string { return c.options["PORT"].ParsedStringValue } diff --git a/internal/model/feed.go b/internal/model/feed.go index 6c752fd217b..fbfdeea279f 100644 --- a/internal/model/feed.go +++ b/internal/model/feed.go @@ -4,12 +4,12 @@ package model // import "miniflux.app/v2/internal/model" import ( - "fmt" - "io" - "math/rand" - "time" + "fmt" + "io" + "math/rand" + "time" - "miniflux.app/v2/internal/config" + "miniflux.app/v2/internal/config" ) // List of supported schedulers. @@ -136,7 +136,7 @@ func (f *Feed) ScheduleNextCheck(weeklyCount int, refreshDelay time.Duration) ti // Use the RSS TTL field, Retry-After, Cache-Control or Expires HTTP headers if defined. interval = max(interval, refreshDelay) - // Limit the max interval value for misconfigured feeds. + // Limit the max interval value for misconfigured feeds. switch config.Opts.PollingScheduler() { case SchedulerRoundRobin: interval = min(interval, config.Opts.SchedulerRoundRobinMaxInterval()) @@ -144,21 +144,19 @@ func (f *Feed) ScheduleNextCheck(weeklyCount int, refreshDelay time.Duration) ti interval = min(interval, config.Opts.SchedulerEntryFrequencyMaxInterval()) } - // Apply a small random jitter to spread next checks and reduce thundering herds. - // Jitter range: [0, 10 minutes]. - jitterMax := 10 * time.Minute - if jitterMax > 0 { - jitter := time.Duration(rand.Int63n(int64(jitterMax + 1))) - interval += jitter - - // Re-apply max clamping after jitter to avoid exceeding configured caps. - switch config.Opts.PollingScheduler() { - case SchedulerRoundRobin: - interval = min(interval, config.Opts.SchedulerRoundRobinMaxInterval()) - case SchedulerEntryFrequency: - interval = min(interval, config.Opts.SchedulerEntryFrequencyMaxInterval()) - } - } + // Apply a small random jitter to spread next checks and reduce thundering herds. + jitterMax := config.Opts.PollingJitter() + + randomJitter := time.Duration(rand.Int63n(int64(jitterMax + 1))) + interval += randomJitter + + // Re-apply max clamping after randomJitter to avoid exceeding configured caps. + switch config.Opts.PollingScheduler() { + case SchedulerRoundRobin: + interval = min(interval, config.Opts.SchedulerRoundRobinMaxInterval()) + case SchedulerEntryFrequency: + interval = min(interval, config.Opts.SchedulerEntryFrequencyMaxInterval()) + } f.NextCheckAt = time.Now().Add(interval) return interval From 46052e3171bd0027739ae1f1e3e4efe774d4c9eb Mon Sep 17 00:00:00 2001 From: vvzvlad Date: Tue, 14 Oct 2025 16:43:33 +0300 Subject: [PATCH 3/7] refactor: remove duplication of code --- internal/model/feed.go | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/internal/model/feed.go b/internal/model/feed.go index fbfdeea279f..f577a661024 100644 --- a/internal/model/feed.go +++ b/internal/model/feed.go @@ -118,6 +118,18 @@ func (f *Feed) CheckedNow() { } } +// getMaxInterval returns the maximum allowed interval based on the configured polling scheduler. +func getMaxInterval() time.Duration { + switch config.Opts.PollingScheduler() { + case SchedulerRoundRobin: + return config.Opts.SchedulerRoundRobinMaxInterval() + case SchedulerEntryFrequency: + return config.Opts.SchedulerEntryFrequencyMaxInterval() + default: + return config.Opts.SchedulerRoundRobinMaxInterval() + } +} + // ScheduleNextCheck set "next_check_at" of a feed based on the scheduler selected from the configuration. func (f *Feed) ScheduleNextCheck(weeklyCount int, refreshDelay time.Duration) time.Duration { // Default to the global config Polling Frequency. @@ -136,13 +148,8 @@ func (f *Feed) ScheduleNextCheck(weeklyCount int, refreshDelay time.Duration) ti // Use the RSS TTL field, Retry-After, Cache-Control or Expires HTTP headers if defined. interval = max(interval, refreshDelay) - // Limit the max interval value for misconfigured feeds. - switch config.Opts.PollingScheduler() { - case SchedulerRoundRobin: - interval = min(interval, config.Opts.SchedulerRoundRobinMaxInterval()) - case SchedulerEntryFrequency: - interval = min(interval, config.Opts.SchedulerEntryFrequencyMaxInterval()) - } + // Compute scheduler cap once. Final clamping is applied after jitter. + maxInterval := getMaxInterval() // Apply a small random jitter to spread next checks and reduce thundering herds. jitterMax := config.Opts.PollingJitter() @@ -151,12 +158,7 @@ func (f *Feed) ScheduleNextCheck(weeklyCount int, refreshDelay time.Duration) ti interval += randomJitter // Re-apply max clamping after randomJitter to avoid exceeding configured caps. - switch config.Opts.PollingScheduler() { - case SchedulerRoundRobin: - interval = min(interval, config.Opts.SchedulerRoundRobinMaxInterval()) - case SchedulerEntryFrequency: - interval = min(interval, config.Opts.SchedulerEntryFrequencyMaxInterval()) - } + interval = min(interval, maxInterval) f.NextCheckAt = time.Now().Add(interval) return interval From f2a35140968cbef6c95d2c24a0edfe4712a52582 Mon Sep 17 00:00:00 2001 From: vvzvlad Date: Tue, 14 Oct 2025 17:05:38 +0300 Subject: [PATCH 4/7] feat(model): add jitter seeding --- internal/model/feed.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/internal/model/feed.go b/internal/model/feed.go index f577a661024..ef3b9342193 100644 --- a/internal/model/feed.go +++ b/internal/model/feed.go @@ -12,6 +12,11 @@ import ( "miniflux.app/v2/internal/config" ) +func init() { + // Seed global math/rand to avoid predictable jitter values in scheduling. + rand.Seed(time.Now().UnixNano()) +} + // List of supported schedulers. const ( SchedulerRoundRobin = "round_robin" From 2c3dac81e6ce14f643b7f654fcdf847523342ea3 Mon Sep 17 00:00:00 2001 From: vvzvlad Date: Tue, 14 Oct 2025 17:06:18 +0300 Subject: [PATCH 5/7] refactor: remove unused default in switch, move maxInterval line --- internal/model/feed.go | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/internal/model/feed.go b/internal/model/feed.go index ef3b9342193..e7f34b211dc 100644 --- a/internal/model/feed.go +++ b/internal/model/feed.go @@ -130,8 +130,6 @@ func getMaxInterval() time.Duration { return config.Opts.SchedulerRoundRobinMaxInterval() case SchedulerEntryFrequency: return config.Opts.SchedulerEntryFrequencyMaxInterval() - default: - return config.Opts.SchedulerRoundRobinMaxInterval() } } @@ -153,16 +151,14 @@ func (f *Feed) ScheduleNextCheck(weeklyCount int, refreshDelay time.Duration) ti // Use the RSS TTL field, Retry-After, Cache-Control or Expires HTTP headers if defined. interval = max(interval, refreshDelay) - // Compute scheduler cap once. Final clamping is applied after jitter. - maxInterval := getMaxInterval() - // Apply a small random jitter to spread next checks and reduce thundering herds. jitterMax := config.Opts.PollingJitter() randomJitter := time.Duration(rand.Int63n(int64(jitterMax + 1))) interval += randomJitter - // Re-apply max clamping after randomJitter to avoid exceeding configured caps. + // Apply max clamping after randomJitter to avoid exceeding configured caps. + maxInterval := getMaxInterval() interval = min(interval, maxInterval) f.NextCheckAt = time.Now().Add(interval) From 967de9cf07d6ec0378ec4a71a8ebe7a7062d6e83 Mon Sep 17 00:00:00 2001 From: vvzvlad Date: Wed, 15 Oct 2025 16:35:32 +0300 Subject: [PATCH 6/7] fix: linter error "feed.go:134:1: missing return" --- internal/model/feed.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/internal/model/feed.go b/internal/model/feed.go index e7f34b211dc..8fcffe52c38 100644 --- a/internal/model/feed.go +++ b/internal/model/feed.go @@ -125,12 +125,14 @@ func (f *Feed) CheckedNow() { // getMaxInterval returns the maximum allowed interval based on the configured polling scheduler. func getMaxInterval() time.Duration { - switch config.Opts.PollingScheduler() { - case SchedulerRoundRobin: - return config.Opts.SchedulerRoundRobinMaxInterval() - case SchedulerEntryFrequency: - return config.Opts.SchedulerEntryFrequencyMaxInterval() - } + switch config.Opts.PollingScheduler() { + case SchedulerRoundRobin: + return config.Opts.SchedulerRoundRobinMaxInterval() + case SchedulerEntryFrequency: + return config.Opts.SchedulerEntryFrequencyMaxInterval() + default: + return config.Opts.SchedulerRoundRobinMaxInterval() + } } // ScheduleNextCheck set "next_check_at" of a feed based on the scheduler selected from the configuration. From dfb515ed2d4575e9ce2dffd0600969cd70e1675f Mon Sep 17 00:00:00 2001 From: vvzvlad Date: Thu, 16 Oct 2025 21:32:06 +0300 Subject: [PATCH 7/7] fix: remove rand.Seed: SA1019 --- internal/model/feed.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/internal/model/feed.go b/internal/model/feed.go index 8fcffe52c38..e9e054ebde2 100644 --- a/internal/model/feed.go +++ b/internal/model/feed.go @@ -12,10 +12,6 @@ import ( "miniflux.app/v2/internal/config" ) -func init() { - // Seed global math/rand to avoid predictable jitter values in scheduling. - rand.Seed(time.Now().UnixNano()) -} // List of supported schedulers. const ( @@ -156,6 +152,7 @@ func (f *Feed) ScheduleNextCheck(weeklyCount int, refreshDelay time.Duration) ti // Apply a small random jitter to spread next checks and reduce thundering herds. jitterMax := config.Opts.PollingJitter() + // No explicit global seeding for math/rand is required since Go 1.20. randomJitter := time.Duration(rand.Int63n(int64(jitterMax + 1))) interval += randomJitter