feat(feed): use exponential backoff for broken feeds

Add an exponential backoff for broken feeds, with an upper-limit of one week.

See #3334 for context/discussions.

Author: jvoisin

internal/model/feed.go

@@ -6,6 +6,7 @@ package model // import "miniflux.app/v2/internal/model"
 import (
 	"fmt"
 	"io"
+	"math"
 	"time"
 
 	"miniflux.app/v2/internal/config"
@@ -20,6 +21,9 @@ const (
 	DefaultFeedSortingDirection = "desc"
 )
 
+// A feed with parsing errors will be checked at least every week.
+var backoffMax = time.Duration(time.Hour * 24 * 7)
+
 // Feed represents a feed in the application.
 type Feed struct {
 	ID                          int64     `json:"id"`
@@ -143,10 +147,24 @@ func (f *Feed) ScheduleNextCheck(weeklyCount int, refreshDelay time.Duration) ti
 		interval = min(interval, config.Opts.SchedulerEntryFrequencyMaxInterval())
 	}
 
+	interval += backoff(f.ParsingErrorCount)
+
 	f.NextCheckAt = time.Now().Add(interval)
 	return interval
 }
 
+func backoff(count int) time.Duration {
+	if count == 0 {
+		return 0
+	}
+	// https://en.wikipedia.org/wiki/Exponential_backoff
+	backoff := time.Duration(math.Pow(2, float64(count))) * time.Hour
+	if backoff > backoffMax {
+		return backoffMax
+	}
+	return backoff
+}
+
 // FeedCreationRequest represents the request to create a feed.
 type FeedCreationRequest struct {
 	FeedURL                     string `json:"feed_url"`

internal/model/feed_test.go

@@ -376,3 +376,50 @@ func TestFeedScheduleNextCheckEntryFrequencyLargeNewTTL(t *testing.T) {
 		t.Error(`The next_check_at should be after timeBefore + entry frequency min interval`)
 	}
 }
+
+func TestFeedScheduleNextCheckParsingErrorsBackoff(t *testing.T) {
+	//timeBefore := time.Now()
+	for count := range 10 {
+		f1 := &Feed{ParsingErrorCount: 0}
+		f2 := &Feed{ParsingErrorCount: count}
+		newTTL := time.Duration(1) * time.Minute * 2
+		f1.ScheduleNextCheck(0, newTTL)
+		f2.ScheduleNextCheck(0, newTTL)
+
+		if f1.NextCheckAt.IsZero() {
+			t.Error(`The next_check_at for f1 must be set`)
+		}
+		if f2.NextCheckAt.IsZero() {
+			t.Error(`The next_check_at for f2 must be set`)
+		}
+
+		if f1.NextCheckAt.Add(backoff(count)).Sub(f2.NextCheckAt).Minutes() > 10 {
+			t.Error("The next_check_at should have been using the exponential backoff.")
+		}
+	}
+
+}
+
+func TestFeedScheduleNextCheckParsingErrorsBackoffMax(t *testing.T) {
+	f1 := &Feed{ParsingErrorCount: 0}
+	newTTL := time.Duration(1) * time.Minute * 2
+	f1.ScheduleNextCheck(0, newTTL)
+	if f1.NextCheckAt.IsZero() {
+		t.Error(`The next_check_at for f1 must be set`)
+	}
+
+	for count := range 128 {
+		f2 := &Feed{ParsingErrorCount: count}
+		f2.ScheduleNextCheck(0, newTTL)
+
+		if f2.NextCheckAt.IsZero() {
+			t.Error(`The next_check_at for f1 must be set`)
+		}
+
+		offset := f2.NextCheckAt.Sub(f1.NextCheckAt)
+		if offset > backoffMax+time.Minute*1 {
+			t.Errorf("The next_check_at's offset for errors (%q) for %d errors should never be bigger than %q.", offset, count, backoffMax)
+		}
+	}
+
+}