Merge #148821 #149367

148821: backup: add retry loop to `doDownloadFiles` r=dt a=kev-cao

Previously, `doDownloadFiles` would immediately fail the download job upon encountering an error. This commit teaches the download workers to retry the download requests before giving up and failing.

Epic: CRDB-51394

Fixes: #148081

Release note: Download phase of fast restore now will retry downloads before giving up.

149367: storage: bump pebble, enable blob-file rewrites metamorphically r=annrpom a=annrpom

### go.mod: bump Pebble to a2947b4a9383
Changes:

 * [`a2947b4a`](cockroachdb/pebble@a2947b4a) cmd/pebble: enable blob file rewriting
 * [`5e1cd273`](cockroachdb/pebble@5e1cd273) compression: deflake TestAdaptiveCompressorCompressible
 * [`652bc0ca`](cockroachdb/pebble@652bc0ca) ascii/table: new package
 * [`2430d94a`](cockroachdb/pebble@2430d94a) internal/ascii: new package
 * [`7d17fe51`](cockroachdb/pebble@7d17fe51) metamorphic: enable blob file rewrite compactions
 * [`09c33960`](cockroachdb/pebble@09c33960) blob: deduplicate values during rewrite
 * [`8e65e154`](cockroachdb/pebble@8e65e154) pebble: setupInputs checks key range conflicts with in progress compactions
 * [`8dea64b3`](cockroachdb/pebble@8dea64b3) db: incorporate separated values when estimating compression ratios
 * [`9fd475aa`](cockroachdb/pebble@9fd475aa) db: add TestTableStats test case exercising separated values
 * [`3b556400`](cockroachdb/pebble@3b556400) tool: add tool for inspecting blob files
 * [`c726afea`](cockroachdb/pebble@c726afea) sstable: add blob value liveness writer random test
 * [`c0d37a5b`](cockroachdb/pebble@c0d37a5b) metamorphic: use leaktest in metamorphic test
 * [`4ad0e5fb`](cockroachdb/pebble@4ad0e5fb) Revert "sstable: lazy load the index block in single level iterator"
 * [`f5351564`](cockroachdb/pebble@f5351564) Revert "sstable: lazy load the index block in two level iterator"
 * [`46650aa8`](cockroachdb/pebble@46650aa8) github: add crl-release-25.3 to nightlies and crossversion
 * [`11cf2c4a`](cockroachdb/pebble@11cf2c4a) pebble: deflake move compaction datadriven test
 * [`fe43e7b5`](cockroachdb/pebble@fe43e7b5) sstable: lazy load the index block in two level iterator
 * [`f4af03ae`](cockroachdb/pebble@f4af03ae) test: introduce two level iterator benchmark test
 * [`527eebf4`](cockroachdb/pebble@527eebf4) sstable: lazy load the index block in single level iterator

Release note: none.
Epic: none.

### storage: metamorphically enable blob-file rewrites
This patch enables blob-file rewrites metamorphically.

Co-authored-by: Kevin Cao <[email protected]>
Co-authored-by: Annie Pompa <[email protected]>

Author: 3 people

DEPS.bzl

@@ -1755,10 +1755,10 @@ def go_deps():
         name = "com_github_cockroachdb_crlib",
         build_file_proto_mode = "disable_global",
         importpath = "github.com/cockroachdb/crlib",
-        sha256 = "1afc910b4ff270de79eecb42ab7bd5e6404e6128666c6c55e96db9e27d28e69e",
-        strip_prefix = "github.com/cockroachdb/[email protected]20241205160938-4a90b184f49c",
+        sha256 = "47ee61f8b5cc8327e3b3b65a93dee277137e762a73650c2f0a75a24a37361114",
+        strip_prefix = "github.com/cockroachdb/[email protected]20250617202621-0794c595bbe6",
         urls = [
-            "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/crlib/com_github_cockroachdb_crlib-v0.0.0-20241205160938-4a90b184f49c.zip",
+            "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/crlib/com_github_cockroachdb_crlib-v0.0.0-20250617202621-0794c595bbe6.zip",
         ],
     )
     go_repository(
@@ -1835,10 +1835,10 @@ def go_deps():
         patches = [
             "@com_github_cockroachdb_cockroach//build/patches:com_github_cockroachdb_pebble.patch",
         ],
-        sha256 = "05c54b72d105a3bfa4a7dd5739be84d72596df37022c6c6df577b59650f87203",
-        strip_prefix = "github.com/cockroachdb/[email protected]20250627191025-c40280e0c860",
+        sha256 = "877549371ab3baba36d81ba2e10be21179453709b8f3f105e21017f3f0f561b4",
+        strip_prefix = "github.com/cockroachdb/[email protected]20250701175647-a2947b4a9383",
         urls = [
-            "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20250627191025-c40280e0c860.zip",
+            "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20250701175647-a2947b4a9383.zip",
         ],
     )
     go_repository(

build/bazelutil/distdir_files.bzl

@@ -347,7 +347,7 @@ DISTDIR_FILES = {
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/cmux/com_github_cockroachdb_cmux-v0.0.0-20250514152509-914d3bf9ec58.zip": "c1cf4cd99a1ad6a00f2ccd4188cbcf004cb0d56895670b2c171061ce564cd791",
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/cockroach-go/v2/com_github_cockroachdb_cockroach_go_v2-v2.4.1.zip": "ba646db91152f3121a6812c7b74d12d8c0e126f7b4d3b927618b159692ceb424",
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/crlfmt/com_github_cockroachdb_crlfmt-v0.0.0-20221214225007-b2fc5c302548.zip": "fedc01bdd6d964da0425d5eaac8efadc951e78e13f102292cc0774197f09ab63",
-    "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/crlib/com_github_cockroachdb_crlib-v0.0.0-20241205160938-4a90b184f49c.zip": "1afc910b4ff270de79eecb42ab7bd5e6404e6128666c6c55e96db9e27d28e69e",
+    "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/crlib/com_github_cockroachdb_crlib-v0.0.0-20250617202621-0794c595bbe6.zip": "47ee61f8b5cc8327e3b3b65a93dee277137e762a73650c2f0a75a24a37361114",
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/datadriven/com_github_cockroachdb_datadriven-v1.0.3-0.20250407164829-2945557346d5.zip": "251593cd9c040fe84a99a3919de7ce6f85030d522159a37d625dc2dea7a4d17f",
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/drpc/com_github_cockroachdb_drpc-v0.0.0-20250618091105-e79a954a2193.zip": "612016b7a145f386a2163d788e3376cb0f63410b8423f2ef3f723f7aa24c1971",
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/errors/com_github_cockroachdb_errors-v1.12.0.zip": "f73d8a5f4d8fcbc4ed61db2b47f17e2601d8b32e9a49c0665667489d9d9d6e7c",
@@ -357,7 +357,7 @@ DISTDIR_FILES = {
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/gostdlib/com_github_cockroachdb_gostdlib-v1.19.0.zip": "c4d516bcfe8c07b6fc09b8a9a07a95065b36c2855627cb3514e40c98f872b69e",
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/logtags/com_github_cockroachdb_logtags-v0.0.0-20241215232642-bb51bb14a506.zip": "920068af09e3846d9ebb4e4a7787ff1dd10f3989c5f940ad861b0f6a9f824f6e",
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/metamorphic/com_github_cockroachdb_metamorphic-v0.0.0-20231108215700-4ba948b56895.zip": "28c8cf42192951b69378cf537be5a9a43f2aeb35542908cc4fe5f689505853ea",
-    "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20250627191025-c40280e0c860.zip": "05c54b72d105a3bfa4a7dd5739be84d72596df37022c6c6df577b59650f87203",
+    "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20250701175647-a2947b4a9383.zip": "877549371ab3baba36d81ba2e10be21179453709b8f3f105e21017f3f0f561b4",
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/redact/com_github_cockroachdb_redact-v1.1.6.zip": "018eccb5fb9ca52d43ec9eaf213539d01c1f2b94e0e822406ebfb2e9321ef6cf",
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/returncheck/com_github_cockroachdb_returncheck-v0.0.0-20200612231554-92cdbca611dd.zip": "ce92ba4352deec995b1f2eecf16eba7f5d51f5aa245a1c362dfe24c83d31f82b",
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/stress/com_github_cockroachdb_stress-v0.0.0-20220803192808-1806698b1b7b.zip": "3fda531795c600daf25532a4f98be2a1335cd1e5e182c72789bca79f5f69fcc1",

go.mod

@@ -133,13 +133,13 @@ require (
 	github.com/cockroachdb/cmux v0.0.0-20250514152509-914d3bf9ec58
 	github.com/cockroachdb/cockroach-go/v2 v2.4.1
 	github.com/cockroachdb/crlfmt v0.0.0-20221214225007-b2fc5c302548
-	github.com/cockroachdb/crlib v0.0.0-20241205160938-4a90b184f49c
+	github.com/cockroachdb/crlib v0.0.0-20250617202621-0794c595bbe6
 	github.com/cockroachdb/datadriven v1.0.3-0.20250407164829-2945557346d5
 	github.com/cockroachdb/errors v1.12.0
 	github.com/cockroachdb/go-test-teamcity v0.0.0-20191211140407-cff980ad0a55
 	github.com/cockroachdb/gostdlib v1.19.0
 	github.com/cockroachdb/logtags v0.0.0-20241215232642-bb51bb14a506
-	github.com/cockroachdb/pebble v0.0.0-20250627191025-c40280e0c860
+	github.com/cockroachdb/pebble v0.0.0-20250701175647-a2947b4a9383
 	github.com/cockroachdb/redact v1.1.6
 	github.com/cockroachdb/returncheck v0.0.0-20200612231554-92cdbca611dd
 	github.com/cockroachdb/stress v0.0.0-20220803192808-1806698b1b7b

go.sum

@@ -556,8 +556,8 @@ github.com/cockroachdb/cockroach-go/v2 v2.4.1 h1:ACVT/zXsuK6waRPVYtDQpsM8pPA7IA/
 github.com/cockroachdb/cockroach-go/v2 v2.4.1/go.mod h1:9U179XbCx4qFWtNhc7BiWLPfuyMVQ7qdAhfrwLz1vH0=
 github.com/cockroachdb/crlfmt v0.0.0-20221214225007-b2fc5c302548 h1:i0bnjanlWAvM50wHMT7EFyxlt5HQusznWrkwl+HBIsU=
 github.com/cockroachdb/crlfmt v0.0.0-20221214225007-b2fc5c302548/go.mod h1:qtkxNlt5i3rrdirfJE/bQeW/IeLajKexErv7jEIV+Uc=
-github.com/cockroachdb/crlib v0.0.0-20241205160938-4a90b184f49c h1:9d6UHu3JQXisFa4JRFzCbX2Ba2ES5e0202N9qpt3VuM=
-github.com/cockroachdb/crlib v0.0.0-20241205160938-4a90b184f49c/go.mod h1:Gq51ZeKaFCXk6QwuGM0w1dnaOqc/F5zKT2zA9D6Xeac=
+github.com/cockroachdb/crlib v0.0.0-20250617202621-0794c595bbe6 h1:PZVolkXzVqPQQZ7Jm8/lGpI9ZM086mB55oOWqy0wG6E=
+github.com/cockroachdb/crlib v0.0.0-20250617202621-0794c595bbe6/go.mod h1:Gq51ZeKaFCXk6QwuGM0w1dnaOqc/F5zKT2zA9D6Xeac=
 github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
 github.com/cockroachdb/datadriven v1.0.2/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU=
 github.com/cockroachdb/datadriven v1.0.3-0.20250407164829-2945557346d5 h1:UycK/E0TkisVrQbSoxvU827FwgBBcZ95nRRmpj/12QI=
@@ -580,8 +580,8 @@ github.com/cockroachdb/logtags v0.0.0-20241215232642-bb51bb14a506 h1:ASDL+UJcILM
 github.com/cockroachdb/logtags v0.0.0-20241215232642-bb51bb14a506/go.mod h1:Mw7HqKr2kdtu6aYGn3tPmAftiP3QPX63LdK/zcariIo=
 github.com/cockroachdb/metamorphic v0.0.0-20231108215700-4ba948b56895 h1:XANOgPYtvELQ/h4IrmPAohXqe2pWA8Bwhejr3VQoZsA=
 github.com/cockroachdb/metamorphic v0.0.0-20231108215700-4ba948b56895/go.mod h1:aPd7gM9ov9M8v32Yy5NJrDyOcD8z642dqs+F0CeNXfA=
-github.com/cockroachdb/pebble v0.0.0-20250627191025-c40280e0c860 h1:KW5Jo6irucGRYI3i6ZShw64tIK9japdTHu9KumyQDEw=
-github.com/cockroachdb/pebble v0.0.0-20250627191025-c40280e0c860/go.mod h1:+E2g3dO+px5IVuHgq5o+O5iy7fRK2eJLdckgrbtWzeE=
+github.com/cockroachdb/pebble v0.0.0-20250701175647-a2947b4a9383 h1:vaCj5U8yIg9ZFnuKBgFM5qvQIPNlA/IVmrKO7tYq6a8=
+github.com/cockroachdb/pebble v0.0.0-20250701175647-a2947b4a9383/go.mod h1:LLa+I2vZrOsrsy4QcuZ6wSXBNQll7erNg3GCr8NpE90=
 github.com/cockroachdb/redact v1.1.3/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
 github.com/cockroachdb/redact v1.1.6 h1:zXJBwDZ84xJNlHl1rMyCojqyIxv+7YUpQiJLQ7n4314=
 github.com/cockroachdb/redact v1.1.6/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=

pkg/backup/restore_online.go

@@ -64,6 +64,7 @@ var onlineRestoreLayerLimit = settings.RegisterIntSetting(
 )
 
 const linkCompleteKey = "link_complete"
+const maxDownloadAttempts = 5
 
 // splitAndScatter runs through all entries produced by genSpans splitting and
 // scattering the key-space designated by the passed rewriter such that if all
@@ -556,15 +557,34 @@ func (r *restoreResumer) sendDownloadWorker(
 		ctx, tsp := tracing.ChildSpan(ctx, "backup.sendDownloadWorker")
 		defer tsp.Finish()
 
-		for rt := retry.StartWithCtx(
-			ctx, retry.Options{InitialBackoff: time.Millisecond * 100, MaxBackoff: time.Second * 10},
-		); ; rt.Next() {
+		testingKnobs := execCtx.ExecCfg().BackupRestoreTestingKnobs
+		for {
 			if err := ctx.Err(); err != nil {
 				return err
 			}
 
-			if err := sendDownloadSpan(ctx, execCtx, spans); err != nil {
-				return err
+			var err error
+			for r := retry.StartWithCtx(ctx, retry.Options{
+				InitialBackoff: time.Millisecond * 100,
+				MaxBackoff:     time.Second,
+				MaxRetries:     maxDownloadAttempts - 1,
+			}); r.Next(); {
+				err = func() error {
+					if testingKnobs != nil && testingKnobs.RunBeforeSendingDownloadSpan != nil {
+						if err := testingKnobs.RunBeforeSendingDownloadSpan(); err != nil {
+							return err
+						}
+					}
+					return sendDownloadSpan(ctx, execCtx, spans)
+				}()
+				if err == nil {
+					break
+				}
+				log.VInfof(ctx, 1, "attempt %d failed to download spans: %v", r.CurrentAttempt(), err)
+			}
+
+			if err != nil {
+				return errors.Wrapf(err, "retries exhausted for sending download spans")
 			}
 
 			// Wait for the completion poller to signal that it has checked our work.
@@ -576,6 +596,11 @@ func (r *restoreResumer) sendDownloadWorker(
 			case <-ctx.Done():
 				return ctx.Err()
 			}
+
+			// Sleep a bit before sending download requests again to avoid a hot loop.
+			// This will only be hit if after a successful download request, there are
+			// still spans to download (e.g. because of a rabalancing).
+			time.Sleep(10 * time.Second)
 		}
 	}
 }

pkg/backup/restore_online_test.go

@@ -14,6 +14,7 @@ import (
 	"path/filepath"
 	"reflect"
 	"strings"
+	"sync/atomic"
 	"testing"
 
 	"github.com/cockroachdb/cockroach/pkg/base"
@@ -32,6 +33,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/util/log"
 	"github.com/cockroachdb/cockroach/pkg/util/randutil"
 	"github.com/cockroachdb/cockroach/pkg/util/stop"
+	"github.com/cockroachdb/errors"
 	"github.com/kr/pretty"
 	"github.com/stretchr/testify/require"
 )
@@ -581,6 +583,65 @@ func TestOnlineRestoreErrors(t *testing.T) {
 	})
 }
 
+func TestOnlineRestoreRetryingDownloadRequests(t *testing.T) {
+	defer leaktest.AfterTest(t)()
+	defer log.Scope(t).Close(t)
+
+	defer nodelocal.ReplaceNodeLocalForTesting(t.TempDir())()
+
+	rng, seed := randutil.NewPseudoRand()
+	t.Logf("random seed: %d", seed)
+
+	alwaysFail := rng.Intn(2) == 0
+	t.Logf("always fail download requests: %t", alwaysFail)
+	totalFailures := int32(rng.Intn(maxDownloadAttempts-1) + 1)
+	var currentFailures atomic.Int32
+
+	clusterArgs := base.TestClusterArgs{
+		ServerArgs: base.TestServerArgs{
+			DefaultTestTenant: base.TestIsSpecificToStorageLayerAndNeedsASystemTenant,
+			Knobs: base.TestingKnobs{
+				BackupRestore: &sql.BackupRestoreTestingKnobs{
+					RunBeforeSendingDownloadSpan: func() error {
+						if alwaysFail {
+							return errors.Newf("always fail download request")
+						}
+						if currentFailures.Load() >= totalFailures {
+							return nil
+						}
+						currentFailures.Add(1)
+						return errors.Newf("injected download request failure")
+					},
+				},
+			},
+		},
+	}
+
+	const numAccounts = 1
+	_, sqlDB, _, cleanupFn := backupRestoreTestSetupWithParams(
+		t, singleNode, numAccounts, InitManualReplication, clusterArgs,
+	)
+	defer cleanupFn()
+
+	externalStorage := "nodelocal://1/backup"
+	sqlDB.Exec(t, fmt.Sprintf("BACKUP INTO '%s'", externalStorage))
+	sqlDB.Exec(
+		t,
+		fmt.Sprintf(`
+		RESTORE DATABASE data FROM LATEST IN '%s'
+		WITH EXPERIMENTAL DEFERRED COPY, new_db_name=data2
+		`, externalStorage),
+	)
+
+	var downloadJobID jobspb.JobID
+	sqlDB.QueryRow(t, latestDownloadJobIDQuery).Scan(&downloadJobID)
+	if alwaysFail {
+		jobutils.WaitForJobToFail(t, sqlDB, downloadJobID)
+	} else {
+		jobutils.WaitForJobToSucceed(t, sqlDB, downloadJobID)
+	}
+}
+
 func bankOnlineRestore(
 	t *testing.T,
 	sqlDB *sqlutils.SQLRunner,

pkg/server/span_download.go

@@ -18,12 +18,15 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/storage"
 	"github.com/cockroachdb/cockroach/pkg/util/ctxgroup"
 	"github.com/cockroachdb/cockroach/pkg/util/log"
+	"github.com/cockroachdb/cockroach/pkg/util/retry"
 	"github.com/cockroachdb/errors"
 	"github.com/cockroachdb/errors/errorspb"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
 )
 
+var perNodeMaxDownloadAttempts = 5
+
 func (t *statusServer) DownloadSpan(
 	ctx context.Context, req *serverpb.DownloadSpanRequest,
 ) (*serverpb.DownloadSpanResponse, error) {
@@ -62,8 +65,31 @@ func (s *systemStatusServer) DownloadSpan(
 	// Send DownloadSpan request to all stores on all nodes.
 	remoteRequest := *req
 	remoteRequest.NodeID = "local"
-	nodeFn := func(ctx context.Context, status serverpb.RPCStatusClient, _ roachpb.NodeID) (*serverpb.DownloadSpanResponse, error) {
-		return status.DownloadSpan(ctx, &remoteRequest)
+	nodeFn := func(
+		ctx context.Context, status serverpb.RPCStatusClient, _ roachpb.NodeID,
+	) (*serverpb.DownloadSpanResponse, error) {
+		var nodeResp *serverpb.DownloadSpanResponse
+		var err error
+		for r := retry.StartWithCtx(ctx, retry.Options{
+			InitialBackoff: time.Millisecond * 100,
+			MaxBackoff:     time.Second,
+			MaxRetries:     perNodeMaxDownloadAttempts - 1,
+		}); r.Next(); {
+			nodeResp, err = status.DownloadSpan(ctx, &remoteRequest)
+			for node, encodedErr := range resp.Errors {
+				err = errors.Wrapf(
+					errors.DecodeError(ctx, encodedErr),
+					"download span request failed with error on node n%d",
+					node,
+				)
+				break
+			}
+			if err == nil {
+				break
+			}
+			log.VInfof(ctx, 1, "attempt %d failed to download span: %v", r.CurrentAttempt(), err)
+		}
+		return nodeResp, err
 	}
 	responseFn := func(nodeID roachpb.NodeID, downloadSpanResp *serverpb.DownloadSpanResponse) {
 		for i, e := range downloadSpanResp.Errors {

pkg/sql/exec_util_backup.go

@@ -69,6 +69,10 @@ type BackupRestoreTestingKnobs struct {
 	RunAfterRestoreProcDrains func()
 
 	RunBeforeResolvingCompactionDest func() error
+
+	// RunBeforeSendingDownloadSpan is called within the retry loop of the
+	// download span worker before sending the download span request.
+	RunBeforeSendingDownloadSpan func() error
 }
 
 var _ base.ModuleTestingKnobs = &BackupRestoreTestingKnobs{}

pkg/storage/pebble.go

@@ -456,7 +456,9 @@ var (
 		"storage.value_separation.compaction_garbage_threshold",
 		"the max garbage threshold configures the percentage of unreferenced value "+
 			"bytes that trigger blob-file rewrite compactions; 100 disables these compactions",
-		100, /* default; disables blob-file rewrites */
+		int64(metamorphic.ConstantWithTestRange("storage.value_separation.compaction_garbage_threshold",
+			10, /* default */
+			1 /* min */, 80 /* max */)),
 		settings.IntInRange(1, 100),
 	)
 )