TUN-5262: Allow to configure max fetch size for listing queries

This can be useful/important for accounts with many tunnels that exceed
the 1000 default page size.

There are various tunnel subcommands that use listing underneath, so we make
that flag a tunnel one, rather than adding it to each subcommand.

Author: nmldiegues

cmd/cloudflared/tunnel/cmd.go

@@ -646,6 +646,12 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 			Value:  "https://api.trycloudflare.com",
 			Hidden: true,
 		}),
+		&cli.UintFlag{
+			Name:    "max-fetch-size",
+			Usage:   `The maximum number of results that cloudflared can fetch from Cloudflare API for any listing operations needed`,
+			EnvVars: []string{"TUNNEL_MAX_FETCH_SIZE"},
+			Hidden:  true,
+		},
 		selectProtocolFlag,
 		overwriteDNSFlag,
 	}...)

cmd/cloudflared/tunnel/subcommand_context.go

@@ -336,6 +336,10 @@ func (sc *subcommandContext) tunnelActive(name string) (*tunnelstore.Tunnel, boo
 	filter := tunnelstore.NewFilter()
 	filter.NoDeleted()
 	filter.ByName(name)
+	if maxFetch := sc.c.Uint("max-fetch-size"); maxFetch > 0 {
+		filter.MaxFetchSize(maxFetch)
+	}
+
 	tunnels, err := sc.list(filter)
 	if err != nil {
 		return nil, false, err
@@ -385,6 +389,10 @@ func (sc *subcommandContext) findIDs(inputs []string) ([]uuid.UUID, error) {
 	// First, look up all tunnels the user has
 	filter := tunnelstore.NewFilter()
 	filter.NoDeleted()
+	if maxFetch := sc.c.Uint("max-fetch-size"); maxFetch > 0 {
+		filter.MaxFetchSize(maxFetch)
+	}
+
 	tunnels, err := sc.list(filter)
 	if err != nil {
 		return nil, err

cmd/cloudflared/tunnel/subcommands.go

@@ -277,6 +277,9 @@ func listCommand(c *cli.Context) error {
 		}
 		filter.ByTunnelID(tunnelID)
 	}
+	if maxFetch := c.Uint("max-fetch-size"); maxFetch > 0 {
+		filter.MaxFetchSize(maxFetch)
+	}
 
 	tunnels, err := sc.list(filter)
 	if err != nil {

tunnelstore/filter.go

@@ -2,6 +2,7 @@ package tunnelstore
 
 import (
 	"net/url"
+	"strconv"
 	"time"
 
 	"github.com/google/uuid"
@@ -45,6 +46,10 @@ func (f *Filter) ByTunnelID(tunnelID uuid.UUID) {
 	f.queryParams.Set("uuid", tunnelID.String())
 }
 
+func (f *Filter) MaxFetchSize(max uint) {
+	f.queryParams.Set("per_page", strconv.Itoa(int(max)))
+}
+
 func (f Filter) encode() string {
 	return f.queryParams.Encode()
 }