Add config for waiting for multiple successful cycles until rendering (#880)

* don't proceed until we've been at 0 queries for several cycles in a row

* rename var

Author: mmandrus

pkg/config/config.go

@@ -322,6 +322,8 @@ type BrowserConfig struct {
 	// ReadinessTimeout is the maximum time to wait for the web-page to become ready (i.e. no longer loading anything).
 	ReadinessTimeout           time.Duration
 	ReadinessIterationInterval time.Duration
+	// ReadinessWaitForNQueryCycles is the number of readiness checks that must pass consecutively before considering the page ready. This handles the case where queries drop to 0 briefly before incrementing again.
+	ReadinessWaitForNQueryCycles int
 	// ReadinessPriorWait is the time to wait before checking for how ready the page is.
 	// This lets you force the webpage to take a beat and just do its thing before the service starts looking for whether it's time to render anything.
 	ReadinessPriorWait              time.Duration
@@ -447,6 +449,17 @@ func BrowserFlags() []cli.Flag {
 				return nil
 			},
 		},
+		&cli.IntFlag{
+			Name:  "browser.readiness.wait-for-n-query-cycles",
+			Usage: "The number of readiness checks that must pass consecutively before considering the page ready.",
+			Value: 1,
+			Validator: func(i int) error {
+				if i < 1 {
+					return fmt.Errorf("browser readiness wait-for-n-query-cycles must be at least 1 (got %d)", i)
+				}
+				return nil
+			},
+		},
 		&cli.DurationFlag{
 			Name:    "browser.readiness.prior-wait",
 			Usage:   "The time to wait before checking for how ready the page is. This lets you force the webpage to take a beat and just do its thing before the service starts looking for whether it's time to render anything. If <= 0, this is disabled.",
@@ -596,6 +609,7 @@ func BrowserConfigFromCommand(c *cli.Command) (BrowserConfig, error) {
 		TimeBetweenScrolls:              c.Duration("browser.time-between-scrolls"),
 		ReadinessTimeout:                c.Duration("browser.readiness.timeout"),
 		ReadinessIterationInterval:      c.Duration("browser.readiness.iteration-interval"),
+		ReadinessWaitForNQueryCycles:    c.Int("browser.readiness.wait-for-n-query-cycles"),
 		ReadinessPriorWait:              c.Duration("browser.readiness.prior-wait"),
 		ReadinessDisableQueryWait:       c.Bool("browser.readiness.disable-query-wait"),
 		ReadinessFirstQueryTimeout:      c.Duration("browser.readiness.give-up-on-first-query"),

pkg/service/browser.go

@@ -994,6 +994,7 @@ func waitForReady(browserCtx context.Context, cfg config.BrowserConfig) chromedp
 		}
 
 		hasSeenAnyQuery := false
+		numSuccessfulCycles := 0
 
 		var domHashCode int
 		initialDOMPass := true
@@ -1034,6 +1035,10 @@ func waitForReady(browserCtx context.Context, cfg config.BrowserConfig) chromedp
 				} else if !hasSeenAnyQuery && (cfg.ReadinessFirstQueryTimeout <= 0 || time.Since(start) < cfg.ReadinessFirstQueryTimeout) {
 					span.AddEvent("no first query detected yet; giving it more time")
 					continue
+				} else if numSuccessfulCycles+1 < cfg.ReadinessWaitForNQueryCycles {
+					numSuccessfulCycles++
+					span.AddEvent("waiting for more successful readiness cycles", trace.WithAttributes(attribute.Int("currentCycle", numSuccessfulCycles), attribute.Int("requiredCycles", cfg.ReadinessWaitForNQueryCycles)))
+					continue // need more successful cycles
 				}
 			}