support for --critical-load-interval-millis

this optional flag gives --critical-load a second chance. When configured to positive value, meeting with critical-load spawns a timer. When this timer expires a second check for critical-load is made; if still met, gh-ost bails out.
By default the interval is zero, in which case gh-ost bails out immediately on meeting critical load.

Author: Shlomi Noach

go/base/context.go

@@ -90,6 +90,7 @@ type MigrationContext struct {
 	ThrottleCommandedByUser             int64
 	maxLoad                             LoadMap
 	criticalLoad                        LoadMap
+	CriticalLoadIntervalMilliseconds    int64
 	PostponeCutOverFlagFile             string
 	CutOverLockTimeoutSeconds           int64
 	ForceNamedCutOverCommand            bool

go/cmd/gh-ost/main.go

@@ -99,6 +99,7 @@ func main() {
 
 	maxLoad := flag.String("max-load", "", "Comma delimited status-name=threshold. e.g: 'Threads_running=100,Threads_connected=500'. When status exceeds threshold, app throttles writes")
 	criticalLoad := flag.String("critical-load", "", "Comma delimited status-name=threshold, same format as `--max-load`. When status exceeds threshold, app panics and quits")
+	flag.Int64Var(&migrationContext.CriticalLoadIntervalMilliseconds, "critical-load-interval-millis", 0, "When 0, migration bails out upon meeting critical-load immediately. When non-zero, a second check is done after given interval, and migration only bails out if 2nd check still meets critical load")
 	quiet := flag.Bool("quiet", false, "quiet")
 	verbose := flag.Bool("verbose", false, "verbose")
 	debug := flag.Bool("debug", false, "debug mode (very verbose)")

go/logic/throttler.go

@@ -130,6 +130,20 @@ func (this *Throttler) collectControlReplicasLag() {
 	}
 }
 
+func (this *Throttler) criticalLoadIsMet() (met bool, variableName string, value int64, threshold int64, err error) {
+	criticalLoad := this.migrationContext.GetCriticalLoad()
+	for variableName, threshold = range criticalLoad {
+		value, err = this.applier.ShowStatusVariable(variableName)
+		if err != nil {
+			return false, variableName, value, threshold, err
+		}
+		if value >= threshold {
+			return true, variableName, value, threshold, nil
+		}
+	}
+	return false, variableName, value, threshold, nil
+}
+
 // collectGeneralThrottleMetrics reads the once-per-sec metrics, and stores them onto this.migrationContext
 func (this *Throttler) collectGeneralThrottleMetrics() error {
 
@@ -144,15 +158,23 @@ func (this *Throttler) collectGeneralThrottleMetrics() error {
 			this.migrationContext.PanicAbort <- fmt.Errorf("Found panic-file %s. Aborting without cleanup", this.migrationContext.PanicFlagFile)
 		}
 	}
-	criticalLoad := this.migrationContext.GetCriticalLoad()
-	for variableName, threshold := range criticalLoad {
-		value, err := this.applier.ShowStatusVariable(variableName)
-		if err != nil {
-			return setThrottle(true, fmt.Sprintf("%s %s", variableName, err))
-		}
-		if value >= threshold {
-			this.migrationContext.PanicAbort <- fmt.Errorf("critical-load met: %s=%d, >=%d", variableName, value, threshold)
-		}
+
+	criticalLoadMet, variableName, value, threshold, err := this.criticalLoadIsMet()
+	if err != nil {
+		return setThrottle(true, fmt.Sprintf("%s %s", variableName, err))
+	}
+	if criticalLoadMet && this.migrationContext.CriticalLoadIntervalMilliseconds == 0 {
+		this.migrationContext.PanicAbort <- fmt.Errorf("critical-load met: %s=%d, >=%d", variableName, value, threshold)
+	}
+	if criticalLoadMet && this.migrationContext.CriticalLoadIntervalMilliseconds > 0 {
+		log.Errorf("critical-load met once: %s=%d, >=%d. Will check again in %d millis", variableName, value, threshold, this.migrationContext.CriticalLoadIntervalMilliseconds)
+		go func() {
+			timer := time.NewTimer(time.Millisecond * time.Duration(this.migrationContext.CriticalLoadIntervalMilliseconds))
+			<-timer.C
+			if criticalLoadMetAgain, variableName, value, threshold, _ := this.criticalLoadIsMet(); criticalLoadMetAgain {
+				this.migrationContext.PanicAbort <- fmt.Errorf("critical-load met again after %d millis: %s=%d, >=%d", this.migrationContext.CriticalLoadIntervalMilliseconds, variableName, value, threshold)
+			}
+		}()
 	}
 
 	// Back to throttle considerations