Merge cockroachdb#107645

107645: teamcity-trigger: limit the number of triggers per unit time r=rickystewart a=healthy-pod

This code change limits the number of triggered stress
runs per every 10 minutes to 20 runs. The goal of this is to avoid
overloading TeamCity.

Part of: DEVINF-834
Release note: None
Epic: none

Co-authored-by: healthy-pod <[email protected]>

Author: craig[bot]

pkg/cmd/teamcity-trigger/main.go

@@ -108,6 +108,17 @@ func main() {
 }
 
 func runTC(queueBuild func(string, map[string]string)) {
+	// queueBuildThenWait sets a limit on the rate at which we trigger TC stress
+	// build config to avoid overloading TC [see DEVINF-834].
+	currentTriggerCount := 0
+	queueBuildThenWait := func(buildID string, opts map[string]string) {
+		if currentTriggerCount == 20 {
+			currentTriggerCount = 0
+			time.Sleep(time.Minute * 10)
+		}
+		currentTriggerCount += 1
+		queueBuild(buildID, opts)
+	}
 	targets, err := exec.Command("bazel", "query", "kind(go_test, //pkg/...)", "--output=label").Output()
 	if err != nil {
 		log.Fatal(err)
@@ -163,11 +174,11 @@ func runTC(queueBuild func(string, map[string]string)) {
 
 		opts["env.STRESSFLAGS"] = fmt.Sprintf("-maxruns %d -maxtime %s -maxfails %d -p %d",
 			maxRuns, maxTime, maxFails, parallelism)
-		queueBuild(buildID, opts)
+		queueBuildThenWait(buildID, opts)
 
 		// Run non-race build with deadlock detection.
 		opts["env.TAGS"] = "deadlock"
-		queueBuild(buildID, opts)
+		queueBuildThenWait(buildID, opts)
 		delete(opts, "env.TAGS")
 
 		// Run race build. With run with -p 1 to avoid overloading the machine.
@@ -179,7 +190,7 @@ func runTC(queueBuild func(string, map[string]string)) {
 		opts["env.STRESSFLAGS"] = fmt.Sprintf("-maxruns %d -maxtime %s -maxfails %d -p %d",
 			maxRuns, maxTime, maxFails, noParallelism)
 		opts["env.TAGS"] = "race"
-		queueBuild(buildID, opts)
+		queueBuildThenWait(buildID, opts)
 		delete(opts, "env.TAGS")
 	}
 }