Add emergency stop button

Author: lavalamp

mungegithub/mungers/submit-queue.go

@@ -24,6 +24,7 @@ import (
 	"net/http"
 	"sort"
 	"strconv"
+	"strings"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -175,6 +176,8 @@ type SubmitQueue struct {
 	health        submitQueueHealth
 	healthHistory []healthRecord
 
+	emergencyMergeStopFlag int32
+
 	adminPort int
 }
 
@@ -194,10 +197,38 @@ func init() {
 }
 
 // Name is the name usable in --pr-mungers
-func (sq SubmitQueue) Name() string { return "submit-queue" }
+func (sq *SubmitQueue) Name() string { return "submit-queue" }
 
 // RequiredFeatures is a slice of 'features' that must be provided
-func (sq SubmitQueue) RequiredFeatures() []string { return []string{} }
+func (sq *SubmitQueue) RequiredFeatures() []string { return []string{} }
+
+func (sq *SubmitQueue) emergencyMergeStop() bool {
+	return atomic.LoadInt32(&sq.emergencyMergeStopFlag) != 0
+}
+
+func (sq *SubmitQueue) setEmergencyMergeStop(stopMerges bool) {
+	if stopMerges {
+		atomic.StoreInt32(&sq.emergencyMergeStopFlag, 1)
+	} else {
+		atomic.StoreInt32(&sq.emergencyMergeStopFlag, 0)
+	}
+}
+
+// EmergencyStopHTTP sets the emergency stop flag. It expects the path of
+// req.URL to contain either "emergency/stop", "emergency/resume", or "emergency/status".
+func (sq *SubmitQueue) EmergencyStopHTTP(res http.ResponseWriter, req *http.Request) {
+	switch {
+	case strings.Contains(req.URL.Path, "emergency/stop"):
+		sq.setEmergencyMergeStop(true)
+	case strings.Contains(req.URL.Path, "emergency/resume"):
+		sq.setEmergencyMergeStop(false)
+	case strings.Contains(req.URL.Path, "emergency/status"):
+	default:
+		http.NotFound(res, req)
+		return
+	}
+	sq.serve(sq.marshal(struct{ EmergencyInProgress bool }{sq.emergencyMergeStop()}), res, req)
+}
 
 func round(num float64) int {
 	return int(num + math.Copysign(0.5, num))
@@ -353,6 +384,10 @@ func (sq *SubmitQueue) internalInitialize(config *github.Config, features *featu
 		go http.ListenAndServe(config.Address, nil)
 	}
 
+	admin.Mux.HandleFunc("/api/emergency/stop", sq.EmergencyStopHTTP)
+	admin.Mux.HandleFunc("/api/emergency/resume", sq.EmergencyStopHTTP)
+	admin.Mux.HandleFunc("/api/emergency/status", sq.EmergencyStopHTTP)
+
 	if sq.githubE2EPollTime == 0 {
 		sq.githubE2EPollTime = githubE2EPollTime
 	}
@@ -431,21 +466,26 @@ func (sq *SubmitQueue) updateHealth() {
 	}
 	// Make the current record
 	stable, _ := sq.e2e.GCSBasedStable()
+	emergencyStop := sq.emergencyMergeStop()
 	newEntry := healthRecord{
 		Time:    time.Now(),
-		Overall: stable,
+		Overall: stable && !emergencyStop,
 		Jobs:    map[string]bool{},
 	}
 	for job, status := range sq.e2e.GetBuildStatus() {
 		// Ignore flakes.
 		newEntry.Jobs[job] = status.Status != "Not Stable"
 	}
+	if emergencyStop {
+		// invent an "emergency stop" job that's failing.
+		newEntry.Jobs["Emergency Stop"] = false
+	}
 	sq.healthHistory = append(sq.healthHistory, newEntry)
 	// Now compute the health structure so we don't have to do it on page load
 	sq.health.TotalLoops = len(sq.healthHistory)
 	sq.health.NumStable = 0
 	sq.health.NumStablePerJob = map[string]int{}
-	sq.health.MergePossibleNow = stable
+	sq.health.MergePossibleNow = stable && !emergencyStop
 	for _, record := range sq.healthHistory {
 		if record.Overall {
 			sq.health.NumStable += 1
@@ -466,6 +506,9 @@ func (sq *SubmitQueue) e2eStable(aboutToMerge bool) bool {
 	wentUnstable := false
 
 	stable, ignorableFlakes := sq.e2e.GCSBasedStable()
+	if stable && sq.emergencyMergeStop() {
+		stable = false
+	}
 
 	weakStable := sq.e2e.GCSWeakStable()
 	if !weakStable {

mungegithub/mungers/submit-queue_test.go

@@ -446,6 +446,9 @@ func TestSubmitQueue(t *testing.T) {
 		reason           string
 		state            string // what the github status context should be for the PR HEAD
 
+		emergencyMergeStop bool
+		isMerged           bool
+
 		imHeadSHA      string
 		imBaseSHA      string
 		masterCommit   *github.RepositoryCommit
@@ -466,6 +469,25 @@ func TestSubmitQueue(t *testing.T) {
 			retest2Pass:     true,
 			reason:          merged,
 			state:           "success",
+			isMerged:        true,
+		},
+		// Entire thing was run and good, but emergency merge stop in progress
+		{
+			name:               "Test1+emergencyStop",
+			pr:                 ValidPR(),
+			issue:              LGTMIssue(),
+			events:             NewLGTMEvents(),
+			commits:            Commits(), // Modified at time.Unix(7), 8, and 9
+			ciStatus:           SuccessStatus(),
+			lastBuildNumber:    LastBuildNumber(),
+			gcsResult:          SuccessGCS(),
+			weakResults:        map[int]utils.FinishedFile{LastBuildNumber(): SuccessGCS()},
+			retest1Pass:        true,
+			retest2Pass:        true,
+			emergencyMergeStop: true,
+			isMerged:           false,
+			reason:             e2eFailure,
+			state:              "success",
 		},
 		// Should pass without running tests because we had a previous run.
 		// TODO: Add a proper test to make sure we don't shuffle queue when we can just merge a PR
@@ -483,6 +505,7 @@ func TestSubmitQueue(t *testing.T) {
 			retest2Pass:     true,
 			reason:          merged,
 			state:           "success",
+			isMerged:        true,
 			retestsAvoided:  1,
 			imHeadSHA:       "mysha", // Set by ValidPR
 			imBaseSHA:       "mastersha",
@@ -520,6 +543,7 @@ func TestSubmitQueue(t *testing.T) {
 			retest2Pass:     false,
 			reason:          merged,
 			state:           "success",
+			isMerged:        true,
 		},
 		// Fail because PR can't automatically merge
 		{
@@ -853,6 +877,7 @@ func TestSubmitQueue(t *testing.T) {
 		config.PendingWaitTime = &d
 
 		stateSet := ""
+		wasMerged := false
 
 		numTestChecks := 0
 		path := "/foo/latest-build.txt"
@@ -963,6 +988,7 @@ func TestSubmitQueue(t *testing.T) {
 			}
 			w.Write(data)
 			test.pr.Merged = boolPtr(true)
+			wasMerged = true
 		})
 		path = fmt.Sprintf("/repos/o/r/statuses/%s", *test.pr.Head.SHA)
 		mux.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) {
@@ -987,6 +1013,7 @@ func TestSubmitQueue(t *testing.T) {
 		})
 
 		sq := getTestSQ(true, config, server)
+		sq.setEmergencyMergeStop(test.emergencyMergeStop)
 
 		obj := github_util.TestObject(config, test.issue, test.pr, test.commits, test.events)
 		if test.imBaseSHA != "" && test.imHeadSHA != "" {
@@ -1037,6 +1064,9 @@ func TestSubmitQueue(t *testing.T) {
 		if test.state != "" && test.state != stateSet {
 			t.Errorf("%d:%q state set to %q but expected %q", testNum, test.name, stateSet, test.state)
 		}
+		if test.isMerged != wasMerged {
+			t.Errorf("%d:%q PR merged = %v but wanted %v", testNum, test.name, wasMerged, test.isMerged)
+		}
 		if e, a := test.retestsAvoided, int(sq.retestsAvoided); e != a {
 			t.Errorf("%d:%q expected %v tests avoided but got %v", testNum, test.name, e, a)
 		}