Skip to content
This repository was archived by the owner on Apr 17, 2019. It is now read-only.

Commit 6baa2d5

Browse files
lavalamplavalamp
authored
Merge pull request #1228 from lavalamp/ff4
add build cop tools
2 parents cc7b564 + 4e07e05 commit 6baa2d5

File tree

5 files changed

+188
-47
lines changed

5 files changed

+188
-47
lines changed

‎mungegithub/build-cop/README.md‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
# Build Cop tools
2+
3+
This directory contains tools for build cops. They give an easier access to the
4+
merge queue's admin console.
5+
6+
**They require you to have kubectl installed and pointing at the cluster running
7+
the submit queue.**
8+
9+
## resolved.sh
10+
11+
Use `resolved.sh` to mark a test as manually resolved (or not), so the merge
12+
queue will merge in spite of a failure. Arguments are the job name and run number.
13+
14+
## emergency.sh
15+
16+
Run `emengency.sh` to halt merges. Run `emergency.sh resume` to start them
17+
again.

‎mungegithub/build-cop/emergency.sh‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
#!/bin/bash
2+
3+
# Copyright 2016 The Kubernetes Authors All rights reserved.
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
set -o errexit
18+
set -o nounset
19+
set -o pipefail
20+
21+
STOP=${1:-"stop"}
22+
PORT=${PORT:-9999}
23+
24+
SELF_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
25+
source ${SELF_DIR}/util.sh
26+
27+
port-forward &>/dev/null &
28+
PID=$!
29+
trap "kill $PID" EXIT
30+
31+
if ! wait-for-port; then
32+
exit 1
33+
fi
34+
35+
echo "About to ${STOP} merges. (Options: 'stop' or 'resume')"
36+
curl "localhost:${PORT}/api/emergency/${STOP}"
37+
echo "" # output doesn't include a trailing newline

‎mungegithub/build-cop/resolve.sh‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
#!/bin/bash
2+
3+
# Copyright 2016 The Kubernetes Authors All rights reserved.
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
set -o errexit
18+
set -o nounset
19+
set -o pipefail
20+
21+
JOB=${1:-}
22+
NUMBER=${2:-}
23+
RESOLVED=${3:-true}
24+
PORT=${PORT:-9999}
25+
26+
SELF_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
27+
source ${SELF_DIR}/util.sh
28+
29+
port-forward &>/dev/null &
30+
PID=$!
31+
trap "kill $PID" EXIT
32+
33+
if ! wait-for-port; then
34+
exit 1
35+
fi
36+
37+
if [[ "${JOB}" == "" ]]; then
38+
curl "localhost:${PORT}/api/list-resolutions"
39+
echo "" # output doesn't include a trailing newline
40+
echo "add/remove a resolution: '$0 jobname runnumber [true/false]'"
41+
exit 0
42+
fi
43+
44+
echo "About to set ${JOB}/${NUMBER} resolved=${RESOLVED}."
45+
curl "localhost:${PORT}/api/mark-resolved?job=${JOB}&number=${NUMBER}&resolved=${RESOLVED}"
46+
echo "" # output doesn't include a trailing newline

‎mungegithub/build-cop/util.sh‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
#!/bin/bash
2+
3+
# Copyright 2016 The Kubernetes Authors All rights reserved.
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
set -o errexit
18+
set -o nounset
19+
set -o pipefail
20+
21+
function port-forward {
22+
PORT=${PORT:-9999}
23+
PODNAME=$(kubectl get endpoints submit-queue-status -o template --template='{{index . "subsets" 0 "addresses" 0 "targetRef" "name"}}')
24+
echo "Admin interface will serve on port ${PORT}. Press ctrl-c to stop serving."
25+
kubectl port-forward "${PODNAME}" "${PORT}:9999"
26+
}
27+
28+
function wait-for-port {
29+
PORT=${PORT:-9999}
30+
for (( i=0; i < 20; i++ )); do
31+
sleep .25
32+
if curl -s2 "localhost:${PORT}/" 2>&1 > /dev/null; then
33+
echo "Admin port opened"
34+
return 0
35+
fi
36+
done
37+
echo "Admin port never became responsive."
38+
return 1
39+
}

‎mungegithub/mungers/e2e/e2e.go‎

Lines changed: 49 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -162,6 +162,52 @@ func (e *RealE2ETester) getGCSResult(j cache.Job, n cache.Number) (*cache.Result
162162
return r, nil
163163
}
164164

165+
func (e *RealE2ETester) checkPassFail(job string, number int) (stable, ignorableFlakes bool) {
166+
if e.resolutionTracker.Resolved(cache.Job(job), cache.Number(number)) {
167+
e.setBuildStatus(job, "Problem Resolved", strconv.Itoa(number))
168+
return true, true
169+
}
170+
171+
thisResult, err := e.GetBuildResult(job, number)
172+
if err != nil || thisResult.Status == cache.ResultFailed {
173+
glog.V(4).Infof("Found unstable job: %v, build number: %v: (err: %v) %#v", job, number, err, thisResult)
174+
e.setBuildStatus(job, "Not Stable", strconv.Itoa(number))
175+
return false, false
176+
}
177+
178+
if thisResult.Status == cache.ResultStable {
179+
e.setBuildStatus(job, "Stable", strconv.Itoa(number))
180+
return true, false
181+
}
182+
183+
lastResult, err := e.GetBuildResult(job, number-1)
184+
if err != nil || lastResult.Status == cache.ResultFailed {
185+
glog.V(4).Infof("prev job doesn't help: %v, build number: %v (the previous build); (err %v) %#v", job, number-1, err, lastResult)
186+
e.setBuildStatus(job, "Not Stable", strconv.Itoa(number))
187+
return true, false
188+
}
189+
190+
if lastResult.Status == cache.ResultStable {
191+
e.setBuildStatus(job, "Ignorable flake", strconv.Itoa(number))
192+
return true, true
193+
}
194+
195+
intersection := sets.NewString()
196+
for testName := range thisResult.Flakes {
197+
if _, ok := lastResult.Flakes[testName]; ok {
198+
intersection.Insert(string(testName))
199+
}
200+
}
201+
if len(intersection) == 0 {
202+
glog.V(2).Infof("Ignoring failure of %v/%v since it didn't happen the previous run this run = %v; prev run = %v.", job, number, thisResult.Flakes, lastResult.Flakes)
203+
e.setBuildStatus(job, "Ignorable flake", strconv.Itoa(number))
204+
return true, true
205+
}
206+
glog.V(2).Infof("Failure of %v/%v is legit. Tests that failed multiple times in a row: %v", job, number, intersection)
207+
e.setBuildStatus(job, "Not Stable", strconv.Itoa(number))
208+
return false, false
209+
}
210+
165211
// GCSBasedStable is a version of Stable function that depends on files stored in GCS instead of Jenkis
166212
func (e *RealE2ETester) GCSBasedStable() (allStable, ignorableFlakes bool) {
167213
allStable = true
@@ -175,53 +221,9 @@ func (e *RealE2ETester) GCSBasedStable() (allStable, ignorableFlakes bool) {
175221
continue
176222
}
177223

178-
if e.resolutionTracker.Resolved(cache.Job(job), cache.Number(lastBuildNumber)) {
179-
e.setBuildStatus(job, "Problem Resolved", strconv.Itoa(lastBuildNumber))
180-
continue
181-
}
182-
183-
thisResult, err := e.GetBuildResult(job, lastBuildNumber)
184-
if err != nil || thisResult.Status == cache.ResultFailed {
185-
glog.V(4).Infof("Found unstable job: %v, build number: %v: (err: %v) %#v", job, lastBuildNumber, err, thisResult)
186-
e.setBuildStatus(job, "Not Stable", strconv.Itoa(lastBuildNumber))
187-
allStable = false
188-
continue
189-
}
190-
191-
if thisResult.Status == cache.ResultStable {
192-
e.setBuildStatus(job, "Stable", strconv.Itoa(lastBuildNumber))
193-
continue
194-
}
195-
196-
lastResult, err := e.GetBuildResult(job, lastBuildNumber-1)
197-
if err != nil || lastResult.Status == cache.ResultFailed {
198-
glog.V(4).Infof("prev job doesn't help: %v, build number: %v (the previous build); (err %v) %#v", job, lastBuildNumber-1, err, lastResult)
199-
allStable = false
200-
e.setBuildStatus(job, "Not Stable", strconv.Itoa(lastBuildNumber))
201-
continue
202-
}
203-
204-
if lastResult.Status == cache.ResultStable {
205-
ignorableFlakes = true
206-
e.setBuildStatus(job, "Ignorable flake", strconv.Itoa(lastBuildNumber))
207-
continue
208-
}
209-
210-
intersection := sets.NewString()
211-
for testName := range thisResult.Flakes {
212-
if _, ok := lastResult.Flakes[testName]; ok {
213-
intersection.Insert(string(testName))
214-
}
215-
}
216-
if len(intersection) == 0 {
217-
glog.V(2).Infof("Ignoring failure of %v/%v since it didn't happen the previous run this run = %v; prev run = %v.", job, lastBuildNumber, thisResult.Flakes, lastResult.Flakes)
218-
ignorableFlakes = true
219-
e.setBuildStatus(job, "Ignorable flake", strconv.Itoa(lastBuildNumber))
220-
continue
221-
}
222-
glog.V(2).Infof("Failure of %v/%v is legit. Tests that failed multiple times in a row: %v", job, lastBuildNumber, intersection)
223-
allStable = false
224-
e.setBuildStatus(job, "Not Stable", strconv.Itoa(lastBuildNumber))
224+
stable, flakes := e.checkPassFail(job, lastBuildNumber)
225+
allStable = allStable && stable
226+
ignorableFlakes = ignorableFlakes || flakes
225227
}
226228

227229
// Also get status for non-blocking jobs

0 commit comments

Comments
 (0)