Skip to content
This repository was archived by the owner on Jul 30, 2021. It is now read-only.

Commit 8f2eb4c

Browse files
rithujohn191rithujohn191
authored
Merge pull request #976 from rithujohn191/conformance-migration
hack/jenkins: add conformance tests
2 parents efed818 + 6ccb061 commit 8f2eb4c

File tree

3 files changed

+374
-0
lines changed

3 files changed

+374
-0
lines changed

‎hack/jenkins/jobs/bootkube_conformance_cncf.groovy‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
// META
2+
repo = "kubernetes-incubator/bootkube"
3+
4+
// CONFIG
5+
org_whitelist = ['coreos', 'coreos-inc']
6+
job_admins = ['ericchiang', 'rithujohn191', 'rphillips']
7+
user_whitelist = job_admins
8+
9+
// JOBS
10+
job_name = "tku-bootkube-conformance-cncf"
11+
12+
pipelineJob(job_name) {
13+
parameters {
14+
stringParam('sha1', 'master', 'git reference to build')
15+
}
16+
definition {
17+
cpsScm {
18+
scm {
19+
git {
20+
remote {
21+
github("${repo}")
22+
refspec('+refs/heads/*:refs/remotes/origin/* +refs/pull/*:refs/remotes/origin/pr/*')
23+
credentials('github_userpass')
24+
}
25+
branch('${sha1}')
26+
}
27+
}
28+
scriptPath('hack/jenkins/pipelines/bootkube-conformance/Jenkinsfile')
29+
}
30+
triggers {
31+
cron('H 11 * * *')
32+
}
33+
}
34+
}

‎hack/jenkins/pipelines/bootkube-conformance/Jenkinsfile‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
// Declarative Pipeline used by `bootkube-conformance` job
2+
3+
def bash(String cmd) { sh("#/usr/bin/env bash\nset -euo pipefail\n${cmd}") }
4+
5+
pipeline {
6+
agent {
7+
kubernetes {
8+
cloud 'kubernetes'
9+
label "${JOB_NAME}-${BUILD_NUMBER}"
10+
containerTemplate {
11+
name 'default'
12+
image 'quay.io/coreos/bootkube-e2e-builder:v0.1'
13+
ttyEnabled true
14+
command 'cat'
15+
}
16+
}
17+
}
18+
options {
19+
timeout(time: 3, unit: 'HOURS')
20+
ansiColor('xterm')
21+
timestamps()
22+
skipDefaultCheckout(true)
23+
}
24+
environment {
25+
CLUSTER_NAME="${JOB_NAME}-${BUILD_NUMBER}"
26+
ARTIFACT_DIR="${WORKSPACE}/artifacts"
27+
REGION = "us-west-2"
28+
GOPATH = "${WORKSPACE}"
29+
WORKDIR = "${WORKSPACE}/src/github.com/kubernetes-incubator/bootkube"
30+
KUBECONFIG = "${WORKSPACE}/src/github.com/kubernetes-incubator/bootkube/hack/quickstart/cluster/auth/kubeconfig"
31+
IDENT = "${WORKSPACE}/src/github.com/kubernetes-incubator/bootkube/hack/quickstart/cluster/auth/id_rsa"
32+
BOOTKUBE_OPTS = "--strict"
33+
34+
AWS_CRED = credentials('aws')
35+
ACCESS_KEY_ID = "${AWS_CRED_USR}"
36+
ACCESS_KEY_SECRET = "${AWS_CRED_PSW}"
37+
}
38+
stages {
39+
stage('checkout') {
40+
steps {
41+
// jnlp slave runs as "jenkins" user, use the escape hatch. (https://hub.docker.com/r/jenkins/slave/~/dockerfile/)
42+
bash "chmod -R go+rw /home/jenkins"
43+
bash "mkdir -p \"${ARTIFACT_DIR}\""
44+
dir("${WORKDIR}") {
45+
checkout scm
46+
}
47+
}
48+
}
49+
stage('build') {
50+
steps {
51+
dir("${WORKDIR}") {
52+
bash "make release |& tee -a ${ARTIFACT_DIR}/build.log"
53+
}
54+
}
55+
}
56+
stage('deploy') {
57+
steps {
58+
dir("${WORKDIR}") {
59+
bash "./hack/jenkins/scripts/tqs-up.sh |& tee -a ${ARTIFACT_DIR}/deploy.log"
60+
}
61+
}
62+
}
63+
stage('conformance') {
64+
steps {
65+
dir("${WORKDIR}") {
66+
bash "./hack/jenkins/scripts/conformance.sh |& tee -a ${ARTIFACT_DIR}/conformance.log"
67+
}
68+
}
69+
}
70+
}
71+
post {
72+
always {
73+
script {
74+
stage('collect-logs') {
75+
bash "${WORKDIR}/hack/jenkins/scripts/gather-logs.sh || true"
76+
bash "cp -r ${WORKDIR}/hack/quickstart/logs-** ${ARTIFACT_DIR}/ || true"
77+
}
78+
stage('cleanup') {
79+
bash "(${WORKDIR}/hack/jenkins/scripts/tqs-down.sh || true) |& tee -a ${ARTIFACT_DIR}/cleanup.log"
80+
}
81+
stage('archive-logs') {
82+
dir("${ARTIFACT_DIR}") {
83+
archiveArtifacts '**/*'
84+
withAWS(credentials: 'aws', region: "${REGION}") {
85+
bash "tar -czf /tmp/artifacts.tar.gz ."
86+
bash "mv /tmp/artifacts.tar.gz \"${ARTIFACT_DIR}/artifacts-${JOB_NAME}-${BUILD_NUMBER}.tar.gz\""
87+
// note: do not use includepathpattern! https://issues.jenkins-ci.org/browse/JENKINS-47046
88+
s3Upload(acl: 'PublicRead', bucket: 'bootkube-pr-logs', path: "pr/${JOB_NAME}-${BUILD_NUMBER}/",
89+
file: "${ARTIFACT_DIR}")
90+
}
91+
}
92+
}
93+
}
94+
}
95+
}
96+
}
97+

‎hack/jenkins/scripts/conformance.sh‎

Lines changed: 243 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,243 @@
1+
#!/bin/bash
2+
set -euo pipefail
3+
4+
export KUBECONFIG="${KUBECONFIG:-"${DIR}/../../quickstart/cluster/auth/kubeconfig"}"
5+
export KUBERNETES_VERSION="v1.10.2"
6+
7+
# Set up kubectl
8+
curl -L -O -v https://storage.googleapis.com/kubernetes-release/release/$KUBERNETES_VERSION/bin/linux/amd64/kubectl
9+
chmod +x ./kubectl
10+
export PATH=$PATH:$PWD
11+
echo "kubectl setup completed"
12+
13+
# sonobuoy
14+
# Conformance yaml file from https://raw.githubusercontent.com/cncf/k8s-conformance/master/sonobuoy-conformance.yaml
15+
cat << EOF >> conformance.yaml
16+
---
17+
apiVersion: v1
18+
kind: Namespace
19+
metadata:
20+
name: sonobuoy
21+
---
22+
apiVersion: v1
23+
kind: ServiceAccount
24+
metadata:
25+
labels:
26+
component: sonobuoy
27+
name: sonobuoy-serviceaccount
28+
namespace: sonobuoy
29+
---
30+
apiVersion: rbac.authorization.k8s.io/v1beta1
31+
kind: ClusterRoleBinding
32+
metadata:
33+
labels:
34+
component: sonobuoy
35+
name: sonobuoy-serviceaccount
36+
roleRef:
37+
apiGroup: rbac.authorization.k8s.io
38+
kind: ClusterRole
39+
name: sonobuoy-serviceaccount
40+
subjects:
41+
- kind: ServiceAccount
42+
name: sonobuoy-serviceaccount
43+
namespace: sonobuoy
44+
---
45+
apiVersion: rbac.authorization.k8s.io/v1beta1
46+
kind: ClusterRole
47+
metadata:
48+
labels:
49+
component: sonobuoy
50+
name: sonobuoy-serviceaccount
51+
namespace: sonobuoy
52+
rules:
53+
- apiGroups:
54+
- '*'
55+
resources:
56+
- '*'
57+
verbs:
58+
- '*'
59+
---
60+
apiVersion: v1
61+
kind: ConfigMap
62+
metadata:
63+
labels:
64+
component: sonobuoy
65+
name: sonobuoy-config-cm
66+
namespace: sonobuoy
67+
data:
68+
config.json: |
69+
{
70+
"Description": "CNCF v1.9 or v.1.10 Conformance Results",
71+
"Filters": {
72+
"LabelSelector": "",
73+
"Namespaces": ".*"
74+
},
75+
"PluginNamespace": "sonobuoy",
76+
"Plugins": [
77+
{
78+
"name": "e2e"
79+
}
80+
],
81+
"Resources": [
82+
],
83+
"ResultsDir": "/tmp/sonobuoy",
84+
"Server": {
85+
"advertiseaddress": "sonobuoy-master:8080",
86+
"bindaddress": "0.0.0.0",
87+
"bindport": 8080,
88+
"timeoutseconds": 5400
89+
},
90+
"Version": "v0.10.0"
91+
}
92+
---
93+
apiVersion: v1
94+
kind: ConfigMap
95+
metadata:
96+
labels:
97+
component: sonobuoy
98+
name: sonobuoy-plugins-cm
99+
namespace: sonobuoy
100+
data:
101+
e2e.tmpl: |
102+
apiVersion: v1
103+
kind: Pod
104+
metadata:
105+
annotations:
106+
sonobuoy-driver: Job
107+
sonobuoy-plugin: e2e
108+
sonobuoy-result-type: e2e
109+
labels:
110+
component: sonobuoy
111+
sonobuoy-run: '{{.SessionID}}'
112+
tier: analysis
113+
name: sonobuoy-e2e-job-{{.SessionID}}
114+
namespace: '{{.Namespace}}'
115+
spec:
116+
containers:
117+
- env:
118+
- name: E2E_FOCUS
119+
value: '\[Conformance\]'
120+
image: gcr.io/heptio-images/kube-conformance:v1.10
121+
imagePullPolicy: Always
122+
name: e2e
123+
volumeMounts:
124+
- mountPath: /tmp/results
125+
name: results
126+
readOnly: false
127+
- command:
128+
- sh
129+
- -c
130+
- /sonobuoy worker global -v 5 --logtostderr
131+
env:
132+
- name: NODE_NAME
133+
valueFrom:
134+
fieldRef:
135+
apiVersion: v1
136+
fieldPath: spec.nodeName
137+
- name: RESULTS_DIR
138+
value: /tmp/results
139+
- name: MASTER_URL
140+
value: '{{.MasterAddress}}'
141+
- name: RESULT_TYPE
142+
value: e2e
143+
image: gcr.io/heptio-images/sonobuoy:v0.10.0
144+
imagePullPolicy: Always
145+
name: sonobuoy-worker
146+
volumeMounts:
147+
- mountPath: /tmp/results
148+
name: results
149+
readOnly: false
150+
restartPolicy: Never
151+
serviceAccountName: sonobuoy-serviceaccount
152+
tolerations:
153+
- effect: NoSchedule
154+
key: node-role.kubernetes.io/master
155+
operator: Exists
156+
- key: CriticalAddonsOnly
157+
operator: Exists
158+
volumes:
159+
- emptyDir: {}
160+
name: results
161+
---
162+
apiVersion: v1
163+
kind: Pod
164+
metadata:
165+
labels:
166+
component: sonobuoy
167+
run: sonobuoy-master
168+
tier: analysis
169+
name: sonobuoy
170+
namespace: sonobuoy
171+
spec:
172+
containers:
173+
- command:
174+
- /bin/bash
175+
- -c
176+
- /sonobuoy master --no-exit=true -v 3 --logtostderr
177+
env:
178+
- name: SONOBUOY_ADVERTISE_IP
179+
valueFrom:
180+
fieldRef:
181+
fieldPath: status.podIP
182+
image: gcr.io/heptio-images/sonobuoy:v0.10.0
183+
imagePullPolicy: Always
184+
name: kube-sonobuoy
185+
volumeMounts:
186+
- mountPath: /etc/sonobuoy
187+
name: sonobuoy-config-volume
188+
- mountPath: /plugins.d
189+
name: sonobuoy-plugins-volume
190+
- mountPath: /tmp/sonobuoy
191+
name: output-volume
192+
restartPolicy: Never
193+
serviceAccountName: sonobuoy-serviceaccount
194+
volumes:
195+
- configMap:
196+
name: sonobuoy-config-cm
197+
name: sonobuoy-config-volume
198+
- configMap:
199+
name: sonobuoy-plugins-cm
200+
name: sonobuoy-plugins-volume
201+
- emptyDir: {}
202+
name: output-volume
203+
---
204+
apiVersion: v1
205+
kind: Service
206+
metadata:
207+
labels:
208+
component: sonobuoy
209+
run: sonobuoy-master
210+
name: sonobuoy-master
211+
namespace: sonobuoy
212+
spec:
213+
ports:
214+
- port: 8080
215+
protocol: TCP
216+
targetPort: 8080
217+
selector:
218+
run: sonobuoy-master
219+
type: ClusterIP
220+
EOF
221+
222+
echo "Waiting for cluster to be up and ready"
223+
# Wait for cluster to be ready
224+
readyNodes() {
225+
kubectl --kubeconfig="${KUBECONFIG}" get nodes -o template --template='{{range .items}}{{range .status.conditions}}{{if eq .type "Ready"}}{{.}}{{end}}{{end}}{{end}}' | grep -o -E True | wc -l
226+
}
227+
228+
until [[ "$(readyNodes)" == "2" ]]; do
229+
echo "$(readyNodes) of 2 nodes are Ready..."
230+
sleep 10
231+
done
232+
233+
echo "Applying conformance tests"
234+
kubectl --kubeconfig="${KUBECONFIG}" apply -f conformance.yaml
235+
236+
until [[ "$(kubectl --kubeconfig="${KUBECONFIG}" logs sonobuoy -n sonobuoy --tail=1)" == *"no-exit was specified, sonobuoy is now blocking"* ]]; do
237+
echo "Waiting for sonobuoy results"
238+
kubectl --kubeconfig="${KUBECONFIG}" logs sonobuoy -n sonobuoy --tail=1 || true
239+
sleep 10
240+
done
241+
242+
# Copy results from pod to Jenkins executor
243+
kubectl --kubeconfig="${KUBECONFIG}" cp sonobuoy/sonobuoy:/tmp/sonobuoy ${ARTIFACT_DIR}/conformance-results

0 commit comments

Comments
 (0)