Skip to content

Commit d4e35bb

Browse files
michaelsembwevermichaelsembwever
committed
Introduce Jenkins k8s spot-instance labels
Fixes a few small errors in jenkins-deployment.yaml Move repeated parts of the yaml (for agent declarations passed to jenkins configuration) to includes files, and move jenkins-deployment.yaml to jenkins-deployment-template.yaml The jenkins-deployment.yaml is now in .gitignore and is generated when running .build/run-ci patch by Mick Semb Wever; reviewed by xxx for CASSSANDRA-21114
1 parent fc0ee48 commit d4e35bb

File tree

10 files changed

+417
-380
lines changed

10 files changed

+417
-380
lines changed

‎.build/run-ci‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -257,6 +257,10 @@ def install_jenkins(kubeconfig: Optional[str], kubecontext: Optional[str], kube_
257257
subprocess.run(["helm", "repo", "add", "jenkins", "https://charts.jenkins.io"], check=True)
258258
subprocess.run(["helm", "repo", "update"], check=True)
259259

260+
# Generate jenkins-deployment.yaml from template
261+
print("Generating jenkins-deployment.yaml from template...")
262+
subprocess.run([str(CASSANDRA_DIR / ".jenkins/k8s/insert-templates.sh")], check=True)
263+
260264
cmd = ["helm"]
261265
if kubeconfig:
262266
cmd += ["--kubeconfig", kubeconfig]
@@ -267,7 +271,7 @@ def install_jenkins(kubeconfig: Optional[str], kubecontext: Optional[str], kube_
267271

268272
run_kubectl_command(kubeconfig, kubecontext, kube_ns,
269273
["exec", DEFAULT_POD_NAME, "--",
270-
"curl", "-sS", "https://svn.apache.org/repos/asf/comdev/project-logos/originals/cassandra-6.svg",
274+
"curl", "-sS", "https://www.apache.org/logos/originals/cassandra-4.svg",
271275
"-o", "/var/jenkins_cache/war/images/svgs/logo.svg"])
272276

273277
if result.returncode != 0:

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
# C*
44
.build/.run-ci.env
5+
.jenkins/k8s/jenkins-deployment.yaml
56
build
67
build/
78
src/gen-java/

‎.jenkins/Jenkinsfile‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,10 @@
4040
// When running builds parameterised to other architectures the corresponding labels are expected.
4141
// For example 'arm64' requires the labels: cassandra-arm64-small, cassandra-arm64-medium, cassandra-arm64-large.
4242
//
43-
// The built-in node must has the "controller" label. There must be more than two agents for each label.
43+
// Nodes running on spot instances have to have labels with `-spot` suffixes.
44+
// For example: cassandra-amd64-large-spot
45+
//
46+
// The built-in node must has the "controller" label. There must be more than two agents for all "cassandra-*" labels.
4447
//
4548
// Plugins required are:
4649
// git, workflow-job, workflow-cps, junit, workflow-aggregator, ws-cleanup, pipeline-build-step, test-stability, copyartifact.
@@ -333,7 +336,7 @@ def build(command, cell) {
333336
def nodeExclusion = ""
334337
retry(maxAttempts) {
335338
attempt++
336-
node(getNodeLabel(command, cell) + nodeExclusion) {
339+
node(getNodeLabel(command, cell, attempt != maxAttempts) + nodeExclusion) {
337340
nodeExclusion = "&&!${NODE_NAME}"
338341
withEnv(cell.collect { k, v -> "${k}=${v}" }) {
339342
ws("workspace/${JOB_NAME}/${BUILD_NUMBER}/${cell.step}/${cell.arch}/jdk-${cell.jdk}") {
@@ -376,7 +379,7 @@ def test(command, cell) {
376379
def nodeExclusion = ""
377380
retry(maxAttempts) {
378381
attempt++
379-
node(getNodeLabel(command, cell) + nodeExclusion) {
382+
node(getNodeLabel(command, cell, attempt != maxAttempts) + nodeExclusion) {
380383
nodeExclusion = "&&!${NODE_NAME}"
381384
withEnv(cell.collect { k, v -> "${k}=${v}" }) {
382385
ws("workspace/${JOB_NAME}/${BUILD_NUMBER}/${cell.step}/${cell.arch}/jdk-${cell.jdk}/python-${cell.python}") {
@@ -482,9 +485,10 @@ def fetchDockerImages(dockerfiles) {
482485
"""
483486
}
484487

485-
def getNodeLabel(command, cell) {
486-
echo "using node label: cassandra-${cell.arch}-${command.size}"
487-
return "cassandra-${cell.arch}-${command.size}"
488+
def getNodeLabel(command, cell, canSpot) {
489+
def label = "cassandra-${cell.arch}-${command.size}" (+ "-spot" if canSpot else "")
490+
echo "using node label: ${label}"
491+
return label
488492
}
489493

490494
def copyToNightlies(sourceFiles, remoteDirectory='') {

‎.jenkins/k8s/includes/dind-container-base.yaml‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
#
17+
- name: dind
18+
alwaysPullImage: 'false'
19+
envVars:
20+
- envVar:
21+
key: DOCKER_TLS_CERTDIR
22+
value: /certs
23+
- envVar:
24+
key: "DOCKER_IPTABLES_LEGACY"
25+
value: "1"
26+
image: docker:dind
27+
args: "--default-address-pool base=192.168.96.0/20,size=24" # overwrite docker subnet in case of overlapping
28+
livenessProbe:
29+
failureThreshold: '0'
30+
initialDelaySeconds: '0'
31+
periodSeconds: '0'
32+
successThreshold: '0'
33+
timeoutSeconds: '0'
34+
privileged: 'true'
35+
ttyEnabled: 'true'
36+
workingDir: /home/jenkins/agent

‎.jenkins/k8s/includes/jnlp-container-base.yaml‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
#
17+
- name: jnlp
18+
alwaysPullImage: true
19+
envVars:
20+
- envVar:
21+
key: DOCKER_TLS_CERTDIR
22+
value: /certs/client/
23+
- envVar:
24+
key: DOCKER_CERT_PATH
25+
value: /certs/client/
26+
- envVar:
27+
key: DOCKER_TLS_VERIFY
28+
value: 'true'
29+
- envVar:
30+
key: DOCKER_HOST
31+
value: tcp://localhost:2376
32+
- envVar:
33+
key: JENKINS_JAVA_OPTS
34+
value: '-Dorg.jenkinsci.plugins.durabletask.BourneShellScript.USE_BINARY_WRAPPER=true -Xlog:gc+heap+exit -XX:+HeapDumpOnOutOfMemoryError'
35+
# there's a lot of docker pulls,
36+
# TODO implement option for docker registry caches :: https://medium.com/@elementtech.dev/#kubernetes-image-proxy-cache-from-minutes-to-milliseconds-fd14173e831f
37+
image: apache.jfrog.io/cassan-docker/apache/cassandra-jenkins-k8s
38+
livenessProbe:
39+
failureThreshold: '0'
40+
initialDelaySeconds: '0'
41+
periodSeconds: '0'
42+
successThreshold: '0'
43+
timeoutSeconds: '0'
44+
privileged: 'true'
45+
ttyEnabled: 'true'
46+
workingDir: /home/jenkins/agent
47+
resourceRequestCpu: 1
48+
resourceLimitCpu: 3
49+
resourceRequestMemory: 1G
50+
resourceLimitMemory: 2G

‎.jenkins/k8s/includes/pod-template-base.yaml‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
#
17+
activeDeadlineSeconds: '0'
18+
idleMinutes: 1
19+
nodeUsageMode: "NORMAL"
20+
showRawYaml: 'true'
21+
slaveConnectTimeout: '30'
22+
yamlMergeStrategy: override

‎.jenkins/k8s/includes/volumes-base.yaml‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
#
17+
18+
- emptyDirVolume:
19+
memory: 'false'
20+
mountPath: /var/lib/docker
21+
- emptyDirVolume:
22+
memory: 'false'
23+
mountPath: /certs

‎.jenkins/k8s/insert-templates.sh‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
#!/bin/bash
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing, software
13+
# distributed under the License is distributed on an "AS IS" BASIS,
14+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
# See the License for the specific language governing permissions and
16+
# limitations under the License.
17+
#
18+
# Script to insert includes files into jenkins-deployment-template.yaml as jenkins-deployment.yaml
19+
20+
set -e
21+
22+
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
23+
INPUT="$SCRIPT_DIR/jenkins-deployment-template.yaml"
24+
OUTPUT="$SCRIPT_DIR/jenkins-deployment.yaml"
25+
26+
clean() {
27+
sed "/^#/d;"
28+
}
29+
30+
# Read template files
31+
POD_TEMPLATE=$(cat "$SCRIPT_DIR/includes/pod-template-base.yaml" | clean )
32+
JNLP_CONTAINER=$(cat "$SCRIPT_DIR/includes/jnlp-container-base.yaml" | clean )
33+
DIND_CONTAINER=$(cat "$SCRIPT_DIR/includes/dind-container-base.yaml" | clean )
34+
VOLUMES=$(cat "$SCRIPT_DIR/includes/volumes-base.yaml" | clean )
35+
36+
# Replace anchor references
37+
awk -v pod="$POD_TEMPLATE" -v jnlp="$JNLP_CONTAINER" -v dind="$DIND_CONTAINER" -v vols="$VOLUMES" '
38+
{
39+
line = $0
40+
gsub(/ <<: \*pod-template-base/, pod, line)
41+
gsub(/ <<: \*jnlp-container-base/, jnlp, line)
42+
gsub(/ <<: \*dind-container-base/, dind, line)
43+
gsub(/volumes: \*volumes-base/, "volumes: " vols, line)
44+
print line
45+
}
46+
' "$INPUT" > "$OUTPUT"

0 commit comments

Comments
 (0)