Skip to content

Commit 5c40a5c

Browse files
ceph-jenkinsceph-jenkins
committed
Add Containerfile and build.sh to build it.
The intent is to replace ceph-container.git, at first for ci containers only, and eventually production containers as well. There is code present for production containers, including a separate "make-manifest-list.py" to scan for and glue the two arch-specific containers into a 'manifest-list' 'fat' container, but that code is not yet fully tested. This code will not be used until a corresponding change to the Jenkins jobs in ceph-build.git is pushed. Note that this tooling does not authenticate to the container repo; it is assumed that will be done elsewhere. Authentication is verified by pushing a minimal image to the requested repo. Signed-off-by: Dan Mick <[email protected]>
1 parent c37a00a commit 5c40a5c

File tree

3 files changed

+548
-0
lines changed

3 files changed

+548
-0
lines changed

‎container/Containerfile‎

Lines changed: 209 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,209 @@
1+
ARG FROM_IMAGE="quay.io/centos/centos:stream9"
2+
FROM $FROM_IMAGE
3+
4+
# allow FROM_IMAGE to be visible inside this stage
5+
ARG FROM_IMAGE
6+
7+
# Ceph branch name
8+
ARG CEPH_REF="main"
9+
10+
# Ceph SHA1
11+
ARG CEPH_SHA1
12+
13+
# Ceph git repo (ceph-ci.git or ceph.git)
14+
ARG CEPH_GIT_REPO
15+
16+
# (optional) Define the baseurl= for the ganesha.repo
17+
ARG GANESHA_REPO_BASEURL="https://buildlogs.centos.org/centos/\$releasever-stream/storage/\$basearch/nfsganesha-5/"
18+
19+
# (optional) Set to "crimson" to install crimson packages.
20+
ARG OSD_FLAVOR="default"
21+
22+
# (optional) Should be 'true' for CI builds (pull from shaman, etc.)
23+
ARG CI_CONTAINER="true"
24+
25+
RUN /bin/echo -e "\
26+
FROM_IMAGE: ${FROM_IMAGE}\n\
27+
CEPH_REF: ${CEPH_REF}\n\
28+
GANESHA_REPO_BASEURL: ${GANESHA_REPO_BASEURL} \n\
29+
OSD_FLAVOR: ${OSD_FLAVOR} \n\
30+
CI_CONTAINER: ${CI_CONTAINER}"
31+
32+
# Other labels are set automatically by container/build github action
33+
# See: https://github.com/opencontainers/image-spec/blob/main/annotations.md
34+
LABEL org.opencontainers.image.authors="Ceph Release Team <[email protected]>" \
35+
org.opencontainers.image.documentation="https://docs.ceph.com/"
36+
37+
LABEL \
38+
FROM_IMAGE=${FROM_IMAGE} \
39+
CEPH_REF=${CEPH_REF} \
40+
CEPH_SHA1=${CEPH_SHA1} \
41+
CEPH_GIT_REPO=${CEPH_GIT_REPO} \
42+
GANESHA_REPO_BASEURL=${GANESHA_REPO_BASEURL} \
43+
OSD_FLAVOR=${OSD_FLAVOR}
44+
45+
46+
#===================================================================================================
47+
# Install ceph and dependencies, and clean up
48+
# IMPORTANT: in official builds, use '--squash' build option to keep image as small as possible
49+
# keeping run steps separate makes local rebuilds quick, but images are big without squash option
50+
#===================================================================================================
51+
52+
# Pre-reqs
53+
RUN dnf install -y --setopt=install_weak_deps=False epel-release jq
54+
55+
# Add NFS-Ganesha repo
56+
RUN \
57+
echo "[ganesha]" > /etc/yum.repos.d/ganesha.repo && \
58+
echo "name=ganesha" >> /etc/yum.repos.d/ganesha.repo && \
59+
echo "baseurl=${GANESHA_REPO_BASEURL}" >> /etc/yum.repos.d/ganesha.repo && \
60+
echo "gpgcheck=0" >> /etc/yum.repos.d/ganesha.repo && \
61+
echo "enabled=1" >> /etc/yum.repos.d/ganesha.repo
62+
63+
# ISCSI repo
64+
RUN set -x && \
65+
curl -s -L https://shaman.ceph.com/api/repos/tcmu-runner/main/latest/centos/9/repo?arch=$(arch) -o /etc/yum.repos.d/tcmu-runner.repo && \
66+
case "${CEPH_REF}" in \
67+
quincy|reef) \
68+
curl -s -L https://download.ceph.com/ceph-iscsi/3/rpm/el9/ceph-iscsi.repo -o /etc/yum.repos.d/ceph-iscsi.repo ;\
69+
;;\
70+
main|*) \
71+
curl -s -L https://shaman.ceph.com/api/repos/ceph-iscsi/main/latest/centos/9/repo -o /etc/yum.repos.d/ceph-iscsi.repo ;\
72+
;;\
73+
esac
74+
75+
# Ceph repo
76+
RUN set -x && \
77+
rpm --import 'https://download.ceph.com/keys/release.asc' && \
78+
ARCH=$(arch); if [ "${ARCH}" == "aarch64" ]; then ARCH="arm64"; fi ;\
79+
IS_RELEASE=0 ;\
80+
if [[ "${CI_CONTAINER}" == "true" ]] ; then \
81+
# TODO: this can return different ceph builds (SHA1) for x86 vs. arm runs. is it important to fix?
82+
REPO_URL=$(curl -s "https://shaman.ceph.com/api/search/?project=ceph&distros=centos/9/${ARCH}&flavor=${OSD_FLAVOR}&ref=${CEPH_REF}&sha1=latest" | jq -r .[0].url) ;\
83+
else \
84+
IS_RELEASE=1 ;\
85+
REPO_URL="http://download.ceph.com/rpm-${CEPH_REF}/el9/" ;\
86+
fi && \
87+
rpm -Uvh "$REPO_URL/noarch/ceph-release-1-${IS_RELEASE}.el9.noarch.rpm"
88+
89+
# Copr repos
90+
# scikit for mgr-diskprediction-local
91+
# ref: https://github.com/ceph/ceph-container/pull/1821
92+
RUN \
93+
dnf install -y --setopt=install_weak_deps=False dnf-plugins-core && \
94+
dnf copr enable -y tchaikov/python-scikit-learn
95+
96+
# Update package mgr
97+
RUN dnf update -y --setopt=install_weak_deps=False
98+
99+
# Define and install packages
100+
# General
101+
RUN echo "ca-certificates" > packages.txt
102+
# Ceph
103+
# TODO: remove lua-devel and luarocks once they are present in ceph.spec.in
104+
# ref: https://github.com/ceph/ceph/pull/54575#discussion_r1401199635
105+
RUN echo \
106+
"ceph-common \
107+
ceph-exporter \
108+
ceph-grafana-dashboards \
109+
ceph-immutable-object-cache \
110+
ceph-mds \
111+
ceph-mgr-cephadm \
112+
ceph-mgr-dashboard \
113+
ceph-mgr-diskprediction-local \
114+
ceph-mgr-k8sevents \
115+
ceph-mgr-rook \
116+
ceph-mgr \
117+
ceph-mon \
118+
ceph-osd \
119+
ceph-radosgw lua-devel luarocks \
120+
ceph-volume \
121+
cephfs-mirror \
122+
cephfs-top \
123+
kmod \
124+
libradosstriper1 \
125+
rbd-mirror" \
126+
>> packages.txt
127+
128+
# Optional crimson package(s)
129+
RUN if [ "${OSD_FLAVOR}" == "crimson" ]; then \
130+
echo "ceph-crimson-osd" >> packages.txt ; \
131+
fi
132+
133+
# Ceph "Recommends"
134+
RUN echo "nvme-cli python3-saml smartmontools" >> packages.txt
135+
# NFS-Ganesha
136+
RUN echo "\
137+
dbus-daemon \
138+
nfs-ganesha-ceph \
139+
nfs-ganesha-rados-grace \
140+
nfs-ganesha-rados-urls \
141+
nfs-ganesha-rgw \
142+
nfs-ganesha \
143+
rpcbind \
144+
sssd-client" >> packages.txt
145+
146+
# ISCSI
147+
RUN echo "ceph-iscsi tcmu-runner python3-rtslib" >> packages.txt
148+
149+
# Ceph-CSI
150+
# TODO: coordinate with @Madhu-1 to have Ceph-CSI install these itself if unused by ceph
151+
# @adk3798 does cephadm use these?
152+
RUN echo "attr ceph-fuse rbd-nbd" >> packages.txt
153+
154+
# Rook (only if packages must be in ceph container image)
155+
RUN echo "systemd-udev" >> packages.txt
156+
157+
# Util packages (should be kept to only utils that are truly very useful)
158+
# 'sgdisk' (from gdisk) is used in docs and scripts for clearing disks (could be a risk? @travisn @guits @ktdreyer ?)
159+
# 'ps' (from procps-ng) and 'hostname' are very valuable for debugging and CI
160+
# TODO: remove sg3_utils once they are moved to ceph.spec.in with libstoragemgmt
161+
# ref: https://github.com/ceph/ceph-container/pull/2013#issuecomment-1248606472
162+
RUN echo "gdisk hostname procps-ng sg3_utils e2fsprogs lvm2 gcc" >> packages.txt
163+
164+
# scikit
165+
RUN echo "python3-scikit-learn" >> packages.txt
166+
167+
# ceph-node-proxy
168+
RUN echo "ceph-node-proxy" >> packages.txt
169+
170+
RUN echo "=== PACKAGES TO BE INSTALLED ==="; cat packages.txt
171+
RUN echo "=== INSTALLING ===" ; \
172+
dnf install -y --setopt=install_weak_deps=False --setopt=skip_missing_names_on_install=False --enablerepo=crb $(cat packages.txt)
173+
174+
# XXX why isn't this done in the ganesha package?
175+
RUN mkdir -p /var/run/ganesha
176+
177+
# Disable sync with udev since the container can not contact udev
178+
RUN \
179+
sed -i -e 's/udev_rules = 1/udev_rules = 0/' \
180+
-e 's/udev_sync = 1/udev_sync = 0/' \
181+
-e 's/obtain_device_list_from_udev = 1/obtain_device_list_from_udev = 0/' \
182+
/etc/lvm/lvm.conf && \
183+
# validate the sed command worked as expected
184+
grep -sqo "udev_sync = 0" /etc/lvm/lvm.conf && \
185+
grep -sqo "udev_rules = 0" /etc/lvm/lvm.conf && \
186+
grep -sqo "obtain_device_list_from_udev = 0" /etc/lvm/lvm.conf
187+
188+
# CLEAN UP!
189+
RUN set -x && \
190+
dnf clean all && \
191+
rm -rf /var/cache/dnf/* && \
192+
rm -rf /var/lib/dnf/* && \
193+
rm -f /var/lib/rpm/__db* && \
194+
# remove unnecessary files with big impact
195+
rm -rf /etc/selinux /usr/share/{doc,man,selinux} && \
196+
# don't keep compiled python binaries
197+
find / -xdev \( -name "*.pyc" -o -name "*.pyo" \) -delete
198+
199+
# Verify that the packages installed haven't been accidentally cleaned, then
200+
# clean the package list and re-clean unnecessary RPM database files
201+
RUN rpm -q $(cat packages.txt) && rm -f /var/lib/rpm/__db* && rm -f *packages.txt
202+
203+
#
204+
# Set some envs in the container for quickly inspecting details about the build at runtime
205+
ENV CEPH_IS_DEVEL="${CI_CONTAINER}" \
206+
CEPH_REF="${CEPH_REF}" \
207+
CEPH_OSD_FLAVOR="${OSD_FLAVOR}" \
208+
FROM_IMAGE="${FROM_IMAGE}"
209+

‎container/build.sh‎

Lines changed: 175 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,175 @@
1+
#!/bin/bash -ex
2+
# vim: ts=4 sw=4 expandtab
3+
4+
# repo auth with write perms must be present (this script does not log into
5+
# CONTAINER_REPO_HOSTNAME and CONTAINER_REPO_ORGANIZATION).
6+
# If NO_PUSH is set, no login is necessary
7+
8+
9+
CFILE=${1:-Containerfile}
10+
shift || true
11+
12+
usage() {
13+
cat << EOF
14+
$0 [containerfile] (defaults to 'Containerfile')
15+
For a CI build (from ceph-ci.git, built and pushed to shaman):
16+
CI_CONTAINER: must be 'true'
17+
FLAVOR (OSD flavor, default or crimson)
18+
BRANCH (of Ceph. <remote>/<ref>)
19+
CEPH_SHA1 (of Ceph)
20+
ARCH (of build host, and resulting container)
21+
CONTAINER_REPO_HOSTNAME (quay.ceph.io, for CI, for instance)
22+
CONTAINER_REPO_ORGANIZATION (ceph-ci, for CI, for instance)
23+
CONTAINER_REPO_USERNAME
24+
CONTAINER_REPO_PASSWORD
25+
26+
For a release build: (from ceph.git, built and pushed to download.ceph.com)
27+
CI_CONTAINER: must be 'false'
28+
and you must also add
29+
VERSION (for instance, 19.1.0) for tagging the image
30+
31+
You can avoid the push step (for testing) by setting NO_PUSH to anything
32+
EOF
33+
}
34+
35+
CI_CONTAINER=${CI_CONTAINER:-false}
36+
FLAVOR=${FLAVOR:-default}
37+
# default: current checked-out branch
38+
BRANCH=${BRANCH:-$(git rev-parse --abbrev-ref HEAD)}
39+
# default: current checked-out branch
40+
CEPH_SHA1=${CEPH_SHA1:-$(git rev-parse HEAD)}
41+
# default: build host arch
42+
ARCH=${ARCH:-$(arch)}
43+
if [[ "${ARCH}" == "aarch64" ]] ; then ARCH=arm64; fi
44+
if [[ ${CI_CONTAINER} == "true" ]] ; then
45+
CONTAINER_REPO_HOSTNAME=${CONTAINER_REPO_HOSTNAME:-quay.ceph.io}
46+
CONTAINER_REPO_ORGANIZATION=${CONTAINER_REPO_ORGANIZATION:-ceph/ceph-${ARCH}}
47+
else
48+
CONTAINER_REPO_HOSTNAME=${CONTAINER_REPO_HOSTNAME:-quay.io}
49+
CONTAINER_REPO_ORGANIZATION=${CONTAINER_REPO_ORGANIZATION:-ceph/ceph}
50+
# default: most-recent annotated tag
51+
VERSION=${VERSION:-$(git describe --abbrev=0)}
52+
fi
53+
54+
# check for existence of all required variables
55+
: "${CI_CONTAINER:?}"
56+
: "${FLAVOR:?}"
57+
: "${BRANCH:?}"
58+
: "${CEPH_SHA1:?}"
59+
: "${ARCH:?}"
60+
: "${CONTAINER_REPO_HOSTNAME:?}"
61+
: "${CONTAINER_REPO_ORGANIZATION:?}"
62+
: "${CONTAINER_REPO_USERNAME:?}"
63+
: "${CONTAINER_REPO_PASSWORD:?}"
64+
if [[ ${CI_CONTAINER} != "true" ]] ; then ${VERSION:?}; fi
65+
66+
# check for valid repo auth (if pushing)
67+
ORGURL=${CONTAINER_REPO_HOSTNAME}/${CONTAINER_REPO_ORGANIZATION}
68+
MINIMAL_IMAGE=${ORGURL}/ceph:minimal-test
69+
if [[ ${NO_PUSH} != "true" ]] ; then
70+
podman rmi ${MINIMAL_IMAGE} || true
71+
echo "FROM scratch" | podman build -f - -t ${MINIMAL_IMAGE}
72+
if ! podman push ${MINIMAL_IMAGE} ; then
73+
echo "Not authenticated to ${ORGURL}; need docker/podman login?"
74+
exit 1
75+
fi
76+
podman rmi ${MINIMAL_IMAGE} | true
77+
fi
78+
79+
if [[ -z "${CEPH_GIT_REPO}" ]] ; then
80+
if [[ ${CI_CONTAINER} == "true" ]]; then
81+
CEPH_GIT_REPO=https://github.com/ceph/ceph-ci.git
82+
else
83+
CEPH_GIT_REPO=https://github.com/ceph/ceph.git
84+
fi
85+
fi
86+
87+
# BRANCH will be, say, origin/main. remove <remote>/
88+
BRANCH=${BRANCH##*/}
89+
90+
podman build --pull=newer --squash -f $CFILE -t build.sh.output \
91+
--build-arg FROM_IMAGE=${FROM_IMAGE:-quay.io/centos/centos:stream9} \
92+
--build-arg CEPH_SHA1=${CEPH_SHA1} \
93+
--build-arg CEPH_GIT_REPO=${CEPH_GIT_REPO} \
94+
--build-arg CEPH_REF=${BRANCH:-main} \
95+
--build-arg OSD_FLAVOR=${FLAVOR:-default} \
96+
--build-arg CI_CONTAINER=${CI_CONTAINER:-default} \
97+
2>&1
98+
99+
image_id=$(podman image ls localhost/build.sh.output --format '{{.ID}}')
100+
101+
# grab useful image attributes for building the tag
102+
#
103+
# the variable settings are prefixed with "export CEPH_CONTAINER_" so that
104+
# an eval or . can be used to put them into the environment
105+
#
106+
# PATH is removed from the output as it would cause problems for this
107+
# parent script and its children
108+
#
109+
# notes:
110+
#
111+
# we want .Architecture and everything in .Config.Env
112+
#
113+
# printf will not accept "\n" (is this a podman bug?)
114+
# so construct vars with two calls to podman inspect, joined by a newline,
115+
# so that vars will get the output of the first command, newline, output
116+
# of the second command
117+
#
118+
vars="$(podman inspect -f '{{printf "export CEPH_CONTAINER_ARCH=%v" .Architecture}}' ${image_id})
119+
$(podman inspect -f '{{range $index, $value := .Config.Env}}export CEPH_CONTAINER_{{$value}}{{println}}{{end}}' ${image_id})"
120+
vars="$(echo "${vars}" | grep -v PATH)"
121+
eval ${vars}
122+
123+
# remove everything up to and including the last slash
124+
fromtag=${CEPH_CONTAINER_FROM_IMAGE##*/}
125+
# translate : to -
126+
fromtag=${fromtag/:/-}
127+
builddate=$(date +%Y%m%d)
128+
local_tag=${fromtag}-${CEPH_CONTAINER_CEPH_REF}-${CEPH_CONTAINER_ARCH}-${builddate}
129+
130+
repopath=${CONTAINER_REPO_HOSTNAME}/${CONTAINER_REPO_ORGANIZATION}
131+
132+
if [[ ${CI_CONTAINER} == "true" ]] ; then
133+
# ceph-ci conventions for remote tags:
134+
# requires ARCH, BRANCH, CEPH_SHA1, FLAVOR
135+
full_repo_tag=$repopath/ceph:${BRANCH}-${fromtag}-${ARCH}-devel
136+
branch_repo_tag=$repopath/ceph:${BRANCH}
137+
sha1_repo_tag=$repopath/ceph:${CEPH_SHA1}
138+
139+
if [[ "${ARCH}" == "aarch64" ]] ; then
140+
branch_repo_tag=${branch_repo_tag}-aarch64
141+
sha1_repo_tag=${sha1_repo_tag}-aarch64
142+
fi
143+
144+
podman tag ${image_id} ${full_repo_tag}
145+
podman tag ${image_id} ${branch_repo_tag}
146+
podman tag ${image_id} ${sha1_repo_tag}
147+
148+
if [[ ${FLAVOR} == "crimson" && ${ARCH} == "x86_64" ]] ; then
149+
sha1_flavor_repo_tag=${sha1_repo_tag}-${FLAVOR}
150+
podman tag ${image_id} ${sha1_flavor_repo_tag}
151+
if [[ -z "${NO_PUSH}" ]] ; then
152+
podman push ${sha1_flavor_repo_tag}
153+
fi
154+
exit
155+
fi
156+
157+
if [[ -z "${NO_PUSH}" ]] ; then
158+
podman push ${full_repo_tag}
159+
podman push ${branch_repo_tag}
160+
podman push ${sha1_repo_tag}
161+
fi
162+
else
163+
#
164+
# non-CI build. Tags are like v19.1.0-20240701
165+
# push to quay.ceph.io/ceph/prerelease
166+
#
167+
version_tag=${repopath}/prerelease/ceph-${ARCH}:${VERSION}-${builddate}
168+
169+
podman tag ${image_id} ${version_tag}
170+
if [[ -z "${NO_PUSH}" ]] ; then
171+
podman push ${image_id} ${version_tag}
172+
fi
173+
fi
174+
175+

0 commit comments

Comments
 (0)