Skip to content

Commit 2de3cd5

Browse files
tonyhuttertonyhutter
committed
Add Github runner to build zfs-release RPM
Add a runner to build the zfs-release RPM and then package it up in artifacts. This re-uses a stripped down version of the QEMU runners from ZFS: openzfs/zfs@bca9b64 Signed-off-by: Tony Hutter <[email protected]>
1 parent ae83f1e commit 2de3cd5

File tree

5 files changed

+422
-0
lines changed

5 files changed

+422
-0
lines changed

‎.github/workflows/scripts/qemu-1-setup.sh‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
#!/usr/bin/env bash
2+
3+
######################################################################
4+
# 1) setup qemu instance on action runner
5+
######################################################################
6+
7+
set -eu
8+
9+
# install needed packages
10+
export DEBIAN_FRONTEND="noninteractive"
11+
sudo apt-get -y update
12+
sudo apt-get install -y axel cloud-image-utils daemonize guestfs-tools \
13+
ksmtuned virt-manager linux-modules-extra-$(uname -r) zfsutils-linux
14+
15+
# generate ssh keys
16+
rm -f ~/.ssh/id_ed25519
17+
ssh-keygen -t ed25519 -f ~/.ssh/id_ed25519 -q -N ""
18+
19+
# we expect RAM shortage
20+
cat << EOF | sudo tee /etc/ksmtuned.conf > /dev/null
21+
# /etc/ksmtuned.conf - Configuration file for ksmtuned
22+
# https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/virtualization_tuning_and_optimization_guide/chap-ksm
23+
KSM_MONITOR_INTERVAL=60
24+
25+
# Millisecond sleep between ksm scans for 16Gb server.
26+
# Smaller servers sleep more, bigger sleep less.
27+
KSM_SLEEP_MSEC=30
28+
29+
KSM_NPAGES_BOOST=0
30+
KSM_NPAGES_DECAY=0
31+
KSM_NPAGES_MIN=1000
32+
KSM_NPAGES_MAX=25000
33+
34+
KSM_THRES_COEF=80
35+
KSM_THRES_CONST=8192
36+
37+
LOGFILE=/var/log/ksmtuned.log
38+
DEBUG=1
39+
EOF
40+
sudo systemctl restart ksm
41+
sudo systemctl restart ksmtuned
42+
43+
# not needed
44+
sudo systemctl stop docker.socket
45+
sudo systemctl stop multipathd.socket
46+
47+
# remove default swapfile and /mnt
48+
sudo swapoff -a
49+
sudo umount -l /mnt
50+
DISK="/dev/disk/cloud/azure_resource-part1"
51+
sudo sed -e "s|^$DISK.*||g" -i /etc/fstab
52+
sudo wipefs -aq $DISK
53+
sudo systemctl daemon-reload
54+
55+
sudo modprobe loop
56+
sudo modprobe zfs
57+
58+
# partition the disk as needed
59+
DISK="/dev/disk/cloud/azure_resource"
60+
sudo sgdisk --zap-all $DISK
61+
sudo sgdisk -p \
62+
-n 1:0:+16G -c 1:"swap" \
63+
-n 2:0:0 -c 2:"tests" \
64+
$DISK
65+
sync
66+
sleep 1
67+
68+
# swap with same size as RAM
69+
sudo mkswap $DISK-part1
70+
sudo swapon $DISK-part1
71+
72+
# 60GB data disk
73+
SSD1="$DISK-part2"
74+
75+
# 10GB data disk on ext4
76+
sudo fallocate -l 10G /test.ssd1
77+
SSD2=$(sudo losetup -b 4096 -f /test.ssd1 --show)
78+
79+
# adjust zfs module parameter and create pool
80+
exec 1>/dev/null
81+
ARC_MIN=$((1024*1024*256))
82+
ARC_MAX=$((1024*1024*512))
83+
echo $ARC_MIN | sudo tee /sys/module/zfs/parameters/zfs_arc_min
84+
echo $ARC_MAX | sudo tee /sys/module/zfs/parameters/zfs_arc_max
85+
echo 1 | sudo tee /sys/module/zfs/parameters/zvol_use_blk_mq
86+
sudo zpool create -f -o ashift=12 zpool $SSD1 $SSD2 \
87+
-O relatime=off -O atime=off -O xattr=sa -O compression=lz4 \
88+
-O mountpoint=/mnt/tests
89+
90+
# no need for some scheduler
91+
for i in /sys/block/s*/queue/scheduler; do
92+
echo "none" | sudo tee $i > /dev/null
93+
done

‎.github/workflows/scripts/qemu-2-start.sh‎

Lines changed: 225 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,225 @@
1+
#!/usr/bin/env bash
2+
3+
######################################################################
4+
# 2) start qemu with some operating system, init via cloud-init
5+
######################################################################
6+
7+
set -eu
8+
9+
# short name used in zfs-qemu.yml
10+
OS="$1"
11+
12+
# OS variant (virt-install --os-variant list)
13+
OSv=$OS
14+
15+
# compressed with .zst extension
16+
REPO="https://github.com/mcmilk/openzfs-freebsd-images"
17+
FREEBSD="$REPO/releases/download/v2024-10-05"
18+
URLzs=""
19+
20+
# Ubuntu mirrors
21+
#UBMIRROR="https://cloud-images.ubuntu.com"
22+
#UBMIRROR="https://mirrors.cloud.tencent.com/ubuntu-cloud-images"
23+
UBMIRROR="https://mirror.citrahost.com/ubuntu-cloud-images"
24+
25+
# default nic model for vm's
26+
NIC="virtio"
27+
28+
case "$OS" in
29+
almalinux8)
30+
OSNAME="AlmaLinux 8"
31+
URL="https://repo.almalinux.org/almalinux/8/cloud/x86_64/images/AlmaLinux-8-GenericCloud-latest.x86_64.qcow2"
32+
;;
33+
almalinux9)
34+
OSNAME="AlmaLinux 9"
35+
URL="https://repo.almalinux.org/almalinux/9/cloud/x86_64/images/AlmaLinux-9-GenericCloud-latest.x86_64.qcow2"
36+
;;
37+
archlinux)
38+
OSNAME="Archlinux"
39+
URL="https://geo.mirror.pkgbuild.com/images/latest/Arch-Linux-x86_64-cloudimg.qcow2"
40+
# dns sometimes fails with that url :/
41+
echo "89.187.191.12 geo.mirror.pkgbuild.com" | sudo tee /etc/hosts > /dev/null
42+
;;
43+
centos-stream9)
44+
OSNAME="CentOS Stream 9"
45+
URL="https://cloud.centos.org/centos/9-stream/x86_64/images/CentOS-Stream-GenericCloud-9-latest.x86_64.qcow2"
46+
;;
47+
debian11)
48+
OSNAME="Debian 11"
49+
URL="https://cloud.debian.org/images/cloud/bullseye/latest/debian-11-generic-amd64.qcow2"
50+
;;
51+
debian12)
52+
OSNAME="Debian 12"
53+
URL="https://cloud.debian.org/images/cloud/bookworm/latest/debian-12-generic-amd64.qcow2"
54+
;;
55+
fedora40)
56+
OSNAME="Fedora 40"
57+
OSv="fedora-unknown"
58+
URL="https://download.fedoraproject.org/pub/fedora/linux/releases/40/Cloud/x86_64/images/Fedora-Cloud-Base-Generic.x86_64-40-1.14.qcow2"
59+
;;
60+
fedora41)
61+
OSNAME="Fedora 41"
62+
OSv="fedora-unknown"
63+
URL="https://download.fedoraproject.org/pub/fedora/linux/releases/41/Cloud/x86_64/images/Fedora-Cloud-Base-Generic-41-1.4.x86_64.qcow2"
64+
;;
65+
freebsd13-3r)
66+
OSNAME="FreeBSD 13.3-RELEASE"
67+
OSv="freebsd13.0"
68+
URLzs="$FREEBSD/amd64-freebsd-13.3-RELEASE.qcow2.zst"
69+
BASH="/usr/local/bin/bash"
70+
NIC="rtl8139"
71+
;;
72+
freebsd13-4r)
73+
OSNAME="FreeBSD 13.4-RELEASE"
74+
OSv="freebsd13.0"
75+
URLzs="$FREEBSD/amd64-freebsd-13.4-RELEASE.qcow2.zst"
76+
BASH="/usr/local/bin/bash"
77+
NIC="rtl8139"
78+
;;
79+
freebsd14-0r)
80+
OSNAME="FreeBSD 14.0-RELEASE"
81+
OSv="freebsd14.0"
82+
URLzs="$FREEBSD/amd64-freebsd-14.0-RELEASE.qcow2.zst"
83+
BASH="/usr/local/bin/bash"
84+
;;
85+
freebsd14-1r)
86+
OSNAME="FreeBSD 14.1-RELEASE"
87+
OSv="freebsd14.0"
88+
URLzs="$FREEBSD/amd64-freebsd-14.1-RELEASE.qcow2.zst"
89+
BASH="/usr/local/bin/bash"
90+
;;
91+
freebsd13-4s)
92+
OSNAME="FreeBSD 13.4-STABLE"
93+
OSv="freebsd13.0"
94+
URLzs="$FREEBSD/amd64-freebsd-13.4-STABLE.qcow2.zst"
95+
BASH="/usr/local/bin/bash"
96+
;;
97+
freebsd14-1s)
98+
OSNAME="FreeBSD 14.1-STABLE"
99+
OSv="freebsd14.0"
100+
URLzs="$FREEBSD/amd64-freebsd-14.1-STABLE.qcow2.zst"
101+
BASH="/usr/local/bin/bash"
102+
;;
103+
freebsd15-0c)
104+
OSNAME="FreeBSD 15.0-CURRENT"
105+
OSv="freebsd14.0"
106+
URLzs="$FREEBSD/amd64-freebsd-15.0-CURRENT.qcow2.zst"
107+
BASH="/usr/local/bin/bash"
108+
;;
109+
tumbleweed)
110+
OSNAME="openSUSE Tumbleweed"
111+
OSv="opensusetumbleweed"
112+
MIRROR="http://opensuse-mirror-gce-us.susecloud.net"
113+
URL="$MIRROR/tumbleweed/appliances/openSUSE-MicroOS.x86_64-OpenStack-Cloud.qcow2"
114+
;;
115+
ubuntu20)
116+
OSNAME="Ubuntu 20.04"
117+
OSv="ubuntu20.04"
118+
URL="$UBMIRROR/focal/current/focal-server-cloudimg-amd64.img"
119+
;;
120+
ubuntu22)
121+
OSNAME="Ubuntu 22.04"
122+
OSv="ubuntu22.04"
123+
URL="$UBMIRROR/jammy/current/jammy-server-cloudimg-amd64.img"
124+
;;
125+
ubuntu24)
126+
OSNAME="Ubuntu 24.04"
127+
OSv="ubuntu24.04"
128+
URL="$UBMIRROR/noble/current/noble-server-cloudimg-amd64.img"
129+
;;
130+
*)
131+
echo "Wrong value for OS variable!"
132+
exit 111
133+
;;
134+
esac
135+
136+
# environment file
137+
ENV="/var/tmp/env.txt"
138+
echo "ENV=$ENV" >> $ENV
139+
140+
# result path
141+
echo 'RESPATH="/var/tmp/test_results"' >> $ENV
142+
143+
# FreeBSD 13 has problems with: e1000+virtio
144+
echo "NIC=$NIC" >> $ENV
145+
146+
# freebsd15 -> used in zfs-qemu.yml
147+
echo "OS=$OS" >> $ENV
148+
149+
# freebsd14.0 -> used for virt-install
150+
echo "OSv=\"$OSv\"" >> $ENV
151+
152+
# FreeBSD 15 (Current) -> used for summary
153+
echo "OSNAME=\"$OSNAME\"" >> $ENV
154+
155+
sudo mkdir -p "/mnt/tests"
156+
sudo chown -R $(whoami) /mnt/tests
157+
158+
# we are downloading via axel, curl and wget are mostly slower and
159+
# require more return value checking
160+
IMG="/mnt/tests/cloudimg.qcow2"
161+
if [ ! -z "$URLzs" ]; then
162+
echo "Loading image $URLzs ..."
163+
time axel -q -o "$IMG.zst" "$URLzs"
164+
zstd -q -d --rm "$IMG.zst"
165+
else
166+
echo "Loading image $URL ..."
167+
time axel -q -o "$IMG" "$URL"
168+
fi
169+
170+
DISK="/dev/zvol/zpool/openzfs"
171+
FORMAT="raw"
172+
sudo zfs create -ps -b 64k -V 80g zpool/openzfs
173+
while true; do test -b $DISK && break; sleep 1; done
174+
echo "Importing VM image to zvol..."
175+
sudo qemu-img dd -f qcow2 -O raw if=$IMG of=$DISK bs=4M
176+
rm -f $IMG
177+
178+
PUBKEY=$(cat ~/.ssh/id_ed25519.pub)
179+
cat <<EOF > /tmp/user-data
180+
#cloud-config
181+
182+
fqdn: $OS
183+
184+
users:
185+
- name: root
186+
shell: $BASH
187+
- name: zfs
188+
sudo: ALL=(ALL) NOPASSWD:ALL
189+
shell: $BASH
190+
ssh_authorized_keys:
191+
- $PUBKEY
192+
193+
growpart:
194+
mode: auto
195+
devices: ['/']
196+
ignore_growroot_disabled: false
197+
EOF
198+
199+
sudo virsh net-update default add ip-dhcp-host \
200+
"<host mac='52:54:00:83:79:00' ip='192.168.122.10'/>" --live --config
201+
202+
sudo virt-install \
203+
--os-variant $OSv \
204+
--name "openzfs" \
205+
--cpu host-passthrough \
206+
--virt-type=kvm --hvm \
207+
--vcpus=4,sockets=1 \
208+
--memory $((1024*12)) \
209+
--memballoon model=virtio \
210+
--graphics none \
211+
--network bridge=virbr0,model=$NIC,mac='52:54:00:83:79:00' \
212+
--cloud-init user-data=/tmp/user-data \
213+
--disk $DISK,bus=virtio,cache=none,format=$FORMAT,driver.discard=unmap \
214+
--import --noautoconsole >/dev/null
215+
216+
# in case the directory isn't there already
217+
mkdir -p $HOME/.ssh
218+
219+
cat <<EOF >> $HOME/.ssh/config
220+
# no questions please
221+
StrictHostKeyChecking no
222+
223+
# small timeout, used in while loops later
224+
ConnectTimeout 1
225+
EOF

‎.github/workflows/scripts/qemu-3-deps.sh‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
#!/usr/bin/env bash
2+
3+
######################################################################
4+
# 3) install dependencies for compiling and loading
5+
######################################################################
6+
7+
set -eu
8+
9+
sudo dnf install rpm-tools
10+
11+
exit 0

‎.github/workflows/scripts/qemu-4-build.sh‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
#!/usr/bin/env bash
2+
3+
######################################################################
4+
# 4) configure and build openzfs modules
5+
######################################################################
6+
7+
set -eu
8+
9+
cd zfsonlinux.github.com/zfs-release
10+
sudo yum -y install rpm-build
11+
mkdir -p ~/rpmbuild/{BUILDROOT,SPECS,RPMS,SRPMS,SOURCES,BUILD}
12+
cp RPM-GPG-KEY-openzfs* *.repo ~/rpmbuild/SOURCES
13+
cp zfs-release.spec ~/rpmbuild/SPECS/
14+
rpmbuild -ba ~/rpmbuild/SPECS/zfs-release.spec
15+
16+
# building the zfs module was ok
17+
echo 0 > /var/tmp/build-exitcode.txt
18+
19+
# reset cloud-init configuration and poweroff
20+
sudo cloud-init clean --logs
21+
exit 0

0 commit comments

Comments
 (0)