Skip to content

Commit 53152af

Browse files
nh2nh2
authored
ceph: Fix missing patch for Ceph with dmcrypt (#335684)
2 parents 4cae00a + 0cd631e commit 53152af

File tree

3 files changed

+278
-1
lines changed

3 files changed

+278
-1
lines changed

‎nixos/tests/all-tests.nix‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -188,6 +188,7 @@ in {
188188
ceph-multi-node = handleTestOn [ "aarch64-linux" "x86_64-linux" ] ./ceph-multi-node.nix {};
189189
ceph-single-node = handleTestOn [ "aarch64-linux" "x86_64-linux" ] ./ceph-single-node.nix {};
190190
ceph-single-node-bluestore = handleTestOn [ "aarch64-linux" "x86_64-linux" ] ./ceph-single-node-bluestore.nix {};
191+
ceph-single-node-bluestore-dmcrypt = handleTestOn [ "aarch64-linux" "x86_64-linux" ] ./ceph-single-node-bluestore-dmcrypt.nix {};
191192
certmgr = handleTest ./certmgr.nix {};
192193
cfssl = handleTestOn ["aarch64-linux" "x86_64-linux"] ./cfssl.nix {};
193194
cgit = handleTest ./cgit.nix {};

‎nixos/tests/ceph-single-node-bluestore-dmcrypt.nix‎

Lines changed: 270 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,270 @@
1+
import ./make-test-python.nix (
2+
{ pkgs, lib, ... }:
3+
4+
let
5+
# the single node ipv6 address
6+
ip = "2001:db8:ffff::";
7+
# the global ceph cluster id
8+
cluster = "54465b37-b9d8-4539-a1f9-dd33c75ee45a";
9+
# the fsids of OSDs
10+
osd-fsid-map = {
11+
"0" = "1c1b7ea9-06bf-4d30-9a01-37ac3a0254aa";
12+
"1" = "bd5a6f49-69d5-428c-ac25-a99f0c44375c";
13+
"2" = "c90de6c7-86c6-41da-9694-e794096dfc5c";
14+
};
15+
16+
in
17+
{
18+
name = "basic-single-node-ceph-cluster-bluestore-dmcrypt";
19+
meta = with pkgs.lib.maintainers; {
20+
maintainers = [ benaryorg nh2 ];
21+
};
22+
23+
nodes = {
24+
ceph =
25+
{ pkgs, config, ... }:
26+
{
27+
# disks for bluestore
28+
virtualisation.emptyDiskImages = [
29+
20480
30+
20480
31+
20480
32+
];
33+
34+
# networking setup (no external connectivity required, only local IPv6)
35+
networking.useDHCP = false;
36+
systemd.network = {
37+
enable = true;
38+
wait-online.extraArgs = [
39+
"-i"
40+
"lo"
41+
];
42+
networks = {
43+
"40-loopback" = {
44+
enable = true;
45+
name = "lo";
46+
DHCP = "no";
47+
addresses = [ { Address = "${ip}/128"; } ];
48+
};
49+
};
50+
};
51+
52+
# do not start the ceph target by default so we can format the disks first
53+
systemd.targets.ceph.wantedBy = lib.mkForce [ ];
54+
55+
# add the packages to systemPackages so the testscript doesn't run into any unexpected issues
56+
# this shouldn't be required on production systems which have their required packages in the unit paths only
57+
# but it helps in case one needs to actually run the tooling anyway
58+
environment.systemPackages = with pkgs; [
59+
ceph
60+
cryptsetup
61+
lvm2
62+
];
63+
64+
services.ceph = {
65+
enable = true;
66+
client.enable = true;
67+
extraConfig = {
68+
public_addr = ip;
69+
cluster_addr = ip;
70+
# ipv6
71+
ms_bind_ipv4 = "false";
72+
ms_bind_ipv6 = "true";
73+
# msgr2 settings
74+
ms_cluster_mode = "secure";
75+
ms_service_mode = "secure";
76+
ms_client_mode = "secure";
77+
ms_mon_cluster_mode = "secure";
78+
ms_mon_service_mode = "secure";
79+
ms_mon_client_mode = "secure";
80+
# less default modules, cuts down on memory and startup time in the tests
81+
mgr_initial_modules = "";
82+
# distribute by OSD, not by host, as per https://docs.ceph.com/en/reef/cephadm/install/#single-host
83+
osd_crush_chooseleaf_type = "0";
84+
};
85+
client.extraConfig."mon.0" = {
86+
host = "ceph";
87+
mon_addr = "v2:[${ip}]:3300";
88+
public_addr = "v2:[${ip}]:3300";
89+
};
90+
global = {
91+
fsid = cluster;
92+
clusterNetwork = "${ip}/64";
93+
publicNetwork = "${ip}/64";
94+
monInitialMembers = "0";
95+
};
96+
97+
mon = {
98+
enable = true;
99+
daemons = [ "0" ];
100+
};
101+
102+
osd = {
103+
enable = true;
104+
daemons = builtins.attrNames osd-fsid-map;
105+
};
106+
107+
mgr = {
108+
enable = true;
109+
daemons = [ "ceph" ];
110+
};
111+
};
112+
113+
systemd.services =
114+
let
115+
osd-name = id: "ceph-osd-${id}";
116+
osd-pre-start = id: [
117+
"!${config.services.ceph.osd.package.out}/bin/ceph-volume lvm activate --bluestore ${id} ${osd-fsid-map.${id}} --no-systemd"
118+
"${config.services.ceph.osd.package.lib}/libexec/ceph/ceph-osd-prestart.sh --id ${id} --cluster ${config.services.ceph.global.clusterName}"
119+
];
120+
osd-post-stop = id: [
121+
"!${config.services.ceph.osd.package.out}/bin/ceph-volume lvm deactivate ${id}"
122+
];
123+
map-osd = id: {
124+
name = osd-name id;
125+
value = {
126+
serviceConfig.ExecStartPre = lib.mkForce (osd-pre-start id);
127+
serviceConfig.ExecStopPost = osd-post-stop id;
128+
unitConfig.ConditionPathExists = lib.mkForce [ ];
129+
unitConfig.StartLimitBurst = lib.mkForce 4;
130+
path = with pkgs; [
131+
util-linux
132+
lvm2
133+
cryptsetup
134+
];
135+
};
136+
};
137+
in
138+
lib.pipe config.services.ceph.osd.daemons [
139+
(builtins.map map-osd)
140+
builtins.listToAttrs
141+
];
142+
};
143+
};
144+
145+
testScript =
146+
{ ... }:
147+
''
148+
start_all()
149+
150+
ceph.wait_for_unit("default.target")
151+
152+
# Bootstrap ceph-mon daemon
153+
ceph.succeed(
154+
"mkdir -p /var/lib/ceph/bootstrap-osd",
155+
"ceph-authtool --create-keyring /tmp/ceph.mon.keyring --gen-key -n mon. --cap mon 'allow *'",
156+
"ceph-authtool --create-keyring /etc/ceph/ceph.client.admin.keyring --gen-key -n client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *'",
157+
"ceph-authtool --create-keyring /var/lib/ceph/bootstrap-osd/ceph.keyring --gen-key -n client.bootstrap-osd --cap mon 'profile bootstrap-osd' --cap mgr 'allow r'",
158+
"ceph-authtool /tmp/ceph.mon.keyring --import-keyring /etc/ceph/ceph.client.admin.keyring",
159+
"ceph-authtool /tmp/ceph.mon.keyring --import-keyring /var/lib/ceph/bootstrap-osd/ceph.keyring",
160+
"monmaptool --create --fsid ${cluster} --addv 0 'v2:[${ip}]:3300/0' --clobber /tmp/ceph.initial-monmap",
161+
"mkdir -p /var/lib/ceph/mon/ceph-0",
162+
"ceph-mon --mkfs -i 0 --monmap /tmp/ceph.initial-monmap --keyring /tmp/ceph.mon.keyring",
163+
"chown ceph:ceph -R /tmp/ceph.mon.keyring /var/lib/ceph",
164+
"systemctl start ceph-mon-0.service",
165+
)
166+
167+
ceph.wait_for_unit("ceph-mon-0.service")
168+
# should the mon not start or bind for some reason this gives us a better error message than the config commands running into a timeout
169+
ceph.wait_for_open_port(3300, "${ip}")
170+
ceph.succeed(
171+
# required for HEALTH_OK
172+
"ceph config set mon auth_allow_insecure_global_id_reclaim false",
173+
# IPv6
174+
"ceph config set global ms_bind_ipv4 false",
175+
"ceph config set global ms_bind_ipv6 true",
176+
# the new (secure) protocol
177+
"ceph config set global ms_bind_msgr1 false",
178+
"ceph config set global ms_bind_msgr2 true",
179+
# just a small little thing
180+
"ceph config set mon mon_compact_on_start true",
181+
)
182+
183+
# Can't check ceph status until a mon is up
184+
ceph.succeed("ceph -s | grep 'mon: 1 daemons'")
185+
186+
# Bootstrap OSDs (do this before starting the mgr because cryptsetup and the mgr both eat a lot of memory)
187+
ceph.succeed(
188+
# this will automatically do what's required for LVM, cryptsetup, and stores all the data in Ceph's internal databases
189+
"ceph-volume lvm prepare --bluestore --data /dev/vdb --dmcrypt --no-systemd --osd-id 0 --osd-fsid ${osd-fsid-map."0"}",
190+
"ceph-volume lvm prepare --bluestore --data /dev/vdc --dmcrypt --no-systemd --osd-id 1 --osd-fsid ${osd-fsid-map."1"}",
191+
"ceph-volume lvm prepare --bluestore --data /dev/vdd --dmcrypt --no-systemd --osd-id 2 --osd-fsid ${osd-fsid-map."2"}",
192+
"sudo ceph-volume lvm deactivate 0",
193+
"sudo ceph-volume lvm deactivate 1",
194+
"sudo ceph-volume lvm deactivate 2",
195+
"chown -R ceph:ceph /var/lib/ceph",
196+
)
197+
198+
# Start OSDs (again, argon2id eats memory, so this happens before starting the mgr)
199+
ceph.succeed(
200+
"systemctl start ceph-osd-0.service",
201+
"systemctl start ceph-osd-1.service",
202+
"systemctl start ceph-osd-2.service",
203+
)
204+
ceph.wait_until_succeeds("ceph -s | grep 'quorum 0'")
205+
ceph.wait_until_succeeds("ceph osd stat | grep -e '3 osds: 3 up[^,]*, 3 in'")
206+
207+
# Start the ceph-mgr daemon, after copying in the keyring
208+
ceph.succeed(
209+
"mkdir -p /var/lib/ceph/mgr/ceph-ceph/",
210+
"ceph auth get-or-create -o /var/lib/ceph/mgr/ceph-ceph/keyring mgr.ceph mon 'allow profile mgr' osd 'allow *' mds 'allow *'",
211+
"chown -R ceph:ceph /var/lib/ceph/mgr/ceph-ceph/",
212+
"systemctl start ceph-mgr-ceph.service",
213+
)
214+
ceph.wait_for_unit("ceph-mgr-ceph")
215+
ceph.wait_until_succeeds("ceph -s | grep 'quorum 0'")
216+
ceph.wait_until_succeeds("ceph -s | grep 'mgr: ceph(active,'")
217+
ceph.wait_until_succeeds("ceph osd stat | grep -e '3 osds: 3 up[^,]*, 3 in'")
218+
ceph.wait_until_succeeds("ceph -s | grep 'HEALTH_OK'")
219+
220+
# test the actual storage
221+
ceph.succeed(
222+
"ceph osd pool create single-node-test 32 32",
223+
"ceph osd pool ls | grep 'single-node-test'",
224+
225+
# We need to enable an application on the pool, otherwise it will
226+
# stay unhealthy in state POOL_APP_NOT_ENABLED.
227+
# Creating a CephFS would do this automatically, but we haven't done that here.
228+
# See: https://docs.ceph.com/en/reef/rados/operations/pools/#associating-a-pool-with-an-application
229+
# We use the custom application name "nixos-test" for this.
230+
"ceph osd pool application enable single-node-test nixos-test",
231+
232+
"ceph osd pool rename single-node-test single-node-other-test",
233+
"ceph osd pool ls | grep 'single-node-other-test'",
234+
)
235+
ceph.wait_until_succeeds("ceph -s | grep '2 pools, 33 pgs'")
236+
ceph.wait_until_succeeds("ceph -s | grep 'HEALTH_OK'")
237+
ceph.wait_until_succeeds("ceph -s | grep '33 active+clean'")
238+
ceph.fail(
239+
# the old pool should be gone
240+
"ceph osd pool ls | grep 'multi-node-test'",
241+
# deleting the pool should fail without setting mon_allow_pool_delete
242+
"ceph osd pool delete single-node-other-test single-node-other-test --yes-i-really-really-mean-it",
243+
)
244+
245+
# rebooting gets rid of any potential tmpfs mounts or device-mapper devices
246+
ceph.shutdown()
247+
ceph.start()
248+
ceph.wait_for_unit("default.target")
249+
250+
# Start it up (again OSDs first due to memory constraints of cryptsetup and mgr)
251+
ceph.systemctl("start ceph-mon-0.service")
252+
ceph.wait_for_unit("ceph-mon-0")
253+
ceph.systemctl("start ceph-osd-0.service")
254+
ceph.wait_for_unit("ceph-osd-0")
255+
ceph.systemctl("start ceph-osd-1.service")
256+
ceph.wait_for_unit("ceph-osd-1")
257+
ceph.systemctl("start ceph-osd-2.service")
258+
ceph.wait_for_unit("ceph-osd-2")
259+
ceph.systemctl("start ceph-mgr-ceph.service")
260+
ceph.wait_for_unit("ceph-mgr-ceph")
261+
262+
# Ensure the cluster comes back up again
263+
ceph.succeed("ceph -s | grep 'mon: 1 daemons'")
264+
ceph.wait_until_succeeds("ceph -s | grep 'quorum 0'")
265+
ceph.wait_until_succeeds("ceph osd stat | grep -E '3 osds: 3 up[^,]*, 3 in'")
266+
ceph.wait_until_succeeds("ceph -s | grep 'mgr: ceph(active,'")
267+
ceph.wait_until_succeeds("ceph -s | grep 'HEALTH_OK'")
268+
'';
269+
}
270+
)

‎pkgs/tools/filesystems/ceph/default.nix‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -323,6 +323,11 @@ in rec {
323323
name = "ceph-reef-ceph-volume-fix-set_dmcrypt_no_workqueue.patch";
324324
hash = "sha256-r+7hcCz2WF/rJfgKwTatKY9unJlE8Uw3fmOyaY5jVH0=";
325325
})
326+
(fetchpatch {
327+
url = "https://github.com/ceph/ceph/commit/607eb34b2c278566c386efcbf3018629cf08ccfd.patch";
328+
name = "ceph-volume-fix-set_dmcrypt_no_workqueue-regex.patch";
329+
hash = "sha256-q28Q7OIyFoMyMBCPXGA+AdNqp+9/6J/XwD4ODjx+JXY=";
330+
})
326331
];
327332

328333
postPatch = ''
@@ -487,7 +492,8 @@ in rec {
487492
inherit (nixosTests)
488493
ceph-multi-node
489494
ceph-single-node
490-
ceph-single-node-bluestore;
495+
ceph-single-node-bluestore
496+
ceph-single-node-bluestore-dmcrypt;
491497
};
492498
};
493499
};

0 commit comments

Comments
 (0)