Skip to content

Commit d5a061e

Browse files
idryomovidryomov
authored
Merge pull request ceph#44366 from orozery/rbd-crypto-migration
librbd/crypto: fix issue when live-migrating from encrypted export Reviewed-by: Ramana Raja <[email protected]>
2 parents 0e06d65 + 0000c34 commit d5a061e

File tree

5 files changed

+315
-18
lines changed

5 files changed

+315
-18
lines changed

‎qa/workunits/rbd/luks-encryption.sh‎

Lines changed: 197 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
set -ex
33

44
CEPH_ID=${CEPH_ID:-admin}
5-
TMP_FILES="/tmp/passphrase /tmp/passphrase2 /tmp/testdata1 /tmp/testdata2 /tmp/cmpdata"
5+
TMP_FILES="/tmp/passphrase /tmp/passphrase2 /tmp/testdata1 /tmp/testdata2 /tmp/cmpdata /tmp/rawexport /tmp/export.qcow2"
66

77
_sudo()
88
{
@@ -32,19 +32,16 @@ function expect_false() {
3232

3333
function test_encryption_format() {
3434
local format=$1
35-
clean_up_cryptsetup
3635

3736
# format
3837
rbd encryption format testimg $format /tmp/passphrase
3938
drop_caches
4039

4140
# open encryption with cryptsetup
4241
sudo cryptsetup open $RAW_DEV --type luks cryptsetupdev -d /tmp/passphrase
43-
sudo chmod 666 /dev/mapper/cryptsetupdev
4442

4543
# open encryption with librbd
4644
LIBRBD_DEV=$(_sudo rbd -p rbd map testimg -t nbd -o encryption-passphrase-file=/tmp/passphrase)
47-
sudo chmod 666 $LIBRBD_DEV
4845

4946
# write via librbd && compare
5047
dd if=/tmp/testdata1 of=$LIBRBD_DEV conv=fsync bs=1M
@@ -68,11 +65,10 @@ function test_encryption_format() {
6865
(( $(sudo blockdev --getsize64 $LIBRBD_DEV) == (32 << 20) ))
6966

7067
_sudo rbd device unmap -t nbd $LIBRBD_DEV
68+
sudo cryptsetup close cryptsetupdev
7169
}
7270

7371
function test_clone_encryption() {
74-
clean_up_cryptsetup
75-
7672
# write 1MB plaintext
7773
dd if=/tmp/testdata1 of=$RAW_DEV conv=fsync bs=1M count=1
7874

@@ -84,7 +80,6 @@ function test_clone_encryption() {
8480

8581
# open encryption with librbd, write one more MB, close
8682
LIBRBD_DEV=$(_sudo rbd -p rbd map testimg1 -t nbd -o encryption-format=luks1,encryption-passphrase-file=/tmp/passphrase)
87-
sudo chmod 666 $LIBRBD_DEV
8883
dd if=$LIBRBD_DEV of=/tmp/cmpdata bs=1M count=1
8984
cmp -n 1MB /tmp/cmpdata /tmp/testdata1
9085
dd if=/tmp/testdata1 of=$LIBRBD_DEV seek=1 skip=1 conv=fsync bs=1M count=1
@@ -98,7 +93,6 @@ function test_clone_encryption() {
9893

9994
# open encryption with librbd, write one more MB, close
10095
LIBRBD_DEV=$(_sudo rbd -p rbd map testimg2 -t nbd -o encryption-format=luks2,encryption-passphrase-file=/tmp/passphrase2,encryption-format=luks1,encryption-passphrase-file=/tmp/passphrase)
101-
sudo chmod 666 $LIBRBD_DEV
10296
dd if=$LIBRBD_DEV of=/tmp/cmpdata bs=1M count=2
10397
cmp -n 2MB /tmp/cmpdata /tmp/testdata1
10498
dd if=/tmp/testdata1 of=$LIBRBD_DEV seek=2 skip=2 conv=fsync bs=1M count=1
@@ -111,10 +105,17 @@ function test_clone_encryption() {
111105
# verify with cryptsetup
112106
RAW_FLAT_DEV=$(_sudo rbd -p rbd map testimg2 -t nbd)
113107
sudo cryptsetup open $RAW_FLAT_DEV --type luks cryptsetupdev -d /tmp/passphrase2
114-
sudo chmod 666 /dev/mapper/cryptsetupdev
115108
dd if=/dev/mapper/cryptsetupdev of=/tmp/cmpdata bs=1M count=3
116109
cmp -n 3MB /tmp/cmpdata /tmp/testdata1
110+
sudo cryptsetup close cryptsetupdev
117111
_sudo rbd device unmap -t nbd $RAW_FLAT_DEV
112+
113+
rbd rm testimg2
114+
rbd snap unprotect testimg1@snap
115+
rbd snap rm testimg1@snap
116+
rbd rm testimg1
117+
rbd snap unprotect testimg@snap
118+
rbd snap rm testimg@snap
118119
}
119120

120121
function test_clone_and_load_with_a_single_passphrase {
@@ -153,6 +154,172 @@ function test_plaintext_detection {
153154
test_clone_and_load_with_a_single_passphrase false
154155
}
155156

157+
function test_migration_read_and_copyup() {
158+
cp /tmp/testdata2 /tmp/cmpdata
159+
160+
# test reading
161+
LIBRBD_DEV=$(_sudo rbd -p rbd map testimg1 -t nbd -o encryption-passphrase-file=/tmp/passphrase)
162+
cmp $LIBRBD_DEV /tmp/cmpdata
163+
164+
# trigger copyup at the beginning and at the end
165+
xfs_io -c 'pwrite -S 0xab -W 0 4k' $LIBRBD_DEV /tmp/cmpdata
166+
xfs_io -c 'pwrite -S 0xba -W 4095k 4k' $LIBRBD_DEV /tmp/cmpdata
167+
168+
cmp $LIBRBD_DEV /tmp/cmpdata
169+
_sudo rbd device unmap -t nbd $LIBRBD_DEV
170+
171+
# test reading on a fresh mapping
172+
LIBRBD_DEV=$(_sudo rbd -p rbd map testimg1 -t nbd -o encryption-passphrase-file=/tmp/passphrase)
173+
cmp $LIBRBD_DEV /tmp/cmpdata
174+
_sudo rbd device unmap -t nbd $LIBRBD_DEV
175+
176+
# test reading on a fresh mapping after migration is executed
177+
rbd migration execute testimg1
178+
LIBRBD_DEV=$(_sudo rbd -p rbd map testimg1 -t nbd -o encryption-passphrase-file=/tmp/passphrase)
179+
cmp $LIBRBD_DEV /tmp/cmpdata
180+
_sudo rbd device unmap -t nbd $LIBRBD_DEV
181+
182+
# test reading on a fresh mapping after migration is committed
183+
rbd migration commit testimg1
184+
LIBRBD_DEV=$(_sudo rbd -p rbd map testimg1 -t nbd -o encryption-passphrase-file=/tmp/passphrase)
185+
cmp $LIBRBD_DEV /tmp/cmpdata
186+
_sudo rbd device unmap -t nbd $LIBRBD_DEV
187+
}
188+
189+
function test_migration_native_with_snaps() {
190+
LIBRBD_DEV=$(_sudo rbd -p rbd map testimg1@snap1 -t nbd -o encryption-passphrase-file=/tmp/passphrase)
191+
cmp $LIBRBD_DEV /tmp/testdata1
192+
_sudo rbd device unmap -t nbd $LIBRBD_DEV
193+
LIBRBD_DEV=$(_sudo rbd -p rbd map testimg1@snap2 -t nbd -o encryption-passphrase-file=/tmp/passphrase)
194+
cmp $LIBRBD_DEV /tmp/testdata2
195+
_sudo rbd device unmap -t nbd $LIBRBD_DEV
196+
197+
test_migration_read_and_copyup
198+
199+
# check that snapshots aren't affected by copyups
200+
LIBRBD_DEV=$(_sudo rbd -p rbd map testimg1@snap1 -t nbd -o encryption-passphrase-file=/tmp/passphrase)
201+
cmp $LIBRBD_DEV /tmp/testdata1
202+
_sudo rbd device unmap -t nbd $LIBRBD_DEV
203+
LIBRBD_DEV=$(_sudo rbd -p rbd map testimg1@snap2 -t nbd -o encryption-passphrase-file=/tmp/passphrase)
204+
cmp $LIBRBD_DEV /tmp/testdata2
205+
_sudo rbd device unmap -t nbd $LIBRBD_DEV
206+
207+
rbd snap rm testimg1@snap2
208+
rbd snap rm testimg1@snap1
209+
rbd rm testimg1
210+
}
211+
212+
function test_migration() {
213+
local format=$1
214+
215+
rbd encryption format testimg $format /tmp/passphrase
216+
217+
LIBRBD_DEV=$(_sudo rbd -p rbd map testimg -t nbd -o encryption-passphrase-file=/tmp/passphrase)
218+
dd if=/tmp/testdata1 of=$LIBRBD_DEV conv=fsync bs=1M
219+
rbd snap create testimg@snap1
220+
dd if=/tmp/testdata2 of=$LIBRBD_DEV conv=fsync bs=1M
221+
rbd snap create testimg@snap2
222+
# FIXME: https://tracker.ceph.com/issues/67401
223+
# leave HEAD with the same data as snap2 as a workaround
224+
# dd if=/tmp/testdata3 of=$LIBRBD_DEV conv=fsync bs=1M
225+
_sudo rbd device unmap -t nbd $LIBRBD_DEV
226+
227+
# live import a raw image
228+
rbd export testimg /tmp/rawexport
229+
rbd migration prepare --import-only --source-spec '{"type": "raw", "stream": {"type": "file", "file_path": "/tmp/rawexport"}}' testimg1
230+
test_migration_read_and_copyup
231+
rbd rm testimg1
232+
233+
# live import a qcow image
234+
qemu-img convert -f raw -O qcow2 /tmp/rawexport /tmp/export.qcow2
235+
rbd migration prepare --import-only --source-spec '{"type": "qcow", "stream": {"type": "file", "file_path": "/tmp/export.qcow2"}}' testimg1
236+
test_migration_read_and_copyup
237+
rbd rm testimg1
238+
239+
# live import a native image
240+
rbd migration prepare --import-only testimg@snap2 testimg1
241+
test_migration_native_with_snaps
242+
243+
# live migrate a native image (removes testimg)
244+
rbd migration prepare testimg testimg1
245+
test_migration_native_with_snaps
246+
247+
rm /tmp/rawexport /tmp/export.qcow2
248+
}
249+
250+
function test_migration_clone() {
251+
local format=$1
252+
253+
truncate -s 0 /tmp/cmpdata
254+
truncate -s 32M /tmp/cmpdata
255+
256+
rbd encryption format testimg $format /tmp/passphrase
257+
LIBRBD_DEV=$(_sudo rbd -p rbd map testimg -t nbd -o encryption-passphrase-file=/tmp/passphrase)
258+
xfs_io -c 'pwrite -S 0xaa -W 4M 1M' $LIBRBD_DEV /tmp/cmpdata
259+
xfs_io -c 'pwrite -S 0xaa -W 14M 1M' $LIBRBD_DEV /tmp/cmpdata
260+
xfs_io -c 'pwrite -S 0xaa -W 25M 1M' $LIBRBD_DEV /tmp/cmpdata
261+
_sudo rbd device unmap -t nbd $LIBRBD_DEV
262+
263+
rbd snap create testimg@snap
264+
rbd snap protect testimg@snap
265+
rbd clone testimg@snap testimg1
266+
267+
rbd encryption format testimg1 $format /tmp/passphrase2
268+
LIBRBD_DEV=$(_sudo rbd -p rbd map testimg1 -t nbd -o encryption-passphrase-file=/tmp/passphrase2,encryption-passphrase-file=/tmp/passphrase)
269+
xfs_io -c 'pwrite -S 0xbb -W 2M 1M' $LIBRBD_DEV /tmp/cmpdata
270+
xfs_io -c 'pwrite -S 0xbb -W 19M 1M' $LIBRBD_DEV /tmp/cmpdata
271+
xfs_io -c 'pwrite -S 0xbb -W 28M 1M' $LIBRBD_DEV /tmp/cmpdata
272+
_sudo rbd device unmap -t nbd $LIBRBD_DEV
273+
274+
# FIXME: https://tracker.ceph.com/issues/67402
275+
rbd config image set testimg1 rbd_sparse_read_threshold_bytes 1
276+
277+
# live migrate a native clone image (removes testimg1)
278+
rbd migration prepare testimg1 testimg2
279+
280+
# test reading
281+
# FIXME: https://tracker.ceph.com/issues/63184
282+
LIBRBD_DEV=$(_sudo rbd -p rbd map testimg2 -t nbd -o encryption-passphrase-file=/tmp/passphrase2,encryption-passphrase-file=/tmp/passphrase2,encryption-passphrase-file=/tmp/passphrase)
283+
cmp $LIBRBD_DEV /tmp/cmpdata
284+
285+
# trigger copyup for an unwritten area
286+
xfs_io -c 'pwrite -S 0xcc -W 24167k 4k' $LIBRBD_DEV /tmp/cmpdata
287+
288+
# trigger copyup for areas written in testimg (parent)
289+
xfs_io -c 'pwrite -S 0xcc -W 4245k 4k' $LIBRBD_DEV /tmp/cmpdata
290+
xfs_io -c 'pwrite -S 0xcc -W 13320k 4k' $LIBRBD_DEV /tmp/cmpdata
291+
292+
# trigger copyup for areas written in testimg1 (clone)
293+
xfs_io -c 'pwrite -S 0xcc -W 2084k 4k' $LIBRBD_DEV /tmp/cmpdata
294+
xfs_io -c 'pwrite -S 0xcc -W 32612k 4k' $LIBRBD_DEV /tmp/cmpdata
295+
296+
cmp $LIBRBD_DEV /tmp/cmpdata
297+
_sudo rbd device unmap -t nbd $LIBRBD_DEV
298+
299+
# test reading on a fresh mapping
300+
# FIXME: https://tracker.ceph.com/issues/63184
301+
LIBRBD_DEV=$(_sudo rbd -p rbd map testimg2 -t nbd -o encryption-passphrase-file=/tmp/passphrase2,encryption-passphrase-file=/tmp/passphrase2,encryption-passphrase-file=/tmp/passphrase)
302+
cmp $LIBRBD_DEV /tmp/cmpdata
303+
_sudo rbd device unmap -t nbd $LIBRBD_DEV
304+
305+
# test reading on a fresh mapping after migration is executed
306+
rbd migration execute testimg2
307+
LIBRBD_DEV=$(_sudo rbd -p rbd map testimg2 -t nbd -o encryption-passphrase-file=/tmp/passphrase2,encryption-passphrase-file=/tmp/passphrase)
308+
cmp $LIBRBD_DEV /tmp/cmpdata
309+
_sudo rbd device unmap -t nbd $LIBRBD_DEV
310+
311+
# test reading on a fresh mapping after migration is committed
312+
rbd migration commit testimg2
313+
LIBRBD_DEV=$(_sudo rbd -p rbd map testimg2 -t nbd -o encryption-passphrase-file=/tmp/passphrase2,encryption-passphrase-file=/tmp/passphrase)
314+
cmp $LIBRBD_DEV /tmp/cmpdata
315+
_sudo rbd device unmap -t nbd $LIBRBD_DEV
316+
317+
rbd rm testimg2
318+
rbd snap unprotect testimg@snap
319+
rbd snap rm testimg@snap
320+
rbd rm testimg
321+
}
322+
156323
function get_nbd_device_paths {
157324
rbd device list -t nbd | tail -n +2 | egrep "\s+rbd\s+testimg" | awk '{print $5;}'
158325
}
@@ -168,10 +335,16 @@ function clean_up {
168335
_sudo rbd device unmap -t nbd $device
169336
done
170337

338+
rbd migration abort testimg2 || true
171339
rbd remove testimg2 || true
340+
rbd migration abort testimg1 || true
341+
rbd snap remove testimg1@snap2 || true
342+
rbd snap remove testimg1@snap1 || true
172343
rbd snap unprotect testimg1@snap || true
173344
rbd snap remove testimg1@snap || true
174345
rbd remove testimg1 || true
346+
rbd snap remove testimg@snap2 || true
347+
rbd snap remove testimg@snap1 || true
175348
rbd snap unprotect testimg@snap || true
176349
rbd snap remove testimg@snap || true
177350
rbd remove testimg || true
@@ -205,7 +378,6 @@ rbd create testimg --size=32M
205378

206379
# map raw data to nbd device
207380
RAW_DEV=$(_sudo rbd -p rbd map testimg -t nbd)
208-
sudo chmod 666 $RAW_DEV
209381

210382
test_plaintext_detection
211383

@@ -214,4 +386,19 @@ test_encryption_format luks2
214386

215387
test_clone_encryption
216388

389+
_sudo rbd device unmap -t nbd $RAW_DEV
390+
rbd rm testimg
391+
392+
rbd create --size 20M testimg
393+
test_migration luks1
394+
395+
rbd create --size 32M testimg
396+
test_migration luks2
397+
398+
rbd create --size 36M testimg
399+
test_migration_clone luks1
400+
401+
rbd create --size 48M testimg
402+
test_migration_clone luks2
403+
217404
echo OK

‎src/librbd/io/ReadResult.cc‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -143,7 +143,7 @@ struct ReadResult::AssembleResultVisitor : public boost::static_visitor<void> {
143143

144144
ReadResult::C_ImageReadRequest::C_ImageReadRequest(
145145
AioCompletion *aio_completion, uint64_t buffer_offset,
146-
const Extents image_extents)
146+
const Extents& image_extents)
147147
: aio_completion(aio_completion), buffer_offset(buffer_offset),
148148
image_extents(image_extents) {
149149
aio_completion->add_request();

‎src/librbd/io/ReadResult.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ class ReadResult {
3434

3535
C_ImageReadRequest(AioCompletion *aio_completion,
3636
uint64_t buffer_offset,
37-
const Extents image_extents);
37+
const Extents& image_extents);
3838

3939
void finish(int r) override;
4040
};

0 commit comments

Comments
 (0)