Skip to content

Commit 7f24e58

Browse files
idryomovidryomov
authored
Merge pull request ceph#44640 from effi-ofer/live-migration-nbd
librbd/migration: add NBD stream Reviewed-by: Mykola Golub <[email protected]> Reviewed-by: Ramana Raja <[email protected]>
2 parents 485717a + 6fd11c0 commit 7f24e58

29 files changed

+1641
-10
lines changed

‎CMakeLists.txt‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -259,6 +259,12 @@ if(WITH_RBD AND LINUX)
259259
set(HAVE_LIBCRYPTSETUP ${LIBCRYPTSETUP_FOUND})
260260
endif()
261261

262+
# libnbd
263+
if(WITH_RBD AND NOT WIN32)
264+
find_package(libnbd 1.0 REQUIRED)
265+
set(HAVE_LIBNBD ${LIBNBD_FOUND})
266+
endif()
267+
262268
include(CMakeDependentOption)
263269

264270
CMAKE_DEPENDENT_OPTION(WITH_LIBURING "Enable io_uring bluestore backend" ON

‎ceph.spec.in‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -286,6 +286,7 @@ BuildRequires: gperftools-devel >= 2.4
286286
BuildRequires: libaio-devel
287287
BuildRequires: libblkid-devel >= 2.17
288288
BuildRequires: cryptsetup-devel
289+
BuildRequires: libnbd-devel
289290
BuildRequires: libcurl-devel
290291
BuildRequires: libcap-devel
291292
BuildRequires: libcap-ng-devel

‎cmake/modules/Findlibnbd.cmake‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
# - Find libnbd
2+
# Sets the following:
3+
#
4+
# LIBNBD_INCLUDE_DIR
5+
# LIBNBD_LIBRARIES
6+
# LIBNBD_VERSION
7+
# LIBNBD_FOUND
8+
9+
find_package(PkgConfig QUIET REQUIRED)
10+
pkg_search_module(PC_libnbd libnbd)
11+
12+
find_path(LIBNBD_INCLUDE_DIR
13+
NAMES libnbd.h
14+
PATHS ${PC_libnbd_INCLUDE_DIRS})
15+
16+
find_library(LIBNBD_LIBRARIES
17+
NAMES libnbd.so
18+
PATHS ${PC_libnbd_LIBRARY_DIRS})
19+
20+
set(LIBNBD_VERSION ${PC_libnbd_VERSION})
21+
22+
include(FindPackageHandleStandardArgs)
23+
24+
find_package_handle_standard_args(libnbd
25+
REQUIRED_VARS
26+
LIBNBD_INCLUDE_DIR
27+
LIBNBD_LIBRARIES
28+
VERSION_VAR LIBNBD_VERSION)
29+
30+
mark_as_advanced(
31+
LIBNBD_LIBRARIES
32+
LIBNBD_INCLUDE_DIR
33+
LIBNBD_VERSION)

‎debian/control‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@ Build-Depends: automake,
5454
liblttng-ust-dev,
5555
liblua5.3-dev,
5656
liblz4-dev (>= 0.0~r131),
57+
libnbd-dev,
5758
libncurses-dev,
5859
libnss3-dev,
5960
liboath-dev,

‎doc/rbd/rbd-live-migration.rst‎

Lines changed: 31 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ parent.
2020
The live-migration process can also be used in an import-only mode where the
2121
source image remains unmodified and the target image can be linked to an image
2222
in another Ceph cluster or to an external data source such as a backing file,
23-
HTTP(s) file, or S3 object.
23+
HTTP(s) file, S3 object, or NBD export.
2424

2525
The live-migration copy process can safely run in the background while the new
2626
target image is in use. There is currently a requirement to temporarily stop
@@ -145,8 +145,8 @@ The general format for the ``source-spec`` JSON is as follows::
145145
}
146146

147147
The following formats are currently supported: ``native``, ``qcow``, and
148-
``raw``. The following streams are currently supported: ``file``, ``http``, and
149-
``s3``.
148+
``raw``. The following streams are currently supported: ``file``, ``http``,
149+
``s3``, and ``nbd``.
150150

151151
Formats
152152
~~~~~~~
@@ -306,6 +306,33 @@ as follows::
306306
stored in the config-key store via ``ceph config-key set <key-path> <value>``
307307
(e.g. ``ceph config-key set rbd/s3/access_key NX5QOQKC6BH2IDN8HC7A``).
308308

309+
The ``nbd`` stream can be used to import from a remote NBD export. Its
310+
``source-spec`` JSON is encoded as follows::
311+
312+
{
313+
<format unique parameters>
314+
"stream": {
315+
"type": "nbd",
316+
"uri": "<nbd-uri>",
317+
}
318+
}
319+
320+
For example, to import a raw-format image from an NBD export located at
321+
``nbd://nbd.ceph.com`` with export name ``image.raw``, its ``source-spec``
322+
JSON is encoded as follows::
323+
324+
{
325+
"type": "raw",
326+
"stream": {
327+
"type": "nbd",
328+
"uri": "nbd://nbd.ceph.com/image.raw",
329+
}
330+
}
331+
332+
``nbd-uri`` parameter should follow the `NBD URI specification`_. The
333+
default NBD port is ``10809``.
334+
335+
309336
Execute Migration
310337
=================
311338

@@ -370,3 +397,4 @@ to the original source image being restored::
370397

371398

372399
.. _layered images: ../rbd-snapshot/#layering
400+
.. _NBD URI specification: https://github.com/NetworkBlockDevice/nbd/blob/master/doc/uri.md

‎qa/suites/rbd/migration/6-prepare/qcow2-nbd.yaml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
tasks:
2+
- exec:
3+
client.0:
4+
- mkdir /home/ubuntu/cephtest/migration
5+
- wget -nv -O /home/ubuntu/cephtest/migration/base.client.0.qcow2 http://download.ceph.com/qa/ubuntu-12.04.qcow2
6+
- qemu-img create -f qcow2 /home/ubuntu/cephtest/migration/empty.qcow2 1G
7+
- qemu-nbd -f qcow2 --read-only --shared 10 --persistent --fork /home/ubuntu/cephtest/migration/base.client.0.qcow2
8+
- qemu-nbd -f qcow2 --read-only --shared 10 --persistent --fork --socket /home/ubuntu/cephtest/migration/qemu-nbd-empty /home/ubuntu/cephtest/migration/empty.qcow2
9+
- chmod 0777 /home/ubuntu/cephtest/migration/qemu-nbd-empty
10+
- echo '{"type":"raw","stream":{"type":"nbd","uri":"nbd://localhost"}}' | rbd migration prepare --import-only --source-spec-path - client.0.0
11+
- rbd migration prepare --import-only --source-spec '{"type":"raw","stream":{"type":"nbd","uri":"nbd+unix:///?socket=/home/ubuntu/cephtest/migration/qemu-nbd-empty"}}' client.0.1
12+
- rbd migration prepare --import-only --source-spec '{"type":"raw","stream":{"type":"nbd","uri":"nbd+unix:///?socket=/home/ubuntu/cephtest/migration/qemu-nbd-empty"}}' client.0.2

‎qa/suites/rbd/migration/6-prepare/raw-nbd.yaml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
tasks:
2+
- exec:
3+
client.0:
4+
- mkdir /home/ubuntu/cephtest/migration
5+
- wget -nv -O /home/ubuntu/cephtest/migration/base.client.0.qcow2 http://download.ceph.com/qa/ubuntu-12.04.qcow2
6+
- qemu-img convert -f qcow2 -O raw /home/ubuntu/cephtest/migration/base.client.0.qcow2 /home/ubuntu/cephtest/migration/base.client.0.raw
7+
- dd if=/dev/zero of=/home/ubuntu/cephtest/migration/empty.raw count=1 bs=1G
8+
- qemu-nbd -f raw --read-only --shared 10 --persistent --fork /home/ubuntu/cephtest/migration/base.client.0.raw
9+
- qemu-nbd -f raw --read-only --shared 10 --persistent --fork --socket /home/ubuntu/cephtest/migration/qemu-nbd-empty /home/ubuntu/cephtest/migration/empty.raw
10+
- chmod 0777 /home/ubuntu/cephtest/migration/qemu-nbd-empty
11+
- echo '{"type":"raw","stream":{"type":"nbd","uri":"nbd://localhost"}}' | rbd migration prepare --import-only --source-spec-path - client.0.0
12+
- rbd migration prepare --import-only --source-spec '{"type":"raw","stream":{"type":"nbd","uri":"nbd+unix:///?socket=/home/ubuntu/cephtest/migration/qemu-nbd-empty"}}' client.0.1
13+
- rbd migration prepare --import-only --source-spec '{"type":"raw","stream":{"type":"nbd","uri":"nbd+unix:///?socket=/home/ubuntu/cephtest/migration/qemu-nbd-empty"}}' client.0.2

‎qa/suites/rbd/migration/9-cleanup/cleanup.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
tasks:
22
- exec:
33
client.0:
4+
- pkill -9 qemu-nbd || true
45
- rm -rf /home/ubuntu/cephtest/migration

‎qa/workunits/rbd/cli_migration.sh‎

Lines changed: 180 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ IMAGE3=image3
88
IMAGES="${IMAGE1} ${IMAGE2} ${IMAGE3}"
99

1010
cleanup() {
11+
kill_nbd_server
1112
cleanup_tempdir
1213
remove_images
1314
}
@@ -68,6 +69,10 @@ remove_images() {
6869
done
6970
}
7071

72+
kill_nbd_server() {
73+
pkill -9 qemu-nbd || true
74+
}
75+
7176
show_diff()
7277
{
7378
local file1=$1
@@ -393,6 +398,177 @@ EOF
393398
remove_image "${dest_image}"
394399
}
395400

401+
test_import_nbd_stream_qcow2() {
402+
local base_image=$1
403+
local dest_image=$2
404+
405+
qemu-nbd -f qcow2 --read-only --shared 10 --persistent --fork \
406+
${TEMPDIR}/${base_image}.qcow2
407+
408+
cat > ${TEMPDIR}/spec.json <<EOF
409+
{
410+
"type": "raw",
411+
"stream": {
412+
"type": "nbd",
413+
"uri": "nbd://localhost"
414+
}
415+
}
416+
EOF
417+
cat ${TEMPDIR}/spec.json
418+
419+
cat ${TEMPDIR}/spec.json | rbd migration prepare --import-only \
420+
--source-spec-path - ${dest_image}
421+
compare_images ${base_image} ${dest_image}
422+
rbd migration abort ${dest_image}
423+
424+
rbd migration prepare --import-only \
425+
--source-spec-path ${TEMPDIR}/spec.json ${dest_image}
426+
compare_images ${base_image} ${dest_image}
427+
rbd migration execute ${dest_image}
428+
compare_images ${base_image} ${dest_image}
429+
rbd migration commit ${dest_image}
430+
compare_images ${base_image} ${dest_image}
431+
remove_image "${dest_image}"
432+
433+
# shortest possible URI
434+
rbd migration prepare --import-only \
435+
--source-spec '{"type": "raw", "stream": {"type": "nbd", "uri": "nbd://"}}' \
436+
${dest_image}
437+
rbd migration abort ${dest_image}
438+
439+
# non-existing export name
440+
expect_false rbd migration prepare --import-only \
441+
--source-spec '{"type": "raw", "stream": {"type": "nbd", "uri": "nbd:///myexport"}}' \
442+
${dest_image}
443+
expect_false rbd migration prepare --import-only \
444+
--source-spec '{"type": "raw", "stream": {"type": "nbd", "uri": "nbd://localhost/myexport"}}' \
445+
${dest_image}
446+
447+
kill_nbd_server
448+
qemu-nbd --export-name myexport -f qcow2 --read-only --shared 10 --persistent --fork \
449+
${TEMPDIR}/${base_image}.qcow2
450+
451+
rbd migration prepare --import-only \
452+
--source-spec '{"type": "raw", "stream": {"type": "nbd", "uri": "nbd:///myexport"}}' \
453+
${dest_image}
454+
rbd migration abort ${dest_image}
455+
456+
rbd migration prepare --import-only \
457+
--source-spec '{"type": "raw", "stream": {"type": "nbd", "uri": "nbd://localhost/myexport"}}' \
458+
${dest_image}
459+
rbd migration abort ${dest_image}
460+
461+
kill_nbd_server
462+
463+
# server not running
464+
expect_false rbd migration prepare --import-only \
465+
--source-spec-path ${TEMPDIR}/spec.json ${dest_image}
466+
expect_false rbd migration prepare --import-only \
467+
--source-spec '{"type": "raw", "stream": {"type": "nbd", "uri": "nbd://"}}' \
468+
${dest_image}
469+
470+
# no URI
471+
expect_false rbd migration prepare --import-only \
472+
--source-spec '{"type": "raw", "stream": {"type": "nbd"}}' \
473+
${dest_image}
474+
475+
# invalid URI
476+
expect_false rbd migration prepare --import-only \
477+
--source-spec '{"type": "raw", "stream": {"type": "nbd", "uri": 123456}}' \
478+
${dest_image}
479+
480+
# libnbd - nbd_get_errno() returns an error
481+
# nbd_connect_uri: unknown URI scheme: NULL: Invalid argument (errno = 22)
482+
expect_false rbd migration prepare --import-only \
483+
--source-spec '{"type": "raw", "stream": {"type": "nbd", "uri": ""}}' \
484+
${dest_image}
485+
expect_false rbd migration prepare --import-only \
486+
--source-spec '{"type": "raw", "stream": {"type": "nbd", "uri": "foo.example.com"}}' \
487+
${dest_image}
488+
489+
# libnbd - nbd_get_errno() returns 0, EIO fallback
490+
# nbd_connect_uri: getaddrinfo: foo.example.com:10809: Name or service not known (errno = 0)
491+
expect_false rbd migration prepare --import-only \
492+
--source-spec '{"type": "raw", "stream": {"type": "nbd", "uri": "nbd://foo.example.com"}}' \
493+
${dest_image}
494+
}
495+
496+
test_import_nbd_stream_raw() {
497+
local base_image=$1
498+
local dest_image=$2
499+
500+
qemu-nbd -f raw --read-only --shared 10 --persistent --fork \
501+
--socket ${TEMPDIR}/qemu-nbd-${base_image} ${TEMPDIR}/${base_image}
502+
qemu-nbd -f raw --read-only --shared 10 --persistent --fork \
503+
--socket ${TEMPDIR}/qemu-nbd-${base_image}@1 ${TEMPDIR}/${base_image}@1
504+
qemu-nbd -f raw --read-only --shared 10 --persistent --fork \
505+
--socket ${TEMPDIR}/qemu-nbd-${base_image}@2 ${TEMPDIR}/${base_image}@2
506+
507+
cat > ${TEMPDIR}/spec.json <<EOF
508+
{
509+
"type": "raw",
510+
"stream": {
511+
"type": "nbd",
512+
"uri": "nbd+unix:///?socket=${TEMPDIR}/qemu-nbd-${base_image}"
513+
},
514+
"snapshots": [{
515+
"type": "raw",
516+
"name": "snap1",
517+
"stream": {
518+
"type": "nbd",
519+
"uri": "nbd+unix:///?socket=${TEMPDIR}/qemu-nbd-${base_image}@1"
520+
}
521+
}, {
522+
"type": "raw",
523+
"name": "snap2",
524+
"stream": {
525+
"type": "nbd",
526+
"uri": "nbd+unix:///?socket=${TEMPDIR}/qemu-nbd-${base_image}@2"
527+
}
528+
}]
529+
}
530+
EOF
531+
cat ${TEMPDIR}/spec.json
532+
533+
rbd migration prepare --import-only \
534+
--source-spec-path ${TEMPDIR}/spec.json ${dest_image}
535+
536+
rbd snap create ${dest_image}@head
537+
rbd bench --io-type write --io-pattern rand --io-size 32K --io-total 4M ${dest_image}
538+
539+
compare_images "${base_image}@1" "${dest_image}@snap1"
540+
compare_images "${base_image}@2" "${dest_image}@snap2"
541+
compare_images "${base_image}" "${dest_image}@head"
542+
543+
rbd migration abort ${dest_image}
544+
545+
cat ${TEMPDIR}/spec.json | rbd migration prepare --import-only \
546+
--source-spec-path - ${dest_image}
547+
548+
rbd snap create ${dest_image}@head
549+
rbd bench --io-type write --io-pattern rand --io-size 64K --io-total 8M ${dest_image}
550+
551+
compare_images "${base_image}@1" "${dest_image}@snap1"
552+
compare_images "${base_image}@2" "${dest_image}@snap2"
553+
compare_images "${base_image}" "${dest_image}@head"
554+
555+
rbd migration execute ${dest_image}
556+
557+
compare_images "${base_image}@1" "${dest_image}@snap1"
558+
compare_images "${base_image}@2" "${dest_image}@snap2"
559+
compare_images "${base_image}" "${dest_image}@head"
560+
561+
rbd migration commit ${dest_image}
562+
563+
compare_images "${base_image}@1" "${dest_image}@snap1"
564+
compare_images "${base_image}@2" "${dest_image}@snap2"
565+
compare_images "${base_image}" "${dest_image}@head"
566+
567+
remove_image "${dest_image}"
568+
569+
kill_nbd_server
570+
}
571+
396572
# make sure rbd pool is EMPTY.. this is a test script!!
397573
rbd ls 2>&1 | wc -l | grep -v '^0$' && echo "nonempty rbd pool, aborting! run this script on an empty test cluster only." && exit 1
398574

@@ -404,7 +580,11 @@ export_base_image ${IMAGE1}
404580

405581
test_import_native_format ${IMAGE1} ${IMAGE2}
406582
test_import_qcow_format ${IMAGE1} ${IMAGE2}
583+
407584
test_import_qcow2_format ${IMAGE2} ${IMAGE3}
585+
test_import_nbd_stream_qcow2 ${IMAGE2} ${IMAGE3}
586+
408587
test_import_raw_format ${IMAGE1} ${IMAGE2}
588+
test_import_nbd_stream_raw ${IMAGE1} ${IMAGE2}
409589

410590
echo OK

‎src/include/config-h.in.cmake‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -393,6 +393,9 @@
393393
/* Define if libcryptsetup can be used (linux only) */
394394
#cmakedefine HAVE_LIBCRYPTSETUP
395395

396+
/* Define if libnbd can be used */
397+
#cmakedefine HAVE_LIBNBD
398+
396399
/* Shared library extension, such as .so, .dll or .dylib */
397400
#cmakedefine CMAKE_SHARED_LIBRARY_SUFFIX "@CMAKE_SHARED_LIBRARY_SUFFIX@"
398401

0 commit comments

Comments
 (0)