rbd: add --estimated-size option for import from stdin

One issue with importing from stdin is that it's not easy to track
progress. The only feasible option is to process messages on the highest
log level looking for lines like

librbd::io::ImageRequestWQ: 0x56342ecc7a50 aio_write: ... off=1187840, len=864256 ...

but when it comes to large images it takes a lot of effort.
This commit introduces --estimated-size option, that makes it possible
to print out progress in percents via the standard mechanism. Obviously,
it requires the knowledge of the amount of provided data in advance and
in case of an error nonsensical percents might be printed, but I don't
think it's that big of a deal.

Also use `estimated size` as the base image size, making resizing not
necessary in cases where we know the exact amount of data provided from
stdin.

Co-authored-by: Ilya Dryomov <[email protected]>
Signed-off-by: Kirill Nazarov <[email protected]>
Signed-off-by: Ilya Dryomov <[email protected]>

Author: kinazarov1999

doc/man/8/rbd.rst

@@ -392,7 +392,7 @@ Commands
 :command:`image-meta set` *image-spec* *key* *value*
   Set metadata key with the value. They will displayed in `image-meta list`.
 
-:command:`import` [--export-format *format (1 or 2)*] [--image-format *format-id*] [--object-size *size-in-B/K/M*] [--stripe-unit *size-in-B/K/M* --stripe-count *num*] [--image-feature *feature-name*]... [--image-shared] *src-path* [*image-spec*]
+:command:`import` [--export-format *format (1 or 2)*] [--image-format *format-id*] [--object-size *size-in-B/K/M*] [--stripe-unit *size-in-B/K/M* --stripe-count *num*] [--image-feature *feature-name*] [--estimated-size *size-in-M/G/T*]... [--image-shared] *src-path* [*image-spec*]
   Create a new image and import its data from path (use - for
   stdin).  The import operation will try to create sparse rbd images 
   if possible.  For import from stdin, the sparsification unit is

src/test/cli/rbd/help.t

@@ -1258,7 +1258,8 @@
                     [--journal-object-size <journal-object-size>] 
                     [--journal-pool <journal-pool>] 
                     [--sparse-size <sparse-size>] [--no-progress] 
-                    [--export-format <export-format>] [--pool <pool>] 
+                    [--export-format <export-format>] 
+                    [--estimated-size <estimated-size>] [--pool <pool>] 
                     [--image <image>] 
                     <path-name> <dest-image-spec> 
 
@@ -1290,6 +1291,8 @@
     --sparse-size arg         sparse size in B/K/M [default: 4K]
     --no-progress             disable progress output
     --export-format arg       format of image file
+    --estimated-size arg      estimated image size (valid only for raw import
+                              from stdin, in M/G/T) [default: M]
   
   Image Features:
     (*) supports enabling/disabling on existing images

src/tools/rbd/ArgumentTypes.cc

@@ -279,6 +279,12 @@ void add_size_option(boost::program_options::options_description *opt) {
      "image size (in M/G/T) [default: M]");
 }
 
+void add_estimated_size_option(boost::program_options::options_description *opt) {
+  opt->add_options()
+    (IMAGE_ESTIMATED_SIZE.c_str(), po::value<ImageSize>(),
+     "estimated image size (valid only for raw import from stdin, in M/G/T) [default: M]");
+}
+
 void add_sparse_size_option(boost::program_options::options_description *opt) {
   opt->add_options()
     (IMAGE_SPARSE_SIZE.c_str(), po::value<ImageObjectSize>(),

src/tools/rbd/ArgumentTypes.h

@@ -67,6 +67,7 @@ static const std::string IMAGE_OBJECT_SIZE("object-size");
 static const std::string IMAGE_FEATURES("image-feature");
 static const std::string IMAGE_SHARED("image-shared");
 static const std::string IMAGE_SIZE("size");
+static const std::string IMAGE_ESTIMATED_SIZE("estimated-size");
 static const std::string IMAGE_STRIPE_UNIT("stripe-unit");
 static const std::string IMAGE_STRIPE_COUNT("stripe-count");
 static const std::string IMAGE_DATA_POOL("data-pool");
@@ -186,6 +187,8 @@ void add_create_journal_options(
 
 void add_size_option(boost::program_options::options_description *opt);
 
+void add_estimated_size_option(boost::program_options::options_description *opt);
+
 void add_sparse_size_option(boost::program_options::options_description *opt);
 
 void add_path_options(boost::program_options::options_description *pos,

src/tools/rbd/action/Import.cc

@@ -725,7 +725,7 @@ static int do_import_v2(librados::Rados &rados, int fd, librbd::Image &image,
 
 static int do_import_v1(int fd, librbd::Image &image, uint64_t size,
                         size_t imgblklen, utils::ProgressContext &pc,
-			size_t sparse_size)
+                        size_t sparse_size, size_t estimated_size)
 {
   int r = 0;
   size_t reqlen = imgblklen;    // amount requested from read
@@ -758,6 +758,8 @@ static int do_import_v1(int fd, librbd::Image &image, uint64_t size,
     }
     if (!from_stdin)
       pc.update_progress(image_pos, size);
+    else if (estimated_size != 0)
+      pc.update_progress(image_pos, estimated_size);
 
     bufferptr blkptr(p, blklen); 
     // resize output image by binary expansion as we go for stdin
@@ -823,7 +825,8 @@ static int do_import_v1(int fd, librbd::Image &image, uint64_t size,
 static int do_import(librados::Rados &rados, librbd::RBD &rbd,
 		     librados::IoCtx& io_ctx, const char *imgname,
 		     const char *path, librbd::ImageOptions& opts,
-		     bool no_progress, int import_format, size_t sparse_size)
+		     bool no_progress, int import_format, size_t sparse_size,
+		     size_t estimated_size)
 {
   int fd, r;
   struct stat stat_buf;
@@ -845,7 +848,11 @@ static int do_import(librados::Rados &rados, librbd::RBD &rbd,
   bool from_stdin = !strcmp(path, "-");
   if (from_stdin) {
     fd = STDIN_FILENO;
-    size = 1ULL << order;
+    if (estimated_size == 0) {
+      size = 1ULL << order;
+    } else {
+      size = estimated_size;
+    }
   } else {
     if ((fd = open(path, O_RDONLY|O_BINARY)) < 0) {
       r = -errno;
@@ -908,7 +915,8 @@ static int do_import(librados::Rados &rados, librbd::RBD &rbd,
   }
 
   if (import_format == 1) {
-    r = do_import_v1(fd, image, size, imgblklen, pc, sparse_size);
+    r = do_import_v1(fd, image, size, imgblklen, pc, sparse_size,
+                     estimated_size);
   } else {
     r = do_import_v2(rados, fd, image, size, imgblklen, pc, sparse_size);
   }
@@ -942,6 +950,7 @@ void get_arguments(po::options_description *positional,
   at::add_sparse_size_option(options);
   at::add_no_progress_option(options);
   at::add_export_format_option(options);
+  at::add_estimated_size_option(options);
 
   // TODO legacy rbd allowed import to accept both 'image'/'dest' and
   //      'pool'/'dest-pool'
@@ -1017,9 +1026,25 @@ int execute(const po::variables_map &vm,
   if (vm.count("export-format"))
     format = vm["export-format"].as<uint64_t>();
 
+  size_t estimated_size = 0;
+  if (vm.count(at::IMAGE_ESTIMATED_SIZE)) {
+    if (path != "-") {
+      std::cerr << "rbd: --estimated-size can be specified "
+                << "only for import from stdin" << std::endl;
+      return -EINVAL;
+    }
+    if (format != 1) {
+      std::cerr << "rbd: --estimated-size can be specified "
+                << "only for raw import (--export-format 1)" << std::endl;
+      return -EINVAL;
+    }
+    estimated_size = vm[at::IMAGE_ESTIMATED_SIZE].as<size_t>();
+  }
+
   librbd::RBD rbd;
   r = do_import(rados, rbd, io_ctx, image_name.c_str(), path.c_str(),
-                opts, vm[at::NO_PROGRESS].as<bool>(), format, sparse_size);
+                opts, vm[at::NO_PROGRESS].as<bool>(), format, sparse_size,
+                estimated_size);
   if (r < 0) {
     std::cerr << "rbd: import failed: " << cpp_strerror(r) << std::endl;
     return r;