Merge pull request ceph#61535 from kinazarov1999/import-stdin-progress

rbd: add --estimated-size option for import from stdin

Reviewed-by: Ilya Dryomov <[email protected]>

Author: idryomov

doc/man/8/rbd.rst

@@ -392,7 +392,7 @@ Commands
 :command:`image-meta set` *image-spec* *key* *value*
   Set metadata key with the value. They will displayed in `image-meta list`.
 
-:command:`import` [--export-format *format (1 or 2)*] [--image-format *format-id*] [--object-size *size-in-B/K/M*] [--stripe-unit *size-in-B/K/M* --stripe-count *num*] [--image-feature *feature-name*]... [--image-shared] *src-path* [*image-spec*]
+:command:`import` [--export-format *format (1 or 2)*] [--image-format *format-id*] [--object-size *size-in-B/K/M*] [--stripe-unit *size-in-B/K/M* --stripe-count *num*] [--image-feature *feature-name*] [--estimated-size *size-in-M/G/T*]... [--image-shared] *src-path* [*image-spec*]
   Create a new image and import its data from path (use - for
   stdin).  The import operation will try to create sparse rbd images 
   if possible.  For import from stdin, the sparsification unit is

src/test/cli/rbd/help.t

@@ -1258,7 +1258,8 @@
                     [--journal-object-size <journal-object-size>] 
                     [--journal-pool <journal-pool>] 
                     [--sparse-size <sparse-size>] [--no-progress] 
-                    [--export-format <export-format>] [--pool <pool>] 
+                    [--export-format <export-format>] 
+                    [--estimated-size <estimated-size>] [--pool <pool>] 
                     [--image <image>] 
                     <path-name> <dest-image-spec> 
 
@@ -1290,6 +1291,8 @@
     --sparse-size arg         sparse size in B/K/M [default: 4K]
     --no-progress             disable progress output
     --export-format arg       format of image file
+    --estimated-size arg      estimated image size (valid only for raw import
+                              from stdin, in M/G/T) [default: M]
   
   Image Features:
     (*) supports enabling/disabling on existing images

src/tools/rbd/ArgumentTypes.cc

@@ -279,6 +279,12 @@ void add_size_option(boost::program_options::options_description *opt) {
      "image size (in M/G/T) [default: M]");
 }
 
+void add_estimated_size_option(boost::program_options::options_description *opt) {
+  opt->add_options()
+    (IMAGE_ESTIMATED_SIZE.c_str(), po::value<ImageSize>(),
+     "estimated image size (valid only for raw import from stdin, in M/G/T) [default: M]");
+}
+
 void add_sparse_size_option(boost::program_options::options_description *opt) {
   opt->add_options()
     (IMAGE_SPARSE_SIZE.c_str(), po::value<ImageObjectSize>(),

src/tools/rbd/ArgumentTypes.h

@@ -67,6 +67,7 @@ static const std::string IMAGE_OBJECT_SIZE("object-size");
 static const std::string IMAGE_FEATURES("image-feature");
 static const std::string IMAGE_SHARED("image-shared");
 static const std::string IMAGE_SIZE("size");
+static const std::string IMAGE_ESTIMATED_SIZE("estimated-size");
 static const std::string IMAGE_STRIPE_UNIT("stripe-unit");
 static const std::string IMAGE_STRIPE_COUNT("stripe-count");
 static const std::string IMAGE_DATA_POOL("data-pool");
@@ -186,6 +187,8 @@ void add_create_journal_options(
 
 void add_size_option(boost::program_options::options_description *opt);
 
+void add_estimated_size_option(boost::program_options::options_description *opt);
+
 void add_sparse_size_option(boost::program_options::options_description *opt);
 
 void add_path_options(boost::program_options::options_description *pos,

src/tools/rbd/action/Import.cc

@@ -725,7 +725,7 @@ static int do_import_v2(librados::Rados &rados, int fd, librbd::Image &image,
 
 static int do_import_v1(int fd, librbd::Image &image, uint64_t size,
                         size_t imgblklen, utils::ProgressContext &pc,
-			size_t sparse_size)
+                        size_t sparse_size, size_t estimated_size)
 {
   int r = 0;
   size_t reqlen = imgblklen;    // amount requested from read
@@ -758,6 +758,8 @@ static int do_import_v1(int fd, librbd::Image &image, uint64_t size,
     }
     if (!from_stdin)
       pc.update_progress(image_pos, size);
+    else if (estimated_size != 0)
+      pc.update_progress(image_pos, estimated_size);
 
     bufferptr blkptr(p, blklen); 
     // resize output image by binary expansion as we go for stdin
@@ -823,7 +825,8 @@ static int do_import_v1(int fd, librbd::Image &image, uint64_t size,
 static int do_import(librados::Rados &rados, librbd::RBD &rbd,
 		     librados::IoCtx& io_ctx, const char *imgname,
 		     const char *path, librbd::ImageOptions& opts,
-		     bool no_progress, int import_format, size_t sparse_size)
+		     bool no_progress, int import_format, size_t sparse_size,
+		     size_t estimated_size)
 {
   int fd, r;
   struct stat stat_buf;
@@ -845,7 +848,11 @@ static int do_import(librados::Rados &rados, librbd::RBD &rbd,
   bool from_stdin = !strcmp(path, "-");
   if (from_stdin) {
     fd = STDIN_FILENO;
-    size = 1ULL << order;
+    if (estimated_size == 0) {
+      size = 1ULL << order;
+    } else {
+      size = estimated_size;
+    }
   } else {
     if ((fd = open(path, O_RDONLY|O_BINARY)) < 0) {
       r = -errno;
@@ -908,7 +915,8 @@ static int do_import(librados::Rados &rados, librbd::RBD &rbd,
   }
 
   if (import_format == 1) {
-    r = do_import_v1(fd, image, size, imgblklen, pc, sparse_size);
+    r = do_import_v1(fd, image, size, imgblklen, pc, sparse_size,
+                     estimated_size);
   } else {
     r = do_import_v2(rados, fd, image, size, imgblklen, pc, sparse_size);
   }
@@ -942,6 +950,7 @@ void get_arguments(po::options_description *positional,
   at::add_sparse_size_option(options);
   at::add_no_progress_option(options);
   at::add_export_format_option(options);
+  at::add_estimated_size_option(options);
 
   // TODO legacy rbd allowed import to accept both 'image'/'dest' and
   //      'pool'/'dest-pool'
@@ -1017,9 +1026,25 @@ int execute(const po::variables_map &vm,
   if (vm.count("export-format"))
     format = vm["export-format"].as<uint64_t>();
 
+  size_t estimated_size = 0;
+  if (vm.count(at::IMAGE_ESTIMATED_SIZE)) {
+    if (path != "-") {
+      std::cerr << "rbd: --estimated-size can be specified "
+                << "only for import from stdin" << std::endl;
+      return -EINVAL;
+    }
+    if (format != 1) {
+      std::cerr << "rbd: --estimated-size can be specified "
+                << "only for raw import (--export-format 1)" << std::endl;
+      return -EINVAL;
+    }
+    estimated_size = vm[at::IMAGE_ESTIMATED_SIZE].as<size_t>();
+  }
+
   librbd::RBD rbd;
   r = do_import(rados, rbd, io_ctx, image_name.c_str(), path.c_str(),
-                opts, vm[at::NO_PROGRESS].as<bool>(), format, sparse_size);
+                opts, vm[at::NO_PROGRESS].as<bool>(), format, sparse_size,
+                estimated_size);
   if (r < 0) {
     std::cerr << "rbd: import failed: " << cpp_strerror(r) << std::endl;
     return r;