tools/rbd: introduce a simple bench for read

Currently, we have a bench-write for in rbd tool, but we don't have
a simple bench for read.

This patch add a new subcommand of bench, with an option of --io-type.
Then we can do a simple bench for write or read.

Signed-off-by: Dongsheng Yang <[email protected]>

doc/man/8/rbd.rst

@@ -369,6 +369,13 @@ Commands
   Release a lock on an image. The lock id and locker are
   as output by lock ls.
 
+:command:`bench` --io-type <read | write> [--io-size *size-in-B/K/M/G/T*] [--io-threads *num-ios-in-flight*] [--io-total *total-size-for-IO-in-B/K/M/G/T*] [--io-pattern seq | rand] *image-spec*
+  Generate a series of IOs to the image and measure the IO throughput and
+  latency.  Defaults are: --io-size 4096, --io-threads 16, --io-total 1G,
+  --io-pattern seq.
+
+  NOTE: --io-type must be specified.
+
 Image and snap specs
 ====================
 

src/CMakeLists.txt

@@ -1261,7 +1261,7 @@ if(${WITH_RBD})
     tools/rbd/OptionPrinter.cc
     tools/rbd/Shell.cc
     tools/rbd/Utils.cc
-    tools/rbd/action/BenchWrite.cc
+    tools/rbd/action/Bench.cc
     tools/rbd/action/Children.cc
     tools/rbd/action/Clone.cc
     tools/rbd/action/Copy.cc

src/test/cli-integration/rbd/formatted-output.t

@@ -33,7 +33,7 @@ clone
   $ rbd clone --image-feature layering,exclusive-lock,object-map,fast-diff bar@snap rbd_other/child
   $ rbd snap create rbd_other/child@snap
   $ rbd flatten rbd_other/child 2> /dev/null
-  $ rbd bench-write rbd_other/child --io-pattern seq --io-total 1B > /dev/null 2>&1
+  $ rbd bench rbd_other/child --io-type write --io-pattern seq --io-total 1B > /dev/null 2>&1
   $ rbd clone bar@snap rbd_other/deep-flatten-child
   $ rbd snap create rbd_other/deep-flatten-child@snap
   $ rbd flatten rbd_other/deep-flatten-child 2> /dev/null

src/test/cli/rbd/help.t

@@ -5,7 +5,7 @@
 
   Positional arguments:
     <command>
-      bench-write                 Simple write benchmark.
+      bench                       Simple benchmark.
       children                    Display children of snapshot.
       clone                       Clone a snapshot into a COW child image.
       copy (cp)                   Copy src image to dest.
@@ -96,13 +96,13 @@
 
   See 'rbd help <command>' for help on a specific command.
   $ rbd help | grep '^    [a-z]' | sed 's/^    \([a-z -]*[a-z]\).*/\1/g' | while read -r line; do echo rbd help $line ; rbd help $line; done
-  rbd help bench-write
-  usage: rbd bench-write [--pool <pool>] [--image <image>] [--io-size <io-size>] 
-                         [--io-threads <io-threads>] [--io-total <io-total>] 
-                         [--io-pattern <io-pattern>] 
-                         <image-spec> 
+  rbd help bench
+  usage: rbd bench [--pool <pool>] [--image <image>] [--io-size <io-size>] 
+                   [--io-threads <io-threads>] [--io-total <io-total>] 
+                   [--io-pattern <io-pattern>] --io-type <io-type> 
+                   <image-spec> 
 
-  Simple write benchmark.
+  Simple benchmark.
 
   Positional arguments
     <image-spec>         image specification
@@ -111,10 +111,11 @@
   Optional arguments
     -p [ --pool ] arg    pool name
     --image arg          image name
-    --io-size arg        write size (in B/K/M/G/T)
+    --io-size arg        IO size (in B/K/M/G/T)
     --io-threads arg     ios in flight
-    --io-total arg       total size to write (in B/K/M/G/T)
-    --io-pattern arg     write pattern (rand or seq)
+    --io-total arg       total size for IO (in B/K/M/G/T)
+    --io-pattern arg     IO pattern (rand or seq)
+    --io-type arg        IO type (read or write)
 
   rbd help children
   usage: rbd children [--pool <pool>] [--image <image>] [--snap <snap>] 

src/test/cli/rbd/invalid-snap-usage.t

@@ -88,7 +88,7 @@
   $ rbd lock remove foo@snap id client.1234
   rbd: snapname specified for a command that doesn't use it
   [22]
-  $ rbd bench-write foo@snap
+  $ rbd bench foo@snap --io-type write
   rbd: snapname specified for a command that doesn't use it
   [22]
 

src/test/cli/rbd/not-enough-args.t

@@ -187,7 +187,7 @@
   $ rbd lock remove foo id
   rbd: locker was not specified
   [22]
-  $ rbd bench-write
+  $ rbd bench --io-type write
   rbd: image name was not specified
   [22]
   $ rbd mirror pool enable rbd

src/tools/Makefile-client.am

@@ -30,7 +30,7 @@ rbd_SOURCES = \
 	tools/rbd/OptionPrinter.cc \
 	tools/rbd/Shell.cc \
 	tools/rbd/Utils.cc \
-	tools/rbd/action/BenchWrite.cc \
+	tools/rbd/action/Bench.cc \
 	tools/rbd/action/Children.cc \
 	tools/rbd/action/Clone.cc \
 	tools/rbd/action/Group.cc \

src/tools/rbd/action/BenchWrite.cc → src/tools/rbd/action/Bench.cc

@@ -16,13 +16,21 @@
 
 namespace rbd {
 namespace action {
-namespace bench_write {
+namespace bench {
 
 namespace at = argument_types;
 namespace po = boost::program_options;
 
 namespace {
 
+enum io_type_t {
+  IO_TYPE_READ = 0,
+  IO_TYPE_WRITE,
+
+  IO_TYPE_NUM,
+};
+
+struct IOType {};
 struct Size {};
 struct IOPattern {};
 
@@ -50,38 +58,95 @@ void validate(boost::any& v, const std::vector<std::string>& values,
   } else {
     throw po::validation_error(po::validation_error::invalid_option_value);
   }
+}
 
+io_type_t get_io_type(string io_type_string) {
+  if (io_type_string == "read")
+    return IO_TYPE_READ;
+  else if (io_type_string == "write")
+    return IO_TYPE_WRITE;
+  else
+    return IO_TYPE_NUM;
+}
+
+void validate(boost::any& v, const std::vector<std::string>& values,
+              IOType *target_type, int) {
+  po::validators::check_first_occurrence(v);
+  const std::string &s = po::validators::get_single_string(values);
+  io_type_t io_type = get_io_type(s);
+  if (io_type >= IO_TYPE_NUM)
+    throw po::validation_error(po::validation_error::invalid_option_value);
+  else
+    v = boost::any(io_type);
 }
 
 } // anonymous namespace
 
 static void rbd_bencher_completion(void *c, void *pc);
 struct rbd_bencher;
 
+struct bencher_completer {
+  rbd_bencher *bencher;
+  bufferlist *bl;
+
+public:
+  bencher_completer(rbd_bencher *bencher, bufferlist *bl)
+    : bencher(bencher), bl(bl)
+  { }
+
+  ~bencher_completer()
+  {
+    if (bl)
+      delete bl;
+  }
+};
+
 struct rbd_bencher {
   librbd::Image *image;
   Mutex lock;
   Cond cond;
   int in_flight;
+  io_type_t io_type;
+  uint64_t io_size;
+  bufferlist write_bl;
 
-  explicit rbd_bencher(librbd::Image *i)
+  explicit rbd_bencher(librbd::Image *i, io_type_t io_type, uint64_t io_size)
     : image(i),
       lock("rbd_bencher::lock"),
-      in_flight(0)
-  { }
+      in_flight(0),
+      io_type(io_type),
+      io_size(io_size)
+  {
+    if (io_type == IO_TYPE_WRITE) {
+      bufferptr bp(io_size);
+      memset(bp.c_str(), rand() & 0xff, io_size);
+      write_bl.push_back(bp);
+    }
+  }
 
-  bool start_write(int max, uint64_t off, uint64_t len, bufferlist& bl,
-                   int op_flags)
+
+  bool start_io(int max, uint64_t off, uint64_t len, int op_flags)
   {
     {
       Mutex::Locker l(lock);
       if (in_flight >= max)
         return false;
       in_flight++;
     }
-    librbd::RBD::AioCompletion *c =
-      new librbd::RBD::AioCompletion((void *)this, rbd_bencher_completion);
-    image->aio_write2(off, len, bl, c, op_flags);
+
+    librbd::RBD::AioCompletion *c;
+    if (io_type == IO_TYPE_READ) {
+      bufferlist *read_bl = new bufferlist();
+      c = new librbd::RBD::AioCompletion((void *)(new bencher_completer(this, read_bl)),
+					 rbd_bencher_completion);
+      image->aio_read2(off, len, *read_bl, c, op_flags);
+    } else if (io_type == IO_TYPE_WRITE) {
+      c = new librbd::RBD::AioCompletion((void *)(new bencher_completer(this, NULL)),
+					 rbd_bencher_completion);
+      image->aio_write2(off, len, write_bl, c, op_flags);
+    } else {
+      assert(0 == "Invalid io_type");
+    }
     //cout << "start " << c << " at " << off << "~" << len << std::endl;
     return true;
   }
@@ -100,27 +165,33 @@ struct rbd_bencher {
 void rbd_bencher_completion(void *vc, void *pc)
 {
   librbd::RBD::AioCompletion *c = (librbd::RBD::AioCompletion *)vc;
-  rbd_bencher *b = static_cast<rbd_bencher *>(pc);
+  bencher_completer *bc = static_cast<bencher_completer *>(pc);
+  rbd_bencher *b = bc->bencher;
   //cout << "complete " << c << std::endl;
   int ret = c->get_return_value();
-  if (ret != 0) {
+  if (b->io_type == IO_TYPE_WRITE && ret != 0) {
     cout << "write error: " << cpp_strerror(ret) << std::endl;
     exit(ret < 0 ? -ret : ret);
+  } else if (b->io_type == IO_TYPE_READ && (unsigned int)ret != b->io_size) {
+    cout << "read error: " << cpp_strerror(ret) << std::endl;
+    exit(ret < 0 ? -ret : ret);
   }
   b->lock.Lock();
   b->in_flight--;
   b->cond.Signal();
   b->lock.Unlock();
   c->release();
+  delete bc;
 }
 
-int do_bench_write(librbd::Image& image, uint64_t io_size,
-                   uint64_t io_threads, uint64_t io_bytes,
-                   bool random)
+int do_bench(librbd::Image& image, io_type_t io_type,
+		   uint64_t io_size, uint64_t io_threads,
+		   uint64_t io_bytes, bool random)
 {
-  rbd_bencher b(&image);
+  rbd_bencher b(&image, io_type, io_size);
 
-  std::cout << "bench-write "
+  std::cout << "bench "
+       << " type " << (io_type == IO_TYPE_READ ? "read" : "write")
        << " io_size " << io_size
        << " io_threads " << io_threads
        << " bytes " << io_bytes
@@ -129,11 +200,6 @@ int do_bench_write(librbd::Image& image, uint64_t io_size,
 
   srand(time(NULL) % (unsigned long) -1);
 
-  bufferptr bp(io_size);
-  memset(bp.c_str(), rand() & 0xff, io_size);
-  bufferlist bl;
-  bl.push_back(bp);
-
   utime_t start = ceph_clock_now(NULL);
   utime_t last;
   unsigned ios = 0;
@@ -145,7 +211,7 @@ int do_bench_write(librbd::Image& image, uint64_t io_size,
   uint64_t i;
   uint64_t start_pos;
 
-  // disturb all thread's offset, used by seq write
+  // disturb all thread's offset, used by seq IO
   for (i = 0; i < io_threads; i++) {
     start_pos = (rand() % (size / io_size)) * io_size;
     thread_offset.push_back(start_pos);
@@ -186,7 +252,7 @@ int do_bench_write(librbd::Image& image, uint64_t io_size,
           thread_offset[i] = 0;
       }
 
-      if (!b.start_write(io_threads, thread_offset[i], io_size, bl, op_flags))
+      if (!b.start_io(io_threads, thread_offset[i], io_size, op_flags))
         break;
 
       ++i;
@@ -233,25 +299,42 @@ int do_bench_write(librbd::Image& image, uint64_t io_size,
   return 0;
 }
 
-void get_arguments(po::options_description *positional,
-                   po::options_description *options) {
+void add_bench_common_options(po::options_description *positional,
+			      po::options_description *options) {
   at::add_image_spec_options(positional, options, at::ARGUMENT_MODIFIER_NONE);
-  // TODO
+
   options->add_options()
-    ("io-size", po::value<Size>(), "write size (in B/K/M/G/T)")
+    ("io-size", po::value<Size>(), "IO size (in B/K/M/G/T)")
     ("io-threads", po::value<uint32_t>(), "ios in flight")
-    ("io-total", po::value<Size>(), "total size to write (in B/K/M/G/T)")
-    ("io-pattern", po::value<IOPattern>(), "write pattern (rand or seq)");
+    ("io-total", po::value<Size>(), "total size for IO (in B/K/M/G/T)")
+    ("io-pattern", po::value<IOPattern>(), "IO pattern (rand or seq)");
 }
 
-int execute(const po::variables_map &vm) {
+void get_arguments_for_write(po::options_description *positional,
+                             po::options_description *options) {
+  add_bench_common_options(positional, options);
+}
+
+void get_arguments_for_bench(po::options_description *positional,
+                             po::options_description *options) {
+  add_bench_common_options(positional, options);
+
+  options->add_options()
+    ("io-type", po::value<IOType>()->required(), "IO type (read or write)");
+}
+
+int bench_execute(const po::variables_map &vm, io_type_t bench_io_type) {
   size_t arg_index = 0;
   std::string pool_name;
   std::string image_name;
   std::string snap_name;
+  utils::SnapshotPresence snap_presence = utils::SNAPSHOT_PRESENCE_NONE;
+  if (bench_io_type == IO_TYPE_READ)
+    snap_presence = utils::SNAPSHOT_PRESENCE_PERMITTED;
+
   int r = utils::get_pool_image_snapshot_names(
     vm, at::ARGUMENT_MODIFIER_NONE, &arg_index, &pool_name, &image_name,
-    &snap_name, utils::SNAPSHOT_PRESENCE_NONE, utils::SPEC_VALIDATION_NONE);
+    &snap_name, snap_presence, utils::SPEC_VALIDATION_NONE);
   if (r < 0) {
     return r;
   }
@@ -293,18 +376,39 @@ int execute(const po::variables_map &vm) {
     return r;
   }
 
-  r = do_bench_write(image, bench_io_size, bench_io_threads, bench_bytes,
-                     bench_random);
+  r = do_bench(image, bench_io_type, bench_io_size, bench_io_threads,
+		     bench_bytes, bench_random);
   if (r < 0) {
-    std::cerr << "bench-write failed: " << cpp_strerror(r) << std::endl;
+    std::cerr << "do_bench failed: " << cpp_strerror(r) << std::endl;
     return r;
   }
   return 0;
 }
 
-Shell::Action action(
-  {"bench-write"}, {}, "Simple write benchmark.", "", &get_arguments, &execute, false);
+int execute_for_write(const po::variables_map &vm) {
+  std::cerr << "rbd: bench-write is deprecated, use rbd bench --io-type write ..." << std::endl;
+  return bench_execute(vm, IO_TYPE_WRITE);
+}
+
+int execute_for_bench(const po::variables_map &vm) {
+  io_type_t bench_io_type;
+  if (vm.count("io-type")) {
+    bench_io_type = vm["io-type"].as<io_type_t>();
+  } else {
+    std::cerr << "rbd: --io-type must be specified." << std::endl;
+    return -EINVAL;
+  }
+
+  return bench_execute(vm, bench_io_type);
+}
+
+Shell::Action action_write(
+  {"bench-write"}, {}, "Simple write benchmark. (Deprecated, please use `rbd bench --io-type write` instead.)",
+		   "", &get_arguments_for_write, &execute_for_write, false);
+
+Shell::Action action_bench(
+  {"bench"}, {}, "Simple benchmark.", "", &get_arguments_for_bench, &execute_for_bench);
 
-} // namespace bench_write
+} // namespace bench
 } // namespace action
 } // namespace rbd