test/bench: add rbd backend to smalliobench

Only supports format 1 images to start, and does not issue flushes, so
it's best used with caching off.

Signed-off-by: Josh Durgin <[email protected]>

.gitignore

@@ -62,10 +62,6 @@ src/ocf/rbd
 src/omapbench
 src/kvstorebench
 ar-lib
-src/smalliobench
-src/smalliobenchdumb
-src/smalliobenchfs
-src/tpbench
 
 # temporary directory used by e.g. "make distcheck", e.g. ceph-0.42
 /ceph-[0-9]*/

src/.gitignore

@@ -111,3 +111,8 @@ Makefile
 /test_librbd_fsx
 /scratchtool
 /scratchtoolpp
+/smalliobench
+/smalliobenchdumb
+/smalliobenchfs
+/tpbench
+/smalliobenchrbd

src/Makefile.am

@@ -236,6 +236,10 @@ smalliobenchdumb_LDADD = librados.la -lboost_program_options $(LIBOS_LDA) $(LIBG
 smalliobenchdumb_CXXFLAGS = ${CRYPTO_CXXFLAGS} ${AM_CXXFLAGS}
 bin_DEBUGPROGRAMS += smalliobenchdumb
 
+smalliobenchrbd_SOURCES = test/bench/small_io_bench_rbd.cc test/bench/rbd_backend.cc test/bench/detailed_stat_collector.cc test/bench/bencher.cc
+smalliobenchrbd_LDADD = librados.la librbd.la -lboost_program_options $(LIBGLOBAL_LDA)
+bin_DEBUGPROGRAMS += smalliobenchrbd
+
 tpbench_SOURCES = test/bench/tp_bench.cc test/bench/detailed_stat_collector.cc
 tpbench_LDADD = librados.la -lboost_program_options $(LIBOS_LDA) $(LIBGLOBAL_LDA)
 bin_DEBUGPROGRAMS += tpbench
@@ -1889,6 +1893,7 @@ noinst_HEADERS = \
 	common/sync_filesystem.h \
         test/bench/distribution.h \
         test/bench/rados_backend.h \
+        test/bench/rbd_backend.h \
         test/bench/bencher.h \
         test/bench/backend.h \
         test/bench/dumb_backend.h \

src/test/bench/rbd_backend.cc

@@ -0,0 +1,51 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+
+#include "rbd_backend.h"
+#include <boost/tuple/tuple.hpp>
+
+typedef boost::tuple<Context*, Context*> arg_type;
+
+void on_complete(void *completion, void *_arg) {
+  arg_type *arg = static_cast<arg_type*>(_arg);
+  librbd::RBD::AioCompletion *comp =
+    static_cast<librbd::RBD::AioCompletion *>(completion);
+  ssize_t r = comp->get_return_value();
+  assert(r >= 0);
+  arg->get<0>()->complete(0);
+  if (arg->get<1>())
+    arg->get<1>()->complete(0);
+  comp->release();
+  delete arg;
+}
+
+void RBDBackend::write(
+  const string &oid,
+  uint64_t offset,
+  const bufferlist &bl,
+  Context *on_write_applied,
+  Context *on_commit)
+{
+  bufferlist &bl_non_const = const_cast<bufferlist&>(bl);
+  std::tr1::shared_ptr<librbd::Image> image = (*m_images)[oid];
+  void *arg = static_cast<void *>(new arg_type(on_commit, on_write_applied));
+  librbd::RBD::AioCompletion *completion =
+    new librbd::RBD::AioCompletion(arg, on_complete);
+  int r = image->aio_write(offset, (size_t) bl_non_const.length(),
+			   bl_non_const, completion);
+  assert(r >= 0);
+}
+
+void RBDBackend::read(
+  const string &oid,
+  uint64_t offset,
+  uint64_t length,
+  bufferlist *bl,
+  Context *on_read_complete)
+{
+  std::tr1::shared_ptr<librbd::Image> image = (*m_images)[oid];
+  void *arg = static_cast<void *>(new arg_type(on_read_complete, NULL));
+  librbd::RBD::AioCompletion *completion =
+    new librbd::RBD::AioCompletion(arg, on_complete);
+  int r = image->aio_read(offset, (size_t) length, *bl, completion);
+  assert(r >= 0);
+}

src/test/bench/rbd_backend.h

@@ -0,0 +1,30 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+
+#ifndef CEPH_TEST_SMALLIOBENCH_RBD_BACKEND_H
+#define CEPH_TEST_SMALLIOBENCH_RBD_BACKEND_H
+
+#include "backend.h"
+#include "include/Context.h"
+#include "include/rbd/librbd.hpp"
+
+class RBDBackend : public Backend {
+  map<string, std::tr1::shared_ptr<librbd::Image> > *m_images;
+public:
+  RBDBackend(map<string, std::tr1::shared_ptr<librbd::Image> > *images)
+    : m_images(images) {}
+  void write(
+    const string &oid,
+    uint64_t offset,
+    const bufferlist &bl,
+    Context *on_applied,
+    Context *on_commit);
+
+  void read(
+    const string &oid,
+    uint64_t offset,
+    uint64_t length,
+    bufferlist *bl,
+    Context *on_complete);
+};
+
+#endif

src/test/bench/small_io_bench_rbd.cc

@@ -0,0 +1,201 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+
+#include <boost/lexical_cast.hpp>
+#include <boost/program_options/option.hpp>
+#include <boost/program_options/options_description.hpp>
+#include <boost/program_options/variables_map.hpp>
+#include <boost/program_options/cmdline.hpp>
+#include <boost/program_options/parsers.hpp>
+#include <iostream>
+#include <set>
+#include <sstream>
+#include <stdlib.h>
+#include <fstream>
+#include <iostream>
+
+#include "common/Formatter.h"
+
+#include "bencher.h"
+#include "rbd_backend.h"
+#include "detailed_stat_collector.h"
+#include "distribution.h"
+
+namespace po = boost::program_options;
+using namespace std;
+
+int main(int argc, char **argv)
+{
+  po::options_description desc("Allowed options");
+  desc.add_options()
+    ("help", "produce help message")
+    ("num-concurrent-ops", po::value<unsigned>()->default_value(10),
+     "set number of concurrent ops")
+    ("num-images", po::value<unsigned>()->default_value(2),
+     "set number of rbd images to use")
+    ("image-size", po::value<unsigned>()->default_value(4096),
+     "set image size in megabytes")
+    ("order", po::value<unsigned>()->default_value(22),
+     "set log_2(object size)")
+    ("io-size", po::value<unsigned>()->default_value(4<<10),
+     "set io size")
+    ("write-ratio", po::value<double>()->default_value(0.25),
+     "set ratio of read to write")
+    ("duration", po::value<unsigned>()->default_value(0),
+     "set max duration, 0 for unlimited")
+    ("max-ops", po::value<unsigned>()->default_value(0),
+     "set max ops, 0 for unlimited")
+    ("seed", po::value<unsigned>(),
+     "seed")
+    ("ceph-client-id", po::value<string>()->default_value("admin"),
+     "set ceph client id")
+    ("pool-name", po::value<string>()->default_value("data"),
+     "set pool")
+    ("op-dump-file", po::value<string>()->default_value(""),
+     "set file for dumping op details, omit for stderr")
+    ("offset-align", po::value<unsigned>()->default_value(4096),
+     "align offset by")
+    ("sequential", po::value<bool>()->default_value(false),
+     "use sequential access pattern")
+    ("disable-detailed-ops", po::value<bool>()->default_value(false),
+     "don't dump per op stats")
+    ;
+
+  po::variables_map vm;
+  po::store(po::parse_command_line(argc, argv, desc), vm);
+  po::notify(vm);
+
+  if (vm.count("help")) {
+    cout << desc << std::endl;
+    return 1;
+  }
+
+  string prefix;
+  char hostname_cstr[100];
+  gethostname(hostname_cstr, 100);
+  stringstream hostpid;
+  hostpid << hostname_cstr << getpid() << "-";
+  prefix = hostpid.str();
+
+  set<string> image_names;
+  for (unsigned i = 0; i < vm["num-images"].as<unsigned>();
+       ++i) {
+    stringstream name;
+    name << prefix << "-image_" << i;
+    image_names.insert(name.str());
+  }
+
+  rngen_t rng;
+  if (vm.count("seed"))
+    rng = rngen_t(vm["seed"].as<unsigned>());
+
+  set<pair<double, Bencher::OpType> > ops;
+  ops.insert(make_pair(vm["write-ratio"].as<double>(), Bencher::WRITE));
+  ops.insert(make_pair(1-vm["write-ratio"].as<double>(), Bencher::READ));
+
+  librados::Rados rados;
+  librados::IoCtx ioctx;
+  int r = rados.init(vm["ceph-client-id"].as<string>().c_str());
+  if (r < 0) {
+    cerr << "error in init r=" << r << std::endl;
+    return -r;
+  }
+  r = rados.conf_read_file(NULL);
+  if (r < 0) {
+    cerr << "error in conf_read_file r=" << r << std::endl;
+    return -r;
+  }
+  r = rados.conf_parse_env(NULL);
+  if (r < 0) {
+    cerr << "error in conf_parse_env r=" << r << std::endl;
+    return -r;
+  }
+  r = rados.connect();
+  if (r < 0) {
+    cerr << "error in connect r=" << r << std::endl;
+    return -r;
+  }
+  r = rados.ioctx_create(vm["pool-name"].as<string>().c_str(), ioctx);
+  if (r < 0) {
+    cerr << "error in ioctx_create r=" << r << std::endl;
+    return -r;
+  }
+
+  ostream *detailed_ops = 0;
+  ofstream myfile;
+  if (vm["disable-detailed-ops"].as<bool>()) {
+    detailed_ops = 0;
+  } else if (vm["op-dump-file"].as<string>().size()) {
+    myfile.open(vm["op-dump-file"].as<string>().c_str());
+    detailed_ops = &myfile;
+  } else {
+    detailed_ops = &cerr;
+  }
+
+  librbd::RBD rbd;
+  {
+    map<string, std::tr1::shared_ptr<librbd::Image> > images;
+    int order = vm["order"].as<unsigned>();
+    uint64_t image_size = ((uint64_t)vm["image-size"].as<unsigned>()) << 20;
+    for (set<string>::const_iterator i = image_names.begin();
+	 i != image_names.end(); ++i) {
+      r = rbd.create(ioctx, i->c_str(), image_size, &order);
+      if (r < 0) {
+	cerr << "error creating image " << *i << " r=" << r << std::endl;
+	return -r;
+      }
+      std::tr1::shared_ptr<librbd::Image> image(new librbd::Image());
+      r = rbd.open(ioctx, *image, i->c_str());
+      if (r < 0) {
+	cerr << "error opening image " << *i << " r=" << r << std::endl;
+	return -r;
+      }
+      images[*i] = image;
+    }
+
+    Distribution<
+      boost::tuple<string, uint64_t, uint64_t, Bencher::OpType> > *gen = 0;
+    if (vm["sequential"].as<bool>()) {
+      std::cout << "Using Sequential generator" << std::endl;
+      gen = new SequentialLoad(
+	image_names,
+	image_size,
+	vm["io-size"].as<unsigned>(),
+	new WeightedDist<Bencher::OpType>(rng, ops)
+	);
+    } else {
+      std::cout << "Using random generator" << std::endl;
+      gen = new FourTupleDist<string, uint64_t, uint64_t, Bencher::OpType>(
+	new RandomDist<string>(rng, image_names),
+	new Align(
+	  new UniformRandom(
+	    rng,
+	    0,
+	    image_size - vm["io-size"].as<unsigned>()),
+	  vm["offset-align"].as<unsigned>()
+	  ),
+	new Uniform(vm["io-size"].as<unsigned>()),
+	new WeightedDist<Bencher::OpType>(rng, ops)
+        );
+    }
+
+    Bencher bencher(
+      gen,
+      new DetailedStatCollector(1, new JSONFormatter, detailed_ops, &cout),
+      new RBDBackend(&images),
+      vm["num-concurrent-ops"].as<unsigned>(),
+      vm["duration"].as<unsigned>(),
+      vm["max-ops"].as<unsigned>());
+
+    bencher.run_bench();
+  }
+
+  for (set<string>::const_iterator i = image_names.begin();
+       i != image_names.end(); ++i) {
+    rbd.remove(ioctx, i->c_str());
+  }
+  rados.shutdown();
+  if (vm["op-dump-file"].as<string>().size()) {
+    myfile.close();
+  }
+  return 0;
+}