Merge pull request ceph#50085 from myoungwon/wip-fix-initialization-v…

…start-rbm

crimson/os/seastore: RBM fixes during vstart startup time 

Reviewed-by: Yingxin Cheng <[email protected]>

src/common/options/crimson.yaml.in

@@ -92,3 +92,13 @@ options:
   level: advanced
   desc: maximum concurrent transactions that seastore allows
   default: 8
+- name: seastore_main_device_type
+  type: str
+  level: dev
+  desc: The main device type seastore uses (SSD or RANDOM_BLOCK_SSD)
+  default: SSD
+- name: seastore_cbjournal_size
+  type: size
+  level: dev
+  desc: Total size to use for CircularBoundedJournal if created, it is valid only if seastore_main_device_type is RANDOM_BLOCK
+  default: 5_G

src/crimson/os/seastore/CMakeLists.txt

@@ -36,7 +36,9 @@ set(crimson_seastore_srcs
   extent_placement_manager.cc
   object_data_handler.cc
   seastore.cc
+  random_block_manager.cc
   random_block_manager/block_rb_manager.cc
+  random_block_manager/rbm_device.cc
   random_block_manager/nvme_block_device.cc
   random_block_manager/avlallocator.cc
   journal/segmented_journal.cc

src/crimson/os/seastore/device.cc

@@ -4,6 +4,8 @@
 #include "device.h"
 
 #include "segment_manager.h"
+#include "random_block_manager.h"
+#include "random_block_manager/rbm_device.h"
 
 namespace crimson::os::seastore {
 
@@ -33,9 +35,14 @@ std::ostream& operator<<(std::ostream& out, const device_config_t& conf)
 seastar::future<DeviceRef>
 Device::make_device(const std::string& device, device_type_t dtype)
 {
-  // TODO: support other backend types
-  assert(get_default_backend_of_device(dtype) == backend_type_t::SEGMENTED);
-  return SegmentManager::get_segment_manager(device
+  if (get_default_backend_of_device(dtype) == backend_type_t::SEGMENTED) {
+    return SegmentManager::get_segment_manager(device
+    ).then([](DeviceRef ret) {
+      return ret;
+    });
+  } 
+  assert(get_default_backend_of_device(dtype) == backend_type_t::RANDOM_BLOCK);
+  return get_rb_device(device
   ).then([](DeviceRef ret) {
     return ret;
   });

src/crimson/os/seastore/journal/circular_bounded_journal.cc

@@ -502,19 +502,21 @@ CircularBoundedJournal::write_ertr::future<>
 CircularBoundedJournal::write_header()
 {
   LOG_PREFIX(CircularBoundedJournal::write_header);
-  ceph::bufferlist bl;
-  try {
-    bl = encode_header();
-  } catch (ceph::buffer::error &e) {
-    DEBUG("unable to encode header block from underlying deivce");
-    return crimson::ct_error::input_output_error::make();
-  }
+  ceph::bufferlist bl = encode_header();
   ceph_assert(bl.length() <= get_block_size());
   DEBUG(
     "sync header of CircularBoundedJournal, length {}",
     bl.length());
   assert(device);
-  return device_write_bl(device->get_journal_start(), bl);
+  auto iter = bl.begin();
+  assert(bl.length() < get_block_size());
+  bufferptr bp = bufferptr(ceph::buffer::create_page_aligned(get_block_size()));
+  iter.copy(bl.length(), bp.c_str());
+  return device->write(device->get_journal_start(), std::move(bp)
+  ).handle_error(
+    write_ertr::pass_further{},
+    crimson::ct_error::assert_all{ "Invalid error device->write" }
+  );
 }
 seastar::future<> CircularBoundedJournal::finish_commit(transaction_type_t type) {
   if (is_trim_transaction(type)) {

src/crimson/os/seastore/journal/circular_bounded_journal.h

@@ -51,7 +51,6 @@ using RBMDevice = random_block_device::RBMDevice;
  *
  */
 
-constexpr uint64_t DEFAULT_TEST_CBJOURNAL_SIZE = 1 << 26;
 constexpr uint64_t DEFAULT_BLOCK_SIZE = 4096;
 
 class CircularBoundedJournal : public Journal {

src/crimson/os/seastore/random_block_manager.cc

@@ -0,0 +1,21 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "crimson/os/seastore/logging.h"
+#include "crimson/os/seastore/random_block_manager.h"
+#include "crimson/os/seastore/random_block_manager/nvme_block_device.h"
+#include "crimson/os/seastore/random_block_manager/rbm_device.h"
+
+namespace crimson::os::seastore {
+
+seastar::future<random_block_device::RBMDeviceRef>
+get_rb_device(
+  const std::string &device)
+{
+  return seastar::make_ready_future<random_block_device::RBMDeviceRef>(
+    std::make_unique<
+      random_block_device::nvme::NVMeBlockDevice
+    >(device + "/block"));
+}
+
+}

src/crimson/os/seastore/random_block_manager.h

@@ -117,6 +117,14 @@ inline rbm_abs_addr convert_paddr_to_abs_addr(const paddr_t& paddr) {
 inline paddr_t convert_abs_addr_to_paddr(rbm_abs_addr addr, device_id_t d_id) {
   return paddr_t::make_blk_paddr(d_id, addr);
 }
+
+namespace random_block_device {
+  class RBMDevice;
+}
+
+seastar::future<std::unique_ptr<random_block_device::RBMDevice>> 
+  get_rb_device(const std::string &device);
+
 std::ostream &operator<<(std::ostream &out, const rbm_metadata_header_t &header);
 }
 

src/crimson/os/seastore/random_block_manager/block_rb_manager.cc

@@ -73,22 +73,15 @@ void BlockRBManager::complete_allocation(
 BlockRBManager::open_ertr::future<> BlockRBManager::open()
 {
   assert(device);
-  return device->read_rbm_header(RBM_START_ADDRESS
-  ).safe_then([this](auto s)
-    -> open_ertr::future<> {
-    auto ool_start = get_start_rbm_addr();
-    allocator->init(
-      ool_start,
-      device->get_available_size() -
-      ool_start,
-      device->get_block_size());
-    return open_ertr::now();
-  }).handle_error(
-    open_ertr::pass_further{},
-    crimson::ct_error::assert_all{
-      "Invalid error read_rbm_header in BlockRBManager::open"
-    }
-  );
+  assert(device->get_available_size() > 0);
+  assert(device->get_block_size() > 0);
+  auto ool_start = get_start_rbm_addr();
+  allocator->init(
+    ool_start,
+    device->get_available_size() -
+    ool_start,
+    device->get_block_size());
+  return open_ertr::now();
 }
 
 BlockRBManager::write_ertr::future<> BlockRBManager::write(
@@ -105,9 +98,11 @@ BlockRBManager::write_ertr::future<> BlockRBManager::write(
       start, end, addr, bptr.length());
     return crimson::ct_error::erange::make();
   }
+  bufferptr bp = bufferptr(ceph::buffer::create_page_aligned(bptr.length()));
+  bp.copy_in(0, bptr.length(), bptr.c_str());
   return device->write(
     addr,
-    bptr);
+    std::move(bp));
 }
 
 BlockRBManager::read_ertr::future<> BlockRBManager::read(
@@ -153,7 +148,7 @@ BlockRBManager::write_ertr::future<> BlockRBManager::write(
   }
   return device->write(
     addr,
-    bptr);
+    std::move(bptr));
 }
 
 std::ostream &operator<<(std::ostream &out, const rbm_metadata_header_t &header)