src: extract backend driver from bluestore to access device

1. Both bluestore or other component e.g. rbd could use the same driver to access storage device. It's better to use one library to integrate the driver. 2. os and crimson-alienstore are static libraries. Link blk library into them. Main changes are below: 1. move backend driver into src/blk src/$ mkdir -p blk/{aio,kernel,spdk,pmem,zns} src/$ mv os/bluestore/{BlockDevice.h,BlockDevice.cc} blk src/$ mv os/bluestore/{ceph_aio.h} aio/aio.h src/$ mv os/bluestore/{aio.cc} aio/ src/$ mv os/bluestore/{KernelDevice.h,KernelDevice.cc} kernel/ src/$ mv os/bluestore/{ceph_io_uring.h} kernel/io_uring.h src/$ mv os/bluestore/{io_uring.cc} kernel_drv/ src/$ mv os/bluestore/{NVMEDevice.h,NVMEDevice.cc} spdk/ src/$ mv os/bluestore/{PMEMDevice.h,PMEMDevice.cc} pmem/ src/$ mv os/bluestore/{HMSMRDevice.h,HMSMRDevice.cc} zns/ 2. customize macro name in header file to remove bluestore specific text 3. adjust header file patch in source code 4. create cmake rule blk/CMakeLists.txt to build blk 5. modify src/CMakeLists.txt to integrate blk 6. modify other CMakeLists.txt to adapt to new file structure. Signed-off-by: Changcheng Liu <[email protected]> Co-authored-by: Kefu Chai <[email protected]>
Conan-Kudo · May 28, 2020 · b734735 · b734735
1 parent b63ae14
commit b734735
Show file tree

Hide file tree

Showing 23 changed files with 92 additions and 92 deletions.
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
@@ -544,6 +544,7 @@ endif()
 
 add_subdirectory(kv)
 add_subdirectory(os)
+add_subdirectory(blk)
 
 add_subdirectory(osd)
 

diff --git a/src/os/bluestore/BlockDevice.cc → src/blk/BlockDevice.cc b/src/os/bluestore/BlockDevice.cc → src/blk/BlockDevice.cc
@@ -20,20 +20,20 @@
 #include "BlockDevice.h"
 
 #if defined(HAVE_LIBAIO) || defined(HAVE_POSIXAIO)
-#include "KernelDevice.h"
+#include "kernel/KernelDevice.h"
 #endif
 
 #if defined(HAVE_SPDK)
-#include "NVMEDevice.h"
+#include "spdk/NVMEDevice.h"
 #endif
 
 #if defined(HAVE_BLUESTORE_PMEM)
-#include "PMEMDevice.h"
+#include "pmem/PMEMDevice.h"
 #include "libpmem.h"
 #endif
 
 #if defined(HAVE_LIBZBC)
-#include "HMSMRDevice.h"
+#include "zns/HMSMRDevice.h"
 extern "C" {
 #include <libzbc/zbc.h>
 }

diff --git a/src/os/bluestore/BlockDevice.h → src/blk/BlockDevice.h b/src/os/bluestore/BlockDevice.h → src/blk/BlockDevice.h
@@ -14,8 +14,8 @@
  *
  */
 
-#ifndef CEPH_OS_BLUESTORE_BLOCKDEVICE_H
-#define CEPH_OS_BLUESTORE_BLOCKDEVICE_H
+#ifndef CEPH_BLK_BLOCKDEVICE_H
+#define CEPH_BLK_BLOCKDEVICE_H
 
 #include <atomic>
 #include <condition_variable>
@@ -31,7 +31,7 @@
 #include "include/common_fwd.h"
 
 #if defined(HAVE_LIBAIO) || defined(HAVE_POSIXAIO)
-#include "ceph_aio.h"
+#include "aio/aio.h"
 #endif
 #include "include/ceph_assert.h"
 #include "include/buffer.h"
@@ -261,4 +261,4 @@ class BlockDevice {
   }
 };
 
-#endif //CEPH_OS_BLUESTORE_BLOCKDEVICE_H
+#endif //CEPH_BLK_BLOCKDEVICE_H
diff --git a/src/blk/CMakeLists.txt b/src/blk/CMakeLists.txt
@@ -0,0 +1,61 @@
+if(WITH_BLUESTORE OR WITH_RBD_RWL)
+list(APPEND libblk_srcs
+  BlockDevice.cc)
+endif()
+
+if(HAVE_LIBAIO OR HAVE_POSIXAIO)
+  list(APPEND libblk_srcs
+    kernel/KernelDevice.cc
+    kernel/io_uring.cc
+    aio/aio.cc)
+endif()
+
+if(WITH_BLUESTORE_PMEM)
+  list(APPEND libblk_srcs
+    pmem/PMEMDevice.cc)
+endif()
+
+if(WITH_SPDK)
+  list(APPEND libblk_srcs
+    spdk/NVMEDevice.cc)
+endif()
+
+if(HAVE_LIBZBC)
+  list(APPEND libblk_srcs
+    zns/HMSMRDevice.cc)
+endif()
+
+add_library(blk ${libblk_srcs})
+target_include_directories(blk PRIVATE "./")
+
+if(HAVE_LIBAIO)
+  target_link_libraries(blk PUBLIC ${AIO_LIBRARIES})
+endif(HAVE_LIBAIO)
+
+if(WITH_SPDK)
+  target_link_libraries(blk PRIVATE ${SPDK_LIBRARIES})
+endif()
+
+if(HAVE_LIBZBC)
+  target_link_libraries(blk ${ZBC_LIBRARIES})
+endif()
+
+if(WITH_BLUESTORE_PMEM OR WITH_RBD_RWL)
+  target_link_libraries(blk
+    PUBLIC pmem::pmemobj
+    PRIVATE pmem::pmem)
+endif()
+
+if(WITH_EVENTTRACE)
+  add_dependencies(blk eventtrace_tp)
+endif()
+
+if(WITH_LIBURING)
+  if(WITH_SYSTEM_LIBURING)
+    find_package(uring REQUIRED)
+  else()
+    include(Builduring)
+    build_uring()
+  endif()
+  target_link_libraries(blk PRIVATE uring::uring)
+endif()
diff --git a/src/os/bluestore/aio.cc → src/blk/aio/aio.cc b/src/os/bluestore/aio.cc → src/blk/aio/aio.cc
@@ -2,7 +2,7 @@
 // vim: ts=8 sw=2 smarttab
 
 #include <algorithm>
-#include "ceph_aio.h"
+#include "aio.h"
 
 std::ostream& operator<<(std::ostream& os, const aio_t& aio)
 {

diff --git a/src/os/bluestore/ceph_aio.h → src/blk/aio/aio.h b/src/os/bluestore/ceph_aio.h → src/blk/aio/aio.h
diff --git a/src/os/bluestore/KernelDevice.cc → src/blk/kernel/KernelDevice.cc b/src/os/bluestore/KernelDevice.cc → src/blk/kernel/KernelDevice.cc
@@ -33,7 +33,7 @@
 #include "common/numa.h"
 
 #include "global/global_context.h"
-#include "ceph_io_uring.h"
+#include "io_uring.h"
 
 #define dout_context cct
 #define dout_subsys ceph_subsys_bdev

diff --git a/src/os/bluestore/KernelDevice.h → src/blk/kernel/KernelDevice.h b/src/os/bluestore/KernelDevice.h → src/blk/kernel/KernelDevice.h
@@ -12,8 +12,8 @@
  *
  */
 
-#ifndef CEPH_OS_BLUESTORE_KERNELDEVICE_H
-#define CEPH_OS_BLUESTORE_KERNELDEVICE_H
+#ifndef CEPH_BLK_KERNELDEVICE_H
+#define CEPH_BLK_KERNELDEVICE_H
 
 #include <atomic>
 
@@ -22,7 +22,7 @@
 #include "common/Thread.h"
 #include "include/utime.h"
 
-#include "ceph_aio.h"
+#include "aio/aio.h"
 #include "BlockDevice.h"
 
 #define RW_IO_MAX (INT_MAX & CEPH_PAGE_MASK)

diff --git a/src/os/bluestore/io_uring.cc → src/blk/kernel/io_uring.cc b/src/os/bluestore/io_uring.cc → src/blk/kernel/io_uring.cc
@@ -1,7 +1,7 @@
 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
 // vim: ts=8 sw=2 smarttab
 
-#include "ceph_io_uring.h"
+#include "io_uring.h"
 
 #if defined(HAVE_LIBURING) && defined(__x86_64__)
 

diff --git a/src/os/bluestore/ceph_io_uring.h → src/blk/kernel/io_uring.h b/src/os/bluestore/ceph_io_uring.h → src/blk/kernel/io_uring.h
@@ -6,7 +6,7 @@
 #include "acconfig.h"
 
 #include "include/types.h"
-#include "ceph_aio.h"
+#include "aio/aio.h"
 
 struct ioring_data;
 

diff --git a/src/os/bluestore/PMEMDevice.cc → src/blk/pmem/PMEMDevice.cc b/src/os/bluestore/PMEMDevice.cc → src/blk/pmem/PMEMDevice.cc
diff --git a/src/os/bluestore/PMEMDevice.h → src/blk/pmem/PMEMDevice.h b/src/os/bluestore/PMEMDevice.h → src/blk/pmem/PMEMDevice.h
@@ -14,14 +14,14 @@
  *
  */
 
-#ifndef CEPH_OS_BLUESTORE_PMEMDEVICE_H
-#define CEPH_OS_BLUESTORE_PMEMDEVICE_H
+#ifndef CEPH_BLK_PMEMDEVICE_H
+#define CEPH_BLK_PMEMDEVICE_H
 
 #include <atomic>
 
 #include "os/fs/FS.h"
 #include "include/interval_set.h"
-#include "ceph_aio.h"
+#include "aio/aio.h"
 #include "BlockDevice.h"
 
 class PMEMDevice : public BlockDevice {

diff --git a/src/os/bluestore/NVMEDevice.cc → src/blk/spdk/NVMEDevice.cc b/src/os/bluestore/NVMEDevice.cc → src/blk/spdk/NVMEDevice.cc
diff --git a/src/os/bluestore/NVMEDevice.h → src/blk/spdk/NVMEDevice.h b/src/os/bluestore/NVMEDevice.h → src/blk/spdk/NVMEDevice.h
@@ -14,8 +14,8 @@
  *
  */
 
-#ifndef CEPH_OS_BLUESTORE_NVMEDEVICE
-#define CEPH_OS_BLUESTORE_NVMEDEVICE
+#ifndef CEPH_BLK_NVMEDEVICE
+#define CEPH_BLK_NVMEDEVICE
 
 #include <queue>
 #include <map>

diff --git a/src/os/bluestore/HMSMRDevice.cc → src/blk/zns/HMSMRDevice.cc b/src/os/bluestore/HMSMRDevice.cc → src/blk/zns/HMSMRDevice.cc
@@ -35,7 +35,7 @@
 #include "common/numa.h"
 
 #include "global/global_context.h"
-#include "ceph_io_uring.h"
+#include "kernel/io_uring.h"
 
 extern "C" {
 #include <libzbc/zbc.h>

diff --git a/src/os/bluestore/HMSMRDevice.h → src/blk/zns/HMSMRDevice.h b/src/os/bluestore/HMSMRDevice.h → src/blk/zns/HMSMRDevice.h
@@ -16,8 +16,8 @@
 // Copied from KernelDevice with HM-SMR specific functionality added.  Will be
 // further specialized for HM-SMR.
 
-#ifndef CEPH_OS_BLUESTORE_HMSMRDEVICE_H
-#define CEPH_OS_BLUESTORE_HMSMRDEVICE_H
+#ifndef CEPH_BLK_HMSMRDEVICE_H
+#define CEPH_BLK_HMSMRDEVICE_H
 
 #include <atomic>
 
@@ -26,7 +26,7 @@
 #include "common/Thread.h"
 #include "include/utime.h"
 
-#include "ceph_aio.h"
+#include "aio/aio.h"
 #include "BlockDevice.h"
 
 #define RW_IO_MAX (INT_MAX & CEPH_PAGE_MASK)
@@ -159,4 +159,4 @@ class HMSMRDevice final : public BlockDevice {
   void close() final;
 };
 
-#endif
+#endif //CEPH_BLK_HMSMRDEVICE_H
diff --git a/src/crimson/os/alienstore/CMakeLists.txt b/src/crimson/os/alienstore/CMakeLists.txt
@@ -28,7 +28,6 @@ list(APPEND crimson_alien_srcs
   ${PROJECT_SOURCE_DIR}/src/os/bluestore/Allocator.cc
   ${PROJECT_SOURCE_DIR}/src/os/bluestore/AvlAllocator.cc
   ${PROJECT_SOURCE_DIR}/src/os/bluestore/BitmapFreelistManager.cc
-  ${PROJECT_SOURCE_DIR}/src/os/bluestore/BlockDevice.cc
   ${PROJECT_SOURCE_DIR}/src/os/bluestore/BlueFS.cc
   ${PROJECT_SOURCE_DIR}/src/os/bluestore/bluefs_types.cc
   ${PROJECT_SOURCE_DIR}/src/os/bluestore/BlueRocksEnv.cc
@@ -37,23 +36,13 @@ list(APPEND crimson_alien_srcs
   ${PROJECT_SOURCE_DIR}/src/os/bluestore/fastbmap_allocator_impl.cc
   ${PROJECT_SOURCE_DIR}/src/os/bluestore/FreelistManager.cc
   ${PROJECT_SOURCE_DIR}/src/os/bluestore/HybridAllocator.cc
-  ${PROJECT_SOURCE_DIR}/src/os/bluestore/io_uring.cc
   ${PROJECT_SOURCE_DIR}/src/os/bluestore/StupidAllocator.cc
   ${PROJECT_SOURCE_DIR}/src/os/bluestore/BitmapAllocator.cc)
 
-if(HAVE_LIBAIO OR HAVE_POSIXAIO)
-  list(APPEND crimson_alien_srcs
-     ${PROJECT_SOURCE_DIR}/src/os/bluestore/KernelDevice.cc
-     ${PROJECT_SOURCE_DIR}/src/os/bluestore/aio.cc)
-endif()
-
 add_library(crimson-alienstore STATIC ${crimson_alien_srcs}
   $<TARGET_OBJECTS:compressor_objs>
   $<TARGET_OBJECTS:crush_objs>
   $<TARGET_OBJECTS:common_prioritycache_obj>)
-if(HAVE_LIBAIO)
-  target_link_libraries(crimson-alienstore ${AIO_LIBRARIES})
-endif(HAVE_LIBAIO)
 
 target_compile_definitions(crimson-alienstore PRIVATE -DWITH_SEASTAR -DWITH_ALIEN)
 target_include_directories(crimson-alienstore PRIVATE
@@ -64,3 +53,4 @@ target_link_libraries(crimson-alienstore heap_profiler)
 target_link_libraries(crimson-alienstore ${BLKID_LIBRARIES})
 target_link_libraries(crimson-alienstore ${UDEV_LIBRARIES})
 target_link_libraries(crimson-alienstore crimson)
+target_link_libraries(crimson-alienstore blk)
diff --git a/src/librbd/CMakeLists.txt b/src/librbd/CMakeLists.txt
@@ -217,8 +217,7 @@ target_link_libraries(rbd_internal PRIVATE
 
 if(WITH_RBD_RWL)
   target_link_libraries(rbd_internal
-    PUBLIC pmem::pmemobj
-    PRIVATE pmem::pmem)
+    PUBLIC blk)
 endif()
 
 add_library(librbd ${CEPH_SHARED}

diff --git a/src/os/CMakeLists.txt b/src/os/CMakeLists.txt
@@ -23,7 +23,6 @@ if(WITH_BLUESTORE)
   list(APPEND libos_srcs
     bluestore/Allocator.cc
     bluestore/BitmapFreelistManager.cc
-    bluestore/BlockDevice.cc
     bluestore/BlueFS.cc
     bluestore/bluefs_types.cc
     bluestore/BlueRocksEnv.cc
@@ -35,19 +34,11 @@ if(WITH_BLUESTORE)
     bluestore/BitmapAllocator.cc
     bluestore/AvlAllocator.cc
     bluestore/HybridAllocator.cc
-    bluestore/io_uring.cc
   )
 endif(WITH_BLUESTORE)
 
-if(HAVE_LIBAIO OR HAVE_POSIXAIO)
-  list(APPEND libos_srcs
-    bluestore/KernelDevice.cc
-    bluestore/aio.cc)
-endif()
-
 if(HAVE_LIBZBC)
   list(APPEND libos_srcs
-    bluestore/HMSMRDevice.cc
     bluestore/ZonedAllocator.cc)
 endif()
 
@@ -56,11 +47,6 @@ if(WITH_FUSE)
     FuseStore.cc)
 endif(WITH_FUSE)
 
-if(WITH_BLUESTORE_PMEM)
-  list(APPEND libos_srcs
-    bluestore/PMEMDevice.cc)
-endif()
-
 if(HAVE_LIBXFS)
   list(APPEND libos_srcs
     filestore/XfsFileStoreBackend.cc
@@ -76,12 +62,8 @@ if(HAVE_LIBZFS)
   list(APPEND libos_srcs $<TARGET_OBJECTS:os_zfs_objs>)
 endif()
 
-if(WITH_SPDK)
-  list(APPEND libos_srcs
-    bluestore/NVMEDevice.cc)
-endif()
-
 add_library(os STATIC ${libos_srcs})
+target_link_libraries(os blk)
 
 target_link_libraries(os heap_profiler kv)
 
@@ -94,14 +76,6 @@ if(WITH_BLUEFS)
   install(TARGETS bluefs DESTINATION lib)
 endif(WITH_BLUEFS)
 
-if(HAVE_LIBAIO)
-  target_link_libraries(os ${AIO_LIBRARIES})
-endif(HAVE_LIBAIO)
-
-if(HAVE_LIBZBC)
-  target_link_libraries(os ${ZBC_LIBRARIES})
-endif()
-
 if(WITH_FUSE)
   target_link_libraries(os FUSE::FUSE)
 endif()
@@ -110,11 +84,6 @@ if(HAVE_LIBZFS)
   target_link_libraries(os ${ZFS_LIBRARIES})
 endif()
 
-if(WITH_SPDK)
-  target_link_libraries(os
-    ${SPDK_LIBRARIES})
-endif()
-
 if(WITH_LTTNG)
   add_dependencies(os objectstore-tp)
   add_dependencies(os bluestore-tp)
@@ -134,21 +103,3 @@ if(WITH_BLUESTORE)
   install(TARGETS ceph-bluestore-tool
     DESTINATION bin)
 endif()
-
-if(WITH_BLUESTORE_PMEM)
-  target_link_libraries(os pmem::pmem)
-endif()
-
-if(WITH_EVENTTRACE)
-  add_dependencies(os eventtrace_tp)
-endif()
-
-if(WITH_LIBURING)
-  if(WITH_SYSTEM_LIBURING)
-    find_package(uring REQUIRED)
-  else()
-    include(Builduring)
-    build_uring()
-  endif()
-  target_link_libraries(os uring::uring)
-endif()
diff --git a/src/os/bluestore/BlueFS.cc b/src/os/bluestore/BlueFS.cc
@@ -8,7 +8,6 @@
 #include "common/debug.h"
 #include "common/errno.h"
 #include "common/perf_counters.h"
-#include "BlockDevice.h"
 #include "Allocator.h"
 #include "include/ceph_assert.h"
 #include "common/admin_socket.h"