curve: support block storage open multiple times

support block storage open multiple times e.g., a single file/image can be mapped multiple times on different hosts by curve-nbd with extra option `--no-exclusive' limitations: 1. provided as a shared disk/device WITHOUT data consistent guarantee, so it's users' responsibility to deal with this problem. 2. each file/image can only open once on each host.
wangxiaodong1 · Dec 23, 2021 · 5f5220c · 5f5220c
1 parent 32ee88c
commit 5f5220c
Show file tree

Hide file tree

Showing 63 changed files with 743 additions and 381 deletions.
diff --git a/curve-ansible/roles/install_package/tasks/include/install_nebd.yml b/curve-ansible/roles/install_package/tasks/include/install_nebd.yml
@@ -64,6 +64,12 @@
         local_file_path: "{{ local_nebd_package_path }}/bin/"
         file_mode: 0755
       include_tasks: copy_file_to_remote.yml
+    - name: install nebd include
+      vars:
+        remote_dir_name: "{{ curve_include_dir }}"
+        local_file_path: "{{ local_nebd_package_path }}/include/"
+        file_mode: 0644
+      include_tasks: copy_file_to_remote.yml
     - name: install nebd lib
       vars:
         remote_dir_name: "{{ curve_lib_dir }}"

diff --git a/include/client/libcurve.h b/include/client/libcurve.h
@@ -450,6 +450,12 @@ typedef struct UserInfo {
     }
 } UserInfo_t;
 
+struct OpenFlags {
+    bool exclusive;
+
+    OpenFlags() : exclusive(true) {}
+};
+
 class CurveClient {
  public:
     CurveClient();
@@ -474,7 +480,7 @@ class CurveClient {
      * @return 成功返回fd，失败返回-1
      */
     virtual int Open(const std::string& filename,
-                     std::string* sessionId);
+                     const OpenFlags& openflags);
 
     /**
      * 重新打开文件
@@ -484,8 +490,7 @@ class CurveClient {
      * @return 成功返回fd，失败返回-1
      */
     virtual int ReOpen(const std::string& filename,
-                       const std::string& sessionId,
-                       std::string* newSessionId);
+                       const OpenFlags& openflags);
 
     /**
      * 关闭文件

diff --git a/nbd/src/ImageInstance.cpp b/nbd/src/ImageInstance.cpp
@@ -23,6 +23,7 @@
  */
 
 #include "nbd/src/ImageInstance.h"
+
 #include <glog/logging.h>
 
 namespace curve {
@@ -42,7 +43,10 @@ bool ImageInstance::Open() {
         return false;
     }
 
-    ret = nebd_lib_open(imageName_.c_str());
+    NebdOpenFlags flags;
+    nebd_lib_init_open_flags(&flags);
+    flags.exclusive = config_->exclusive ? 1 : 0;
+    ret = nebd_lib_open_with_flags(imageName_.c_str(), &flags);
     if (ret < 0) {
         LOG(ERROR) << "open image failed, ret = " << ret;
         return false;

diff --git a/nbd/src/define.h b/nbd/src/define.h
@@ -50,14 +50,6 @@
 #include <sstream>
 #include <string>
 
-namespace std {
-
-inline std::string to_string(const std::string& val) {
-    return val;
-}
-
-}  // namespace std
-
 namespace curve {
 namespace nbd {
 
@@ -87,6 +79,8 @@ struct NBDConfig {
     bool try_netlink = false;
     // 需要映射的后端文件名称
     std::string imgname;
+    // exclusive open image or not
+    bool exclusive = true;
     // 指定需要映射的nbd设备路径
     std::string devpath;
     // force unmap even if the device is mounted
@@ -142,17 +136,24 @@ inline std::string BoolOption(const std::string& name, bool value,
     return opts;
 }
 
+inline std::string to_string(const std::string& val) {
+    return val;
+}
+
 template <typename T>
 inline std::string KeyValueOption(const std::string& optName, const T& value,
                                   const T& defaultValue, bool* firstOpt) {
+    using curve::nbd::to_string;
+    using std::to_string;
+
     std::string opts;
 
     if (value != defaultValue) {
         if (!*firstOpt) {
             opts += ",";
         }
 
-        opts += std::string(optName + "=") + std::to_string(value);
+        opts += std::string(optName + "=") + to_string(value);
         *firstOpt = false;
     }
 
@@ -171,6 +172,7 @@ inline std::string NBDConfig::MapOptions() const {
     opts.append(KeyValueOption("timeout", timeout, 3600, &firstOpt));
     opts.append(KeyValueOption("block-size", block_size, 4096, &firstOpt));
     opts.append(KeyValueOption("nebd-conf", nebd_conf, {}, &firstOpt));
+    opts.append(BoolOption("no-exclusive", !exclusive, &firstOpt));
 
     return opts.empty() ? "defaults" : opts;
 }

diff --git a/nbd/src/main.cpp b/nbd/src/main.cpp
@@ -89,6 +89,8 @@ static void Usage() {
         << "  --try-netlink           Use the nbd netlink interface\n"
         << "  --block-size            NBD Devices's block size, default is 4096, support 512 and 4096\n"  // NOLINT
         << "  --nebd-conf             LibNebd config file\n"
+        << "  --no-exclusive          Map image non exclusive\n"
+        << "\n"
         << "Unmap options:\n"
         << "  -f, --force                 Force unmap even if the device is mounted\n"              // NOLINT
         << "  --retry_times <limit>       The number of retries waiting for the process to exit\n"  // NOLINT
@@ -103,7 +105,8 @@ static int AddRecord(int flag) {
     std::string record;
     int fd = open(CURVETAB_PATH, O_WRONLY | O_APPEND);
     if (fd < 0) {
-        std::cerr << "curve-nbd: open curvetab file failed.";
+        std::cerr << "curve-nbd: open curvetab file failed, error: "
+                  << strerror(errno);
         return -EINVAL;
     }
     if (1 == flag) {

diff --git a/nbd/src/util.cpp b/nbd/src/util.cpp
@@ -50,6 +50,8 @@
 namespace curve {
 namespace nbd {
 
+constexpr uint32_t kSectorSize = 512;
+
 std::string cpp_strerror(int err) {
     char buf[128];
     if (err < 0)
@@ -107,6 +109,8 @@ static int parse_imgpath(const std::string &imgpath, NBDConfig *cfg,
     return 0;
 }
 
+// TODO(all): replace this function with gflags
+//            but, currently, gflags `--help` command is messy
 int parse_args(std::vector<const char*>& args, std::ostream *err_msg,   // NOLINT
                Command *command, NBDConfig *cfg) {
     std::vector<const char*>::iterator i;
@@ -153,9 +157,11 @@ int parse_args(std::vector<const char*>& args, std::ostream *err_msg,   // NOLIN
             }
         } else if (argparse_flag(args, i, "--try-netlink", (char *)NULL)) { // NOLINT
             cfg->try_netlink = true;
-        }  else if (argparse_flag(args, i, "-f", "--force", (char *)NULL)) {  // NOLINT
+        } else if (argparse_flag(args, i, "-f", "--force", (char *)NULL)) {  // NOLINT
             cfg->force_unmap = true;
-        } else if (argparse_witharg(args, i, &cfg->retry_times, err, "--retry_times", (char*)(NULL))) {  // NOLINT
+        } else if (argparse_flag(args, i, "--no-exclusive", (char*)NULL)) {   // NOLINT
+            cfg->exclusive = false;
+        }else if (argparse_witharg(args, i, &cfg->retry_times, err, "--retry_times", (char*)(NULL))) {  // NOLINT
             if (!err.str().empty()) {
                 *err_msg << "curve-nbd: " << err.str();
                 return -EINVAL;
@@ -328,7 +334,8 @@ int check_dev_can_unmap(const NBDConfig *cfg) {
     return -EINVAL;
 }
 
-int check_size_from_file(const std::string& path, uint64_t expected_size) {
+int check_size_from_file(const std::string &path, uint64_t expected_size,
+                         bool sizeInSector = false) {
     std::ifstream ifs;
     ifs.open(path.c_str(), std::ifstream::in);
     if (!ifs.is_open()) {
@@ -345,6 +352,10 @@ int check_size_from_file(const std::string& path, uint64_t expected_size) {
         return 0;
     }
 
+    if (sizeInSector) {
+        size *= kSectorSize;
+    }
+
     if (size != expected_size) {
         dout << "curve-nbd: kernel reported invalid size (" << size
             << ", expected " << expected_size << ")" << std::endl;
@@ -376,7 +387,7 @@ int check_device_size(int nbd_index, uint64_t expected_size) {
     // not affected.
 
     std::string path = "/sys/block/nbd" + std::to_string(nbd_index) + "/size";
-    return check_size_from_file(path, expected_size);
+    return check_size_from_file(path, expected_size, true);
 }
 
 static int run_command(const char *command) {
@@ -494,7 +505,7 @@ ssize_t safe_read_exact(int fd, void* buf, size_t count) {
     if (ret < 0) {
         return ret;
     }
-    if ((size_t)ret != count) {
+    if (static_cast<size_t>(ret) != count) {
         return -EDOM;
     }
     return 0;

diff --git a/nbd/test/nbd_tool_test.cpp b/nbd/test/nbd_tool_test.cpp
@@ -77,6 +77,8 @@ class NBDToolTest : public ::testing::Test {
         NebdClientAioContext aioCtx;
         EXPECT_CALL(*image_, AioRead(_))
             .WillRepeatedly(Invoke([](NebdClientAioContext* context){
+                memset(context->buf, 0, context->length);
+                context->ret = 0;
                 context->cb(context);
             }));
         nbdThread_ = std::thread(task, config, &isRunning_, &tool_);
@@ -91,10 +93,13 @@ class NBDToolTest : public ::testing::Test {
         EXPECT_CALL(*image_, AioWrite(_))
             .WillOnce(Invoke([&](NebdClientAioContext* context){
                 tempContext = *context;
+                memset(context->buf, 0, context->length);
+                context->ret = 0;
                 context->cb(context);
             }));
         EXPECT_CALL(*image_, Flush(_))
             .WillOnce(Invoke([](NebdClientAioContext* context){
+                context->ret = 0;
                 context->cb(context);
             }));
 
@@ -143,6 +148,7 @@ class NBDToolTest : public ::testing::Test {
 
 TEST_F(NBDToolTest, ioctl_connect_test) {
     NBDConfig config;
+    config.devpath = "/dev/nbd10";
     config.imgname = kTestImage;
     StartInAnotherThread(&config);
     ASSERT_TRUE(isRunning_);
@@ -154,6 +160,7 @@ TEST_F(NBDToolTest, ioctl_connect_test) {
 
 TEST_F(NBDToolTest, netlink_connect_test) {
     NBDConfig config;
+    config.devpath = "/dev/nbd10";
     config.imgname = kTestImage;
     config.try_netlink = true;
     StartInAnotherThread(&config);
@@ -165,6 +172,7 @@ TEST_F(NBDToolTest, netlink_connect_test) {
 
 TEST_F(NBDToolTest, readonly_test) {
     NBDConfig config;
+    config.devpath = "/dev/nbd10";
     config.imgname = kTestImage;
     config.readonly = true;
     StartInAnotherThread(&config);
@@ -177,6 +185,7 @@ TEST_F(NBDToolTest, readonly_test) {
 
 TEST_F(NBDToolTest, timeout_test) {
     NBDConfig config;
+    config.devpath = "/dev/nbd10";
     config.imgname = kTestImage;
     config.timeout = 3;
     StartInAnotherThread(&config);

diff --git a/nebd/proto/client.proto b/nebd/proto/client.proto
@@ -21,9 +21,15 @@ package nebd.client;
 
 option cc_generic_services = true;
 
+message ProtoOpenFlags {
+   required bool exclusive = 1;
+}
+
 message OpenFileRequest {
    required string fileName = 1;
+   optional ProtoOpenFlags flags = 2;
 }
+
 message OpenFileResponse {
    required RetCode retCode = 1;
    optional string retMsg = 2;

diff --git a/nebd/src/part1/libnebd.cpp b/nebd/src/part1/libnebd.cpp
@@ -68,7 +68,11 @@ int nebd_lib_uninit() {
 }
 
 int nebd_lib_open(const char* filename) {
-    return Open4Nebd(filename);
+    return Open4Nebd(filename, nullptr);
+}
+
+int nebd_lib_open_with_flags(const char* filename, const NebdOpenFlags* flags) {
+    return Open4Nebd(filename, flags);
 }
 
 int nebd_lib_close(int fd) {
@@ -121,4 +125,8 @@ int nebd_lib_invalidcache(int fd) {
     return InvalidCache4Nebd(fd);
 }
 
+void nebd_lib_init_open_flags(NebdOpenFlags* flags) {
+    flags->exclusive = 1;
+}
+
 }  // extern "C"
diff --git a/nebd/src/part1/libnebd.h b/nebd/src/part1/libnebd.h
@@ -47,6 +47,12 @@ typedef enum LIBAIO_OP {
     LIBAIO_OP_FLUSH,
 } LIBAIO_OP;
 
+typedef struct NebdOpenFlags {
+    int exclusive;
+} NebdOpenFlags;
+
+void nebd_lib_init_open_flags(NebdOpenFlags* flags);
+
 struct NebdClientAioContext;
 
 // nebd回调函数的类型
@@ -85,6 +91,8 @@ int nebd_lib_uninit(void);
  *  @return 成功返回文件fd，失败返回错误码
  */
 int nebd_lib_open(const char* filename);
+int nebd_lib_open_with_flags(const char* filename,
+                             const NebdOpenFlags* openflags);
 
 /**
  *  @brief close文件

diff --git a/nebd/src/part1/libnebd_file.cpp b/nebd/src/part1/libnebd_file.cpp
@@ -35,8 +35,8 @@ void Uninit4Nebd() {
     return;
 }
 
-int Open4Nebd(const char* filename) {
-    return nebd::client::nebdClient.Open(filename);
+int Open4Nebd(const char* filename, const NebdOpenFlags* openflags) {
+    return nebd::client::nebdClient.Open(filename, openflags);
 }
 
 int Close4Nebd(int fd) {

diff --git a/nebd/src/part1/libnebd_file.h b/nebd/src/part1/libnebd_file.h
@@ -42,7 +42,7 @@ void Uninit4Nebd();
  *  @param filename：文件名
  *  @return 成功返回文件fd，失败返回错误码
  */
-int Open4Nebd(const char* filename);
+int Open4Nebd(const char* filename, const NebdOpenFlags* flags);
 /**
  *  @brief close文件
  *  @param fd：文件的fd

diff --git a/nebd/src/part1/nebd_client.cpp b/nebd/src/part1/nebd_client.cpp
@@ -57,6 +57,13 @@ NebdClient &nebdClient = NebdClient::GetInstance();
 
 constexpr int32_t kBufSize = 128;
 
+ProtoOpenFlags ConverToProtoOpenFlags(const NebdOpenFlags* flags) {
+    ProtoOpenFlags protoFlags;
+    protoFlags.set_exclusive(flags->exclusive);
+
+    return protoFlags;
+}
+
 int NebdClient::Init(const char* confpath) {
     nebd::common::Configuration conf;
     conf.SetConfigPath(confpath);
@@ -131,7 +138,7 @@ void NebdClient::Uninit() {
     google::ShutdownGoogleLogging();
 }
 
-int NebdClient::Open(const char* filename) {
+int NebdClient::Open(const char* filename, const NebdOpenFlags* flags) {
     // 加文件锁
     std::string fileLockName =
         option_.fileLockPath + "/" + ReplaceSlash(filename);
@@ -151,6 +158,12 @@ int NebdClient::Open(const char* filename) {
         OpenFileResponse response;
 
         request.set_filename(filename);
+
+        if (flags != nullptr) {
+            auto* p = request.mutable_flags();
+            *p = ConverToProtoOpenFlags(flags);
+        }
+
         stub.OpenFile(cntl, &request, &response, nullptr);
 
         *rpcFailed = cntl->Failed();

diff --git a/nebd/src/part1/nebd_client.h b/nebd/src/part1/nebd_client.h
@@ -76,7 +76,7 @@ class NebdClient {
      *  @param filename：文件名
      *  @return 成功返回文件fd，失败返回错误码
      */
-    int Open(const char* filename);
+    int Open(const char* filename, const NebdOpenFlags* flags);
 
     /**
      *  @brief close文件