msg/async/rdma: add configuration to select gid_idx

With Mellanox NIC, different gid_idx maps to different gid combined with different RoCE protocol e.g. RoCEv1 or RoCEv2 1. Add configuration ms_async_rdma_gid_idx to set gid_idx. 1. Use "0" as default gid_idx to keep compatibility. 2. Initialize gid_idx in class member construction list. Signed-off-by: Changcheng Liu <[email protected]>
Conan-Kudo · Nov 11, 2019 · b971cff · b971cff
1 parent b5c7978
commit b971cff
Show file tree

Hide file tree

Showing 3 changed files with 8 additions and 3 deletions.
diff --git a/src/common/options.cc b/src/common/options.cc
@@ -1148,6 +1148,10 @@ std::vector<Option> get_global_options() {
     .set_default(1000)
     .set_description(""),
 
+    Option("ms_async_rdma_gid_idx", Option::TYPE_INT, Option::LEVEL_ADVANCED)
+    .set_default(0)
+    .set_description("use gid_idx to select GID for choosing RoCEv1 or RoCEv2"),
+
     Option("ms_async_rdma_local_gid", Option::TYPE_STR, Option::LEVEL_ADVANCED)
     .set_default("")
     .set_description(""),

diff --git a/src/msg/async/rdma/Infiniband.cc b/src/msg/async/rdma/Infiniband.cc
@@ -30,7 +30,8 @@ static const uint32_t MAX_INLINE_DATA = 0;
 static const uint32_t TCP_MSG_LEN = sizeof("0000:00000000:00000000:00000000:00000000000000000000000000000000");
 static const uint32_t CQ_DEPTH = 30000;
 
-Port::Port(CephContext *cct, struct ibv_context* ictxt, uint8_t ipn): ctxt(ictxt), port_num(ipn)
+Port::Port(CephContext *cct, struct ibv_context* ictxt, uint8_t ipn): ctxt(ictxt), port_num(ipn),
+  gid_idx(cct->_conf.get_val<int64_t>("ms_async_rdma_gid_idx"))
 {
   int r = ibv_query_port(ctxt, port_num, &port_attr);
   if (r == -1) {
@@ -39,7 +40,7 @@ Port::Port(CephContext *cct, struct ibv_context* ictxt, uint8_t ipn): ctxt(ictxt
   }
 
   lid = port_attr.lid;
-
+  ceph_assert(gid_idx < port_attr.gid_tbl_len);
 #ifdef HAVE_IBV_EXP
   union ibv_gid cgid;
   struct ibv_exp_gid_attr gid_attr;

diff --git a/src/msg/async/rdma/Infiniband.h b/src/msg/async/rdma/Infiniband.h
@@ -62,7 +62,7 @@ class Port {
   int port_num;
   struct ibv_port_attr port_attr;
   uint16_t lid;
-  int gid_idx = 0;
+  int gid_idx;
   union ibv_gid gid;
 
  public: