Merge pull request ceph#8789 from vuhuong/wip-msgr-flex-prop-list

msgr,xio: flexible Messenger::create options Reviewed-by: Sage Weil <[email protected]> Reviewed-by: Gregory Farnum <[email protected]> Reviewed-by: Haomai Wang <[email protected]> Reviewed-by: Casey Bodley <[email protected]>
allenk · Jun 8, 2016 · 9d54c92 · 9d54c92
2 parents f69f1bd + 5b218fb
commit 9d54c92
Show file tree

Hide file tree

Showing 9 changed files with 51 additions and 24 deletions.
diff --git a/src/ceph_mds.cc b/src/ceph_mds.cc
@@ -138,7 +138,7 @@ int main(int argc, const char **argv)
 
   Messenger *msgr = Messenger::create(g_ceph_context, g_conf->ms_type,
 				      entity_name_t::MDS(-1), "mds",
-				      getpid());
+				      getpid(), 0, Messenger::HAS_MANY_CONNECTIONS);
   if (!msgr)
     exit(1);
   msgr->set_cluster_protocol(CEPH_MDS_PROTOCOL);

diff --git a/src/ceph_mon.cc b/src/ceph_mon.cc
@@ -663,9 +663,8 @@ int main(int argc, const char **argv)
   // bind
   int rank = monmap.get_rank(g_conf->name.get_id());
   Messenger *msgr = Messenger::create(g_ceph_context, g_conf->ms_type,
-				      entity_name_t::MON(rank),
-				      "mon",
-				      0);
+				      entity_name_t::MON(rank), "mon",
+				      0, 0, Messenger::HAS_MANY_CONNECTIONS);
   if (!msgr)
     exit(1);
   msgr->set_cluster_protocol(CEPH_MON_PROTOCOL);

diff --git a/src/ceph_osd.cc b/src/ceph_osd.cc
@@ -447,19 +447,23 @@ int main(int argc, const char **argv)
 
   Messenger *ms_public = Messenger::create(g_ceph_context, g_conf->ms_type,
 					   entity_name_t::OSD(whoami), "client",
-					   getpid());
+					   getpid(), 0,
+					   Messenger::HAS_HEAVY_TRAFFIC |
+					   Messenger::HAS_MANY_CONNECTIONS);
   Messenger *ms_cluster = Messenger::create(g_ceph_context, g_conf->ms_type,
 					    entity_name_t::OSD(whoami), "cluster",
-					    getpid(), CEPH_FEATURES_ALL);
+					    getpid(), CEPH_FEATURES_ALL,
+					    Messenger::HAS_HEAVY_TRAFFIC |
+					    Messenger::HAS_MANY_CONNECTIONS);
   Messenger *ms_hbclient = Messenger::create(g_ceph_context, g_conf->ms_type,
 					     entity_name_t::OSD(whoami), "hbclient",
-					     getpid());
+					     getpid(), 0, Messenger::HEARTBEAT);
   Messenger *ms_hb_back_server = Messenger::create(g_ceph_context, g_conf->ms_type,
 						   entity_name_t::OSD(whoami), "hb_back_server",
-						   getpid());
+						   getpid(), 0, Messenger::HEARTBEAT);
   Messenger *ms_hb_front_server = Messenger::create(g_ceph_context, g_conf->ms_type,
 						    entity_name_t::OSD(whoami), "hb_front_server",
-						    getpid());
+						    getpid(), 0, Messenger::HEARTBEAT);
   Messenger *ms_objecter = Messenger::create(g_ceph_context, g_conf->ms_type,
 					     entity_name_t::OSD(whoami), "ms_objecter",
 					     getpid());

diff --git a/src/msg/Messenger.cc b/src/msg/Messenger.cc
@@ -21,7 +21,7 @@ Messenger *Messenger::create_client_messenger(CephContext *cct, string lname)
 
 Messenger *Messenger::create(CephContext *cct, const string &type,
 			     entity_name_t name, string lname,
-			     uint64_t nonce, uint64_t features)
+			     uint64_t nonce, uint64_t features, uint64_t cflags)
 {
   int r = -1;
   if (type == "random") {
@@ -36,7 +36,7 @@ Messenger *Messenger::create(CephContext *cct, const string &type,
 #ifdef HAVE_XIO
   else if ((type == "xio") &&
 	   cct->check_experimental_feature_enabled("ms-type-xio"))
-    return new XioMessenger(cct, name, lname, nonce, features);
+    return new XioMessenger(cct, name, lname, nonce, features, cflags);
 #endif
   lderr(cct) << "unrecognized ms_type '" << type << "'" << dendl;
   return nullptr;

diff --git a/src/msg/Messenger.h b/src/msg/Messenger.h
@@ -52,6 +52,14 @@ class Messenger {
   int socket_priority;
 
 public:
+  /**
+   * Various Messenger conditional config/type flags to allow
+   * different "transport" Messengers to tune themselves
+   */
+  static const int HAS_HEAVY_TRAFFIC    = 0x0001;
+  static const int HAS_MANY_CONNECTIONS = 0x0002;
+  static const int HEARTBEAT            = 0x0004;
+
   /**
    *  The CephContext this Messenger uses. Many other components initialize themselves
    *  from this value.
@@ -151,13 +159,15 @@ class Messenger {
    * @param lname logical name of the messenger in this process (e.g., "client")
    * @param nonce nonce value to uniquely identify this instance on the current host
    * @param features bits for the local connection
+   * @param cflags general set of flags to configure transport resources
    */
   static Messenger *create(CephContext *cct,
                            const string &type,
                            entity_name_t name,
 			   string lname,
                            uint64_t nonce,
-			   uint64_t features = 0);
+			   uint64_t features = 0,
+			   uint64_t cflags = 0);
 
   /**
    * create a new messenger

diff --git a/src/msg/xio/XioMessenger.cc b/src/msg/xio/XioMessenger.cc
@@ -351,12 +351,12 @@ static ostream& _prefix(std::ostream *_dout, XioMessenger *msgr) {
 
 XioMessenger::XioMessenger(CephContext *cct, entity_name_t name,
 			   string mname, uint64_t _nonce, uint64_t features,
-			   DispatchStrategy *ds)
+			   uint64_t cflags, DispatchStrategy *ds)
   : SimplePolicyMessenger(cct, name, mname, _nonce),
     XioInit(cct),
     nsessions(0),
     shutdown_called(false),
-    portals(this, get_nportals(), get_nconns_per_portal()),
+    portals(this, get_nportals(cflags), get_nconns_per_portal(cflags)),
     dispatch_strategy(ds),
     loop_con(new XioLoopbackConnection(this)),
     special_handling(0),
@@ -383,8 +383,8 @@ XioMessenger::XioMessenger(CephContext *cct, entity_name_t name,
 
   ldout(cct,2) << "Create msgr: " << this << " instance: "
     << nInstances.read() << " type: " << name.type_str()
-    << " subtype: " << mname << " nportals: " << get_nportals()
-    << " nconns_per_portal: " << get_nconns_per_portal() << " features: "
+    << " subtype: " << mname << " nportals: " << get_nportals(cflags)
+    << " nconns_per_portal: " << get_nconns_per_portal(cflags) << " features: "
     << features << dendl;
 
 } /* ctor */
@@ -399,14 +399,27 @@ int XioMessenger::pool_hint(uint32_t dsize) {
 				   XMSG_MEMPOOL_QUANTUM, 0);
 }
 
-int XioMessenger::get_nconns_per_portal()
+int XioMessenger::get_nconns_per_portal(uint64_t cflags)
 {
-  return max(cct->_conf->xio_max_conns_per_portal, 32);
+  const int XIO_DEFAULT_NUM_CONNS_PER_PORTAL = 8;
+  int nconns = XIO_DEFAULT_NUM_CONNS_PER_PORTAL;
+
+  if (cflags & Messenger::HAS_MANY_CONNECTIONS)
+    nconns = max(cct->_conf->xio_max_conns_per_portal, XIO_DEFAULT_NUM_CONNS_PER_PORTAL);
+  else if (cflags & Messenger::HEARTBEAT)
+    nconns = max(cct->_conf->osd_heartbeat_min_peers * 4, XIO_DEFAULT_NUM_CONNS_PER_PORTAL);
+
+  return nconns;
 }
 
-int XioMessenger::get_nportals()
+int XioMessenger::get_nportals(uint64_t cflags)
 {
-  return max(cct->_conf->xio_portal_threads, 1);
+  int nportals = 1;
+
+  if (cflags & Messenger::HAS_HEAVY_TRAFFIC)
+    nportals = max(cct->_conf->xio_portal_threads, 1);
+
+  return nportals;
 }
 
 void XioMessenger::learned_addr(const entity_addr_t &peer_addr_for_me)

diff --git a/src/msg/xio/XioMessenger.h b/src/msg/xio/XioMessenger.h
@@ -64,6 +64,7 @@ class XioMessenger : public SimplePolicyMessenger, XioInit
 public:
   XioMessenger(CephContext *cct, entity_name_t name,
 	       string mname, uint64_t nonce, uint64_t features,
+	       uint64_t cflags = 0,
 	       DispatchStrategy* ds = new QueueStrategy(1));
 
   virtual ~XioMessenger();
@@ -149,8 +150,8 @@ class XioMessenger : public SimplePolicyMessenger, XioInit
   void learned_addr(const entity_addr_t& peer_addr_for_me);
 
 private:
-  int get_nconns_per_portal();
-  int get_nportals();
+  int get_nconns_per_portal(uint64_t cflags);
+  int get_nportals(uint64_t cflags);
 
 protected:
   virtual void ready()

diff --git a/src/test/messenger/xio_client.cc b/src/test/messenger/xio_client.cc
@@ -117,7 +117,7 @@ int main(int argc, const char **argv)
 	messenger = new XioMessenger(g_ceph_context,
 				     entity_name_t::MON(-1),
 				     "xio_client",
-				     0 /* nonce */, XIO_ALL_FEATURES,
+				     0 /* nonce */, XIO_ALL_FEATURES, 0 /* cflags */,
 				     dstrategy);
 
 	// enable timing prints

diff --git a/src/test/messenger/xio_server.cc b/src/test/messenger/xio_server.cc
@@ -88,7 +88,7 @@ int main(int argc, const char **argv)
 	messenger = new XioMessenger(g_ceph_context,
 				     entity_name_t::MON(-1),
 				     "xio_server",
-				     0 /* nonce */, XIO_ALL_FEATURES,
+				     0 /* nonce */, XIO_ALL_FEATURES, 0 /* cflags */,
 				     dstrategy);
 
 	static_cast<XioMessenger*>(messenger)->set_magic(