Merge remote-tracking branch 'origin/wip-15065'

Support new standby_for_fscid option, letting us assign daemons to FS clusters.

Reviewed-by: Greg Farnum <[email protected]>

src/common/config_opts.h

@@ -522,6 +522,7 @@ OPTION(mds_skip_ino, OPT_INT, 0)
 OPTION(max_mds, OPT_INT, 1)
 OPTION(mds_standby_for_name, OPT_STR, "")
 OPTION(mds_standby_for_rank, OPT_INT, -1)
+OPTION(mds_standby_for_fscid, OPT_INT, -1)
 OPTION(mds_standby_replay, OPT_BOOL, false)
 OPTION(mds_enable_op_tracker, OPT_BOOL, true) // enable/disable MDS op tracking
 OPTION(mds_op_history_size, OPT_U32, 20)    // Max number of completed ops to track

src/mds/Beacon.cc

@@ -34,14 +34,14 @@
 
 Beacon::Beacon(CephContext *cct_, MonClient *monc_, std::string name_) :
   Dispatcher(cct_), lock("Beacon"), monc(monc_), timer(g_ceph_context, lock),
-  name(name_), awaiting_seq(-1)
+  name(name_), standby_for_rank(MDSMap::MDS_NO_STANDBY_PREF),
+  standby_for_fscid(FS_CLUSTER_ID_NONE), awaiting_seq(-1)
 {
   want_state = MDSMap::STATE_NULL;
   last_seq = 0;
   sender = NULL;
   was_laggy = false;
 
-  standby_for_rank = MDSMap::MDS_NO_STANDBY_PREF;
   epoch = 0;
 }
 
@@ -52,7 +52,8 @@ Beacon::~Beacon()
 
 
 void Beacon::init(MDSMap const *mdsmap, MDSMap::DaemonState want_state_,
-    mds_rank_t standby_rank_, std::string const & standby_name_)
+    mds_rank_t standby_rank_, std::string const & standby_name_,
+    fs_cluster_id_t standby_fscid_)
 {
   Mutex::Locker l(lock);
   assert(mdsmap != NULL);
@@ -61,6 +62,7 @@ void Beacon::init(MDSMap const *mdsmap, MDSMap::DaemonState want_state_,
   _notify_mdsmap(mdsmap);
   standby_for_rank = standby_rank_;
   standby_for_name = standby_name_;
+  standby_for_fscid = standby_fscid_;
 
   // Spawn threads and start messaging
   timer.init();
@@ -207,6 +209,7 @@ void Beacon::_send()
 
   beacon->set_standby_for_rank(standby_for_rank);
   beacon->set_standby_for_name(standby_for_name);
+  beacon->set_standby_for_fscid(standby_for_fscid);
   beacon->set_health(health);
   beacon->set_compat(compat);
   // piggyback the sys info on beacon msg

src/mds/Beacon.h

@@ -50,6 +50,7 @@ class Beacon : public Dispatcher
   CompatSet compat;
   mds_rank_t standby_for_rank;
   std::string standby_for_name;
+  fs_cluster_id_t standby_for_fscid;
   MDSMap::DaemonState want_state;
 
   // Internal beacon state
@@ -85,7 +86,9 @@ class Beacon : public Dispatcher
   Beacon(CephContext *cct_, MonClient *monc_, std::string name);
   ~Beacon();
 
-  void init(MDSMap const *mdsmap, MDSMap::DaemonState want_state_, mds_rank_t standby_rank_, std::string const &standby_name_);
+  void init(MDSMap const *mdsmap, MDSMap::DaemonState want_state_,
+      mds_rank_t standby_rank_, std::string const &standby_name_,
+      fs_cluster_id_t standby_fscid_);
   void shutdown();
 
   bool ms_dispatch(Message *m); 
@@ -96,8 +99,6 @@ class Beacon : public Dispatcher
   void notify_mdsmap(MDSMap const *mdsmap);
   void notify_health(MDSRank const *mds);
 
-  void set_standby_for(mds_rank_t rank_, std::string const &name_);
-
   void handle_mds_beacon(MMDSBeacon *m);
   void send();
 

src/mds/FSMap.cc

@@ -495,14 +495,18 @@ mds_gid_t FSMap::find_standby_for(mds_role_t role, const std::string& name) cons
       continue;
     }
 
-    if ((info.standby_for_rank == role.rank && info.standby_for_ns == role.fscid)
+    if ((info.standby_for_rank == role.rank && info.standby_for_fscid == role.fscid)
         || (name.length() && info.standby_for_name == name)) {
       // It's a named standby for *me*, use it.
       return gid;
-    } else if (info.standby_for_rank < 0 && info.standby_for_name.length() == 0)
-      // It's not a named standby for anyone, use it if we don't find
-      // a named standby for me later.
-      result = gid;
+    } else if (
+        info.standby_for_rank < 0 && info.standby_for_name.length() == 0 &&
+        (info.standby_for_fscid == FS_CLUSTER_ID_NONE ||
+         info.standby_for_fscid == role.fscid)) {
+        // It's not a named standby for anyone, use it if we don't find
+        // a named standby for me later, unless it targets another FSCID.
+        result = gid;
+      }
   }
 
   return result;

src/mds/MDSDaemon.cc

@@ -559,7 +559,9 @@ int MDSDaemon::init(MDSMap::DaemonState wanted_state)
   if (wanted_state == MDSMap::STATE_NULL) {
     wanted_state = MDSMap::STATE_BOOT;
   }
-  beacon.init(mdsmap, wanted_state, standby_for_rank, standby_for_name);
+  beacon.init(mdsmap, wanted_state,
+    standby_for_rank, standby_for_name,
+    fs_cluster_id_t(g_conf->mds_standby_for_fscid));
   messenger->set_myname(entity_name_t::MDS(MDS_RANK_NONE));
 
   // schedule tick

src/mds/MDSMap.cc

@@ -81,7 +81,7 @@ void MDSMap::mds_info_t::dump(Formatter *f) const
     f->dump_stream("laggy_since") << laggy_since;
 
   f->dump_int("standby_for_rank", standby_for_rank);
-  f->dump_int("standby_for_ns", standby_for_ns);
+  f->dump_int("standby_for_fscid", standby_for_fscid);
   f->dump_string("standby_for_name", standby_for_name);
   f->open_array_section("export_targets");
   for (set<mds_rank_t>::iterator p = export_targets.begin();
@@ -419,7 +419,7 @@ void MDSMap::mds_info_t::encode_versioned(bufferlist& bl, uint64_t features) con
   ::encode(standby_for_name, bl);
   ::encode(export_targets, bl);
   ::encode(mds_features, bl);
-  ::encode(standby_for_ns, bl);
+  ::encode(standby_for_fscid, bl);
   ENCODE_FINISH(bl);
 }
 
@@ -458,7 +458,7 @@ void MDSMap::mds_info_t::decode(bufferlist::iterator& bl)
   if (struct_v >= 5)
     ::decode(mds_features, bl);
   if (struct_v >= 6) {
-    ::decode(standby_for_ns, bl);
+    ::decode(standby_for_fscid, bl);
   }
   DECODE_FINISH(bl);
 }

src/mds/MDSMap.h

@@ -140,13 +140,13 @@ class MDSMap {
     utime_t laggy_since;
     mds_rank_t standby_for_rank;
     std::string standby_for_name;
-    fs_cluster_id_t standby_for_ns;
+    fs_cluster_id_t standby_for_fscid;
     std::set<mds_rank_t> export_targets;
     uint64_t mds_features;
 
     mds_info_t() : global_id(MDS_GID_NONE), rank(MDS_RANK_NONE), inc(0), state(STATE_STANDBY), state_seq(0),
 		   standby_for_rank(MDS_NO_STANDBY_PREF),
-                   standby_for_ns(FS_CLUSTER_ID_NONE)
+                   standby_for_fscid(FS_CLUSTER_ID_NONE)
     { }
 
     bool laggy() const { return !(laggy_since == utime_t()); }

src/messages/MMDSBeacon.h

@@ -121,7 +121,7 @@ WRITE_CLASS_ENCODER(MDSHealth)
 
 class MMDSBeacon : public PaxosServiceMessage {
 
-  static const int HEAD_VERSION = 5;
+  static const int HEAD_VERSION = 6;
   static const int COMPAT_VERSION = 2;
 
   uuid_d fsid;
@@ -130,8 +130,10 @@ class MMDSBeacon : public PaxosServiceMessage {
 
   MDSMap::DaemonState state;
   version_t seq;
-  mds_rank_t standby_for_rank;
-  string standby_for_name;
+
+  mds_rank_t      standby_for_rank;
+  string          standby_for_name;
+  fs_cluster_id_t standby_for_fscid;
 
   CompatSet compat;
 
@@ -146,7 +148,7 @@ class MMDSBeacon : public PaxosServiceMessage {
   MMDSBeacon(const uuid_d &f, mds_gid_t g, string& n, epoch_t les, MDSMap::DaemonState st, version_t se, uint64_t feat) :
     PaxosServiceMessage(MSG_MDS_BEACON, les, HEAD_VERSION, COMPAT_VERSION),
     fsid(f), global_id(g), name(n), state(st), seq(se),
-    standby_for_rank(MDS_RANK_NONE),
+    standby_for_rank(MDS_RANK_NONE), standby_for_fscid(FS_CLUSTER_ID_NONE),
     mds_features(feat) {
   }
 private:
@@ -162,6 +164,7 @@ class MMDSBeacon : public PaxosServiceMessage {
   const char *get_type_name() const { return "mdsbeacon"; }
   mds_rank_t get_standby_for_rank() { return standby_for_rank; }
   const string& get_standby_for_name() { return standby_for_name; }
+  const fs_cluster_id_t& get_standby_for_fscid() { return standby_for_fscid; }
   uint64_t get_mds_features() const { return mds_features; }
 
   CompatSet const& get_compat() const { return compat; }
@@ -173,6 +176,7 @@ class MMDSBeacon : public PaxosServiceMessage {
   void set_standby_for_rank(mds_rank_t r) { standby_for_rank = r; }
   void set_standby_for_name(string& n) { standby_for_name = n; }
   void set_standby_for_name(const char* c) { standby_for_name.assign(c); }
+  void set_standby_for_fscid(fs_cluster_id_t f) { standby_for_fscid = f; }
 
   const map<string, string>& get_sys_info() const { return sys_info; }
   void set_sys_info(const map<string, string>& i) { sys_info = i; }
@@ -197,6 +201,7 @@ class MMDSBeacon : public PaxosServiceMessage {
       ::encode(sys_info, payload);
     }
     ::encode(mds_features, payload);
+    ::encode(standby_for_fscid, payload);
   }
   void decode_payload() {
     bufferlist::iterator p = payload.begin();
@@ -220,6 +225,9 @@ class MMDSBeacon : public PaxosServiceMessage {
     if (header.version >= 5) {
       ::decode(mds_features, p);
     }
+    if (header.version >= 6) {
+      ::decode(standby_for_fscid, p);
+    }
   }
 };
 

src/mon/MDSMonitor.cc

@@ -524,6 +524,7 @@ bool MDSMonitor::prepare_beacon(MonOpRequestRef op)
       new_info.state_seq = seq;
       new_info.standby_for_rank = m->get_standby_for_rank();
       new_info.standby_for_name = m->get_standby_for_name();
+      new_info.standby_for_fscid = m->get_standby_for_fscid();
       pending_fsmap.insert(new_info);
     }
 
@@ -540,7 +541,7 @@ bool MDSMonitor::prepare_beacon(MonOpRequestRef op)
         pending_fsmap.modify_daemon(gid, [fscid, leaderinfo, followable](
               MDSMap::mds_info_t *info) {
             info->standby_for_rank = leaderinfo->rank;
-            info->standby_for_ns = fscid;
+            info->standby_for_fscid = fscid;
         });
       }
     }
@@ -611,7 +612,7 @@ bool MDSMonitor::prepare_beacon(MonOpRequestRef op)
           pending_fsmap.modify_daemon(info.global_id,
               [target_info, target_ns, seq](MDSMap::mds_info_t *info) {
             info->standby_for_rank = target_info->rank;
-            info->standby_for_ns = target_ns;
+            info->standby_for_fscid = target_ns;
             info->state = MDSMap::STATE_STANDBY_REPLAY;
             info->state_seq = seq;
           });
@@ -620,14 +621,11 @@ bool MDSMonitor::prepare_beacon(MonOpRequestRef op)
           return false;
         }
       } else if (m->get_standby_for_rank() >= 0) {
-        // TODO get this from MDS message
-        // >>
-        fs_cluster_id_t target_ns = FS_CLUSTER_ID_NONE;
-        // <<
+        fs_cluster_id_t target_ns = m->get_standby_for_fscid();
 
         mds_role_t target_role = {
           target_ns == FS_CLUSTER_ID_NONE ?
-            pending_fsmap.legacy_client_fscid : info.standby_for_ns,
+            pending_fsmap.legacy_client_fscid : info.standby_for_fscid,
           m->get_standby_for_rank()};
 
         if (target_role.fscid != FS_CLUSTER_ID_NONE) {
@@ -636,7 +634,7 @@ bool MDSMonitor::prepare_beacon(MonOpRequestRef op)
             pending_fsmap.modify_daemon(info.global_id,
                 [target_role, seq](MDSMap::mds_info_t *info) {
               info->standby_for_rank = target_role.rank;
-              info->standby_for_ns = target_role.fscid;
+              info->standby_for_fscid = target_role.fscid;
               info->state = MDSMap::STATE_STANDBY_REPLAY;
               info->state_seq = seq;
             });
@@ -2766,8 +2764,8 @@ bool MDSMonitor::maybe_promote_standby(std::shared_ptr<Filesystem> fs)
         // The mds_info_t may or may not tell us exactly which filesystem
         // the standby_for_rank refers to: lookup via legacy_client_fscid
         mds_role_t target_role = {
-          info.standby_for_ns == FS_CLUSTER_ID_NONE ?
-            pending_fsmap.legacy_client_fscid : info.standby_for_ns,
+          info.standby_for_fscid == FS_CLUSTER_ID_NONE ?
+            pending_fsmap.legacy_client_fscid : info.standby_for_fscid,
           info.standby_for_rank};
 
         // If we managed to resolve a full target role