diff --git a/qa/cephfs/conf/mon.yaml b/qa/cephfs/conf/mon.yaml index eea56004a40b6..e32f0fd354732 100644 --- a/qa/cephfs/conf/mon.yaml +++ b/qa/cephfs/conf/mon.yaml @@ -3,3 +3,5 @@ overrides: conf: mon: mon op complaint time: 120 + # cephadm can take up to 5 minutes to bring up remaining mons + mon down mkfs grace: 300 diff --git a/qa/cephfs/overrides/ignorelist_wrongly_marked_down.yaml b/qa/cephfs/overrides/ignorelist_wrongly_marked_down.yaml index c9c46cc48f413..41ba84f04db5e 100644 --- a/qa/cephfs/overrides/ignorelist_wrongly_marked_down.yaml +++ b/qa/cephfs/overrides/ignorelist_wrongly_marked_down.yaml @@ -7,4 +7,3 @@ overrides: - but it is still running # MDS daemon 'b' is not responding, replacing it as rank 0 with standby 'a' - is not responding - - MON_DOWN diff --git a/qa/tasks/mgr/dashboard/test_health.py b/qa/tasks/mgr/dashboard/test_health.py index 3ebea97e3d29e..5b181968633c1 100644 --- a/qa/tasks/mgr/dashboard/test_health.py +++ b/qa/tasks/mgr/dashboard/test_health.py @@ -268,7 +268,8 @@ def test_full_health(self): 'state': str, # @TODO: What type should be expected here? 'sync_provider': JList(JAny(none=True)), - 'stretch_mode': bool + 'stretch_mode': bool, + 'uptime': int, }), 'osd_map': JObj({ # @TODO: define schema for crush map and osd_metadata, among diff --git a/src/common/options/mon.yaml.in b/src/common/options/mon.yaml.in index ff8813c982f94..379917445c0f0 100644 --- a/src/common/options/mon.yaml.in +++ b/src/common/options/mon.yaml.in @@ -55,6 +55,15 @@ options: default: 1_min services: - mon +- name: mon_down_uptime_grace + type: secs + level: advanced + desc: Period in seconds that the cluster may have a mon down after this (leader) monitor comes up. + default: 1_min + services: + - mon + flags: + - runtime - name: mon_mgr_beacon_grace type: secs level: advanced diff --git a/src/mon/HealthMonitor.cc b/src/mon/HealthMonitor.cc index 4d2303d09fbcf..45563f87d3d04 100644 --- a/src/mon/HealthMonitor.cc +++ b/src/mon/HealthMonitor.cc @@ -803,9 +803,14 @@ void HealthMonitor::check_for_mon_down(health_check_map_t *checks) { int max = mon.monmap->size(); int actual = mon.get_quorum().size(); - const auto now = ceph::real_clock::now(); + const auto rcnow = ceph::real_clock::now(); + const auto created = mon.monmap->created.to_real_time(); + const auto mcnow = ceph::coarse_mono_clock::now(); + const auto starttime = mon.get_starttime(); + if (actual < max && - now > mon.monmap->created.to_real_time() + g_conf().get_val("mon_down_mkfs_grace")) { + (rcnow - created) > g_conf().get_val("mon_down_mkfs_grace") && + (mcnow - starttime) > g_conf().get_val("mon_down_uptime_grace")) { ostringstream ss; ss << (max-actual) << "/" << max << " mons down, quorum " << mon.get_quorum_names(); diff --git a/src/mon/Monitor.cc b/src/mon/Monitor.cc index a0fe3fa038e60..a70bfbe33c9de 100644 --- a/src/mon/Monitor.cc +++ b/src/mon/Monitor.cc @@ -2672,6 +2672,7 @@ void Monitor::get_mon_status(Formatter *f) f->dump_int("rank", rank); f->dump_string("state", get_state_name()); f->dump_int("election_epoch", get_epoch()); + f->dump_int("uptime", get_uptime().count()); f->open_array_section("quorum"); for (set::iterator p = quorum.begin(); p != quorum.end(); ++p) { diff --git a/src/mon/Monitor.h b/src/mon/Monitor.h index ed1e2a4e8a95d..13afacafde7dd 100644 --- a/src/mon/Monitor.h +++ b/src/mon/Monitor.h @@ -1099,6 +1099,18 @@ class Monitor : public Dispatcher, } bool is_keyring_required(); + +public: + ceph::coarse_mono_time get_starttime() const { + return starttime; + } + std::chrono::milliseconds get_uptime() const { + auto now = ceph::coarse_mono_clock::now(); + return std::chrono::duration_cast(now-starttime); + } + +private: + ceph::coarse_mono_time const starttime = coarse_mono_clock::now(); }; #define CEPH_MON_FEATURE_INCOMPAT_BASE CompatSet::Feature (1, "initial feature set (~v.18)")