From 4182362c1195f3f322ae50dc0054518e62fea73a Mon Sep 17 00:00:00 2001 From: Patrick Donnelly Date: Mon, 18 Mar 2024 11:03:23 -0400 Subject: [PATCH 1/4] mon: do not log MON_DOWN if monitor uptime is less than threshold Fixes: https://tracker.ceph.com/issues/64968 Signed-off-by: Patrick Donnelly --- src/common/options/mon.yaml.in | 9 +++++++++ src/mon/HealthMonitor.cc | 9 +++++++-- src/mon/Monitor.cc | 1 + src/mon/Monitor.h | 12 ++++++++++++ 4 files changed, 29 insertions(+), 2 deletions(-) diff --git a/src/common/options/mon.yaml.in b/src/common/options/mon.yaml.in index ff8813c982f94..379917445c0f0 100644 --- a/src/common/options/mon.yaml.in +++ b/src/common/options/mon.yaml.in @@ -55,6 +55,15 @@ options: default: 1_min services: - mon +- name: mon_down_uptime_grace + type: secs + level: advanced + desc: Period in seconds that the cluster may have a mon down after this (leader) monitor comes up. + default: 1_min + services: + - mon + flags: + - runtime - name: mon_mgr_beacon_grace type: secs level: advanced diff --git a/src/mon/HealthMonitor.cc b/src/mon/HealthMonitor.cc index 4d2303d09fbcf..45563f87d3d04 100644 --- a/src/mon/HealthMonitor.cc +++ b/src/mon/HealthMonitor.cc @@ -803,9 +803,14 @@ void HealthMonitor::check_for_mon_down(health_check_map_t *checks) { int max = mon.monmap->size(); int actual = mon.get_quorum().size(); - const auto now = ceph::real_clock::now(); + const auto rcnow = ceph::real_clock::now(); + const auto created = mon.monmap->created.to_real_time(); + const auto mcnow = ceph::coarse_mono_clock::now(); + const auto starttime = mon.get_starttime(); + if (actual < max && - now > mon.monmap->created.to_real_time() + g_conf().get_val("mon_down_mkfs_grace")) { + (rcnow - created) > g_conf().get_val("mon_down_mkfs_grace") && + (mcnow - starttime) > g_conf().get_val("mon_down_uptime_grace")) { ostringstream ss; ss << (max-actual) << "/" << max << " mons down, quorum " << mon.get_quorum_names(); diff --git a/src/mon/Monitor.cc b/src/mon/Monitor.cc index a0fe3fa038e60..a70bfbe33c9de 100644 --- a/src/mon/Monitor.cc +++ b/src/mon/Monitor.cc @@ -2672,6 +2672,7 @@ void Monitor::get_mon_status(Formatter *f) f->dump_int("rank", rank); f->dump_string("state", get_state_name()); f->dump_int("election_epoch", get_epoch()); + f->dump_int("uptime", get_uptime().count()); f->open_array_section("quorum"); for (set::iterator p = quorum.begin(); p != quorum.end(); ++p) { diff --git a/src/mon/Monitor.h b/src/mon/Monitor.h index ed1e2a4e8a95d..13afacafde7dd 100644 --- a/src/mon/Monitor.h +++ b/src/mon/Monitor.h @@ -1099,6 +1099,18 @@ class Monitor : public Dispatcher, } bool is_keyring_required(); + +public: + ceph::coarse_mono_time get_starttime() const { + return starttime; + } + std::chrono::milliseconds get_uptime() const { + auto now = ceph::coarse_mono_clock::now(); + return std::chrono::duration_cast(now-starttime); + } + +private: + ceph::coarse_mono_time const starttime = coarse_mono_clock::now(); }; #define CEPH_MON_FEATURE_INCOMPAT_BASE CompatSet::Feature (1, "initial feature set (~v.18)") From 23de8e318f56b3052873fdd3744d313423e871bf Mon Sep 17 00:00:00 2001 From: Patrick Donnelly Date: Mon, 18 Mar 2024 15:55:04 -0400 Subject: [PATCH 2/4] qa: update dashboard schema for mon_status Signed-off-by: Patrick Donnelly --- qa/tasks/mgr/dashboard/test_health.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/qa/tasks/mgr/dashboard/test_health.py b/qa/tasks/mgr/dashboard/test_health.py index 3ebea97e3d29e..5b181968633c1 100644 --- a/qa/tasks/mgr/dashboard/test_health.py +++ b/qa/tasks/mgr/dashboard/test_health.py @@ -268,7 +268,8 @@ def test_full_health(self): 'state': str, # @TODO: What type should be expected here? 'sync_provider': JList(JAny(none=True)), - 'stretch_mode': bool + 'stretch_mode': bool, + 'uptime': int, }), 'osd_map': JObj({ # @TODO: define schema for crush map and osd_metadata, among From 57b9e161f44d2c685b4a0ab68aff1b43bfd63e66 Mon Sep 17 00:00:00 2001 From: Patrick Donnelly Date: Mon, 18 Mar 2024 20:38:08 -0400 Subject: [PATCH 3/4] qa: extend mon timeout coming up after mondb creation Fixes: https://tracker.ceph.com/issues/64968 Signed-off-by: Patrick Donnelly --- qa/cephfs/conf/mon.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/qa/cephfs/conf/mon.yaml b/qa/cephfs/conf/mon.yaml index eea56004a40b6..e32f0fd354732 100644 --- a/qa/cephfs/conf/mon.yaml +++ b/qa/cephfs/conf/mon.yaml @@ -3,3 +3,5 @@ overrides: conf: mon: mon op complaint time: 120 + # cephadm can take up to 5 minutes to bring up remaining mons + mon down mkfs grace: 300 From ae96ed4afb288fbc5a7ed818ba556c21418c631c Mon Sep 17 00:00:00 2001 From: Patrick Donnelly Date: Mon, 18 Mar 2024 14:32:01 -0400 Subject: [PATCH 4/4] qa/cephfs: stop ignoring MON_DOWN globally This should hopefully no longer be necessary with the addition of mon_down_uptime_grace. Signed-off-by: Patrick Donnelly --- qa/cephfs/overrides/ignorelist_wrongly_marked_down.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/qa/cephfs/overrides/ignorelist_wrongly_marked_down.yaml b/qa/cephfs/overrides/ignorelist_wrongly_marked_down.yaml index c9c46cc48f413..41ba84f04db5e 100644 --- a/qa/cephfs/overrides/ignorelist_wrongly_marked_down.yaml +++ b/qa/cephfs/overrides/ignorelist_wrongly_marked_down.yaml @@ -7,4 +7,3 @@ overrides: - but it is still running # MDS daemon 'b' is not responding, replacing it as rank 0 with standby 'a' - is not responding - - MON_DOWN