Merge PR ceph#48720 into main

* refs/pull/48720/head: qa: fix api failure issue caused during a dashboard test PendingReleaseNotes: noted new MDSMap field refuse_client_session qa: added two testcases client: do not initiate session if flag refuse_client_session is set mds: do not reconnect when refuse_client_session is set mds: add new feature to block clients from establishing sessions Reviewed-by: Rishabh Dave <[email protected]> Reviewed-by: Kotresh Hiremath Ravishankar <[email protected]> Reviewed-by: Patrick Donnelly <[email protected]> Reviewed-by: Milind Changire <[email protected]> Reviewed-by: Ramana Raja <[email protected]> Reviewed-by: Venky Shankar <[email protected]>
mitsu-ko · Mar 1, 2023 · cca84e6 · cca84e6
2 parents 707af78 + 2ecd8d6
commit cca84e6
Show file tree

Hide file tree

Showing 11 changed files with 110 additions and 4 deletions.
diff --git a/PendingReleaseNotes b/PendingReleaseNotes
@@ -173,3 +173,8 @@ of the feature, refer this link on how to perform it:
 https://docs.ceph.com/en/quincy/cephadm/upgrade/#staggered-upgrade
 Relevant tracker: https://tracker.ceph.com/issues/55715
 
+* Introduced a new file system flag `refuse_client_session` that can be set using the
+`fs set` command. This flag allows blocking any incoming session
+request from client(s). This can be useful during some recovery situations
+where it's desirable to bring MDS up but have no client workload.
+Relevant tracker: https://tracker.ceph.com/issues/57090
diff --git a/qa/tasks/cephfs/filesystem.py b/qa/tasks/cephfs/filesystem.py
@@ -598,6 +598,9 @@ def set_allow_new_snaps(self, yes):
     def set_bal_rank_mask(self, bal_rank_mask):
         self.set_var("bal_rank_mask", bal_rank_mask)
 
+    def set_refuse_client_session(self, yes):
+        self.set_var("refuse_client_session", yes)
+
     def compat(self, *args):
         a = map(lambda x: str(x).lower(), args)
         self.mon_manager.raw_cluster_cmd("fs", "compat", self.name, *a)

diff --git a/qa/tasks/cephfs/test_client_recovery.py b/qa/tasks/cephfs/test_client_recovery.py
@@ -724,3 +724,34 @@ def test_reconnect_after_blocklisted(self):
 
         rproc.wait()
         self.assertEqual(rproc.exitstatus, 0)
+
+    def test_refuse_client_session(self):
+        """
+        Test that client cannot start session when file system flag
+        refuse_client_session is set
+        """
+
+        self.mount_a.umount_wait()
+        self.fs.set_refuse_client_session(True)
+        with self.assertRaises(CommandFailedError):
+            self.mount_a.mount_wait()
+
+    def test_refuse_client_session_on_reconnect(self):
+        """
+        Test that client cannot reconnect when filesystem comes online and
+        file system flag refuse_client_session is set
+        """
+
+        self.mount_a.create_files()
+        self.mount_a.check_files()
+
+        self.fs.fail()
+        self.fs.set_refuse_client_session(True)
+        self.fs.set_joinable()
+        with self.assert_cluster_log('client could not reconnect as'
+                                     ' file system flag'
+                                     ' refuse_client_session is set'):
+            time.sleep(self.fs.get_var("session_timeout") * 1.5)
+            self.assertEqual(len(self.fs.mds_tell(["session", "ls"])), 0)
+        self.mount_a.umount_wait(force=True)
+
diff --git a/qa/tasks/mgr/dashboard/test_health.py b/qa/tasks/mgr/dashboard/test_health.py
@@ -56,7 +56,8 @@ class HealthTest(DashboardTestCase):
             'joinable': bool,
             'allow_snaps': bool,
             'allow_multimds_snaps': bool,
-            'allow_standby_replay': bool
+            'allow_standby_replay': bool,
+            'refuse_client_session': bool
         }),
         'ever_allowed_features': int,
         'root': int

diff --git a/src/client/Client.cc b/src/client/Client.cc
@@ -6360,6 +6360,12 @@ int Client::mount(const std::string &mount_root, const UserPerm& perms,
     }
   }
 
+  if(mdsmap->test_flag(CEPH_MDSMAP_REFUSE_CLIENT_SESSION)) {
+    lderr(cct) << "connections cannot be made while" 
+                  " the flag refuse_client_session is set" << dendl;
+    return -CEPHFS_EACCES;
+  }
+
   populate_metadata(mount_root.empty() ? "/" : mount_root);
 
   filepath fp(CEPH_INO_ROOT);

diff --git a/src/include/ceph_fs.h b/src/include/ceph_fs.h
@@ -288,7 +288,8 @@ struct ceph_mon_subscribe_ack {
 #define CEPH_MDSMAP_ALLOW_MULTIMDS_SNAPS	     (1<<4)  /* cluster alllowed to enable MULTIMDS
                                                             and SNAPS at the same time */
 #define CEPH_MDSMAP_ALLOW_STANDBY_REPLAY         (1<<5)  /* cluster alllowed to enable MULTIMDS */
-
+#define CEPH_MDSMAP_REFUSE_CLIENT_SESSION        (1<<6)  /* cluster allowed to refuse client session
+                                                            request */
 #define CEPH_MDSMAP_DEFAULTS (CEPH_MDSMAP_ALLOW_SNAPS | \
 			      CEPH_MDSMAP_ALLOW_MULTIMDS_SNAPS)
 

diff --git a/src/mds/MDSMap.cc b/src/mds/MDSMap.cc
@@ -234,6 +234,7 @@ void MDSMap::dump_flags_state(Formatter *f) const
     f->dump_bool(flag_display.at(CEPH_MDSMAP_ALLOW_SNAPS), allows_snaps());
     f->dump_bool(flag_display.at(CEPH_MDSMAP_ALLOW_MULTIMDS_SNAPS), allows_multimds_snaps());
     f->dump_bool(flag_display.at(CEPH_MDSMAP_ALLOW_STANDBY_REPLAY), allows_standby_replay());
+    f->dump_bool(flag_display.at(CEPH_MDSMAP_REFUSE_CLIENT_SESSION), test_flag(CEPH_MDSMAP_REFUSE_CLIENT_SESSION));
     f->close_section();
 }
 
@@ -373,6 +374,8 @@ void MDSMap::print_flags(std::ostream& out) const {
     out << " " << flag_display.at(CEPH_MDSMAP_ALLOW_MULTIMDS_SNAPS);
   if (allows_standby_replay())
     out << " " << flag_display.at(CEPH_MDSMAP_ALLOW_STANDBY_REPLAY);
+  if (test_flag(CEPH_MDSMAP_REFUSE_CLIENT_SESSION))
+    out << " " << flag_display.at(CEPH_MDSMAP_REFUSE_CLIENT_SESSION);
 }
 
 void MDSMap::get_health(list<pair<health_status_t,string> >& summary,

diff --git a/src/mds/MDSMap.h b/src/mds/MDSMap.h
@@ -663,7 +663,8 @@ class MDSMap {
     {CEPH_MDSMAP_NOT_JOINABLE, "joinable"}, //inverse for user display
     {CEPH_MDSMAP_ALLOW_SNAPS, "allow_snaps"},
     {CEPH_MDSMAP_ALLOW_MULTIMDS_SNAPS, "allow_multimds_snaps"},
-    {CEPH_MDSMAP_ALLOW_STANDBY_REPLAY, "allow_standby_replay"}
+    {CEPH_MDSMAP_ALLOW_STANDBY_REPLAY, "allow_standby_replay"},
+    {CEPH_MDSMAP_REFUSE_CLIENT_SESSION, "refuse_client_session"}
   };
 };
 WRITE_CLASS_ENCODER_FEATURES(MDSMap::mds_info_t)

diff --git a/src/mds/Server.cc b/src/mds/Server.cc
@@ -589,6 +589,16 @@ void Server::handle_client_session(const cref_t<MClientSession> &m)
   uint64_t sseq = 0;
   switch (m->get_op()) {
   case CEPH_SESSION_REQUEST_OPEN:
+    if(mds->mdsmap->test_flag(CEPH_MDSMAP_REFUSE_CLIENT_SESSION)) {
+      dout(0) << "new sessions are not permitted, enable again via"
+                 "`ceph fs set <fs_name> refuse_client_session false`" << dendl;
+      auto reply = make_message<MClientSession>(CEPH_SESSION_REJECT);
+      reply->metadata["error_string"] = "new sessions are not permitted,"
+                                        " enable again via `ceph fs set"
+                                        " <fs_name> refuse_client_session false`";
+      mds->send_message(reply, m->get_connection());
+      return;
+    }
     if (session->is_opening() ||
 	session->is_open() ||
 	session->is_stale() ||
@@ -1446,6 +1456,18 @@ void Server::handle_client_reconnect(const cref_t<MClientReconnect> &m)
     return;
   }
 
+  if(mds->mdsmap->test_flag(CEPH_MDSMAP_REFUSE_CLIENT_SESSION)) {
+    mds->clog->warn() << "client could not reconnect as" 
+                         " file system flag refuse_client_session is set";
+    dout(0) << "client cannot reconnect when file system flag"
+               " refuse_client_session is set" << dendl;
+    auto reply = make_message<MClientSession>(CEPH_SESSION_CLOSE);
+    reply->metadata["error_string"] = "client cannot reconnect when file system flag" 
+                                        " refuse_client_session is set";
+    mds->send_message(reply, m->get_connection());
+    return;
+  }
+
   if (!session->is_open()) {
     dout(0) << " ignoring msg from not-open session" << *m << dendl;
     auto reply = make_message<MClientSession>(CEPH_SESSION_CLOSE);

diff --git a/src/mon/FSCommands.cc b/src/mon/FSCommands.cc
@@ -681,6 +681,38 @@ class SetHandler : public FileSystemCommandHandler
         fs->mds_map.set_min_compat_client(vno);
       };
       fsmap.modify_filesystem(fs->fscid, std::move(f));
+    } else if (var == "refuse_client_session") {
+      bool refuse_session = false;
+      int r = parse_bool(val, &refuse_session, ss);
+      if (r != 0) {
+        return r;
+      }
+
+      if (refuse_session) {
+        if (!(fs->mds_map.test_flag(CEPH_MDSMAP_REFUSE_CLIENT_SESSION))) {
+          fsmap.modify_filesystem(
+            fs->fscid,
+            [](std::shared_ptr<Filesystem> fs)
+          {
+            fs->mds_map.set_flag(CEPH_MDSMAP_REFUSE_CLIENT_SESSION);
+          });
+          ss << "client(s) blocked from establishing new session(s)"; 
+        } else {
+          ss << "client(s) already blocked from establishing new session(s)";
+        }     
+      } else {
+          if (fs->mds_map.test_flag(CEPH_MDSMAP_REFUSE_CLIENT_SESSION)) {
+            fsmap.modify_filesystem(
+              fs->fscid,
+              [](std::shared_ptr<Filesystem> fs)
+            {
+              fs->mds_map.clear_flag(CEPH_MDSMAP_REFUSE_CLIENT_SESSION);
+            });
+            ss << "client(s) allowed to establish new session(s)"; 
+          } else {
+            ss << "client(s) already allowed to establish new session(s)";
+          }
+      }
     } else {
       ss << "unknown variable " << var;
       return -EINVAL;

diff --git a/src/mon/MonCommands.h b/src/mon/MonCommands.h
@@ -377,7 +377,8 @@ COMMAND("fs set "
 	"name=var,type=CephChoices,strings=max_mds|max_file_size"
         "|allow_new_snaps|inline_data|cluster_down|allow_dirfrags|balancer"
         "|standby_count_wanted|session_timeout|session_autoclose"
-        "|allow_standby_replay|down|joinable|min_compat_client|bal_rank_mask "
+        "|allow_standby_replay|down|joinable|min_compat_client|bal_rank_mask"
+	"|refuse_client_session "
 	"name=val,type=CephString "
 	"name=yes_i_really_mean_it,type=CephBool,req=false "
 	"name=yes_i_really_really_mean_it,type=CephBool,req=false",