Merge pull request ceph#3681 from ceph/wip-fusesystem-10710

ceph-fuse: check for failures on system() invocation

Reviewed-by: John Spray <[email protected]>

src/ceph_fuse.cc

@@ -13,6 +13,7 @@
  */
 
 #include <sys/stat.h>
+#include <sys/utsname.h>
 #include <iostream>
 #include <string>
 using namespace std;
@@ -84,6 +85,8 @@ int main(int argc, const char **argv, const char *envp[]) {
   // we need to handle the forking ourselves.
   int fd[2] = {0, 0};  // parent's, child's
   pid_t childpid = 0;
+  int tester_r = 0;
+  void *tester_rp = NULL;
   bool restart_log = false;
   if (g_conf->daemonize) {
     int r = socketpair(AF_UNIX, SOCK_STREAM, 0, fd);
@@ -107,13 +110,53 @@ int main(int argc, const char **argv, const char *envp[]) {
     if (restart_log)
       g_ceph_context->_log->start();
 
+    class RemountTest : public Thread {
+    public:
+      CephFuse *cfuse;
+      Client *client;
+      RemountTest() : Thread() {}
+      void init(CephFuse *cf, Client *cl) {
+	cfuse = cf;
+	client = cl;
+      }
+      virtual ~RemountTest() {}
+      virtual void *entry() {
+	struct utsname os_info;
+	int tr = uname(&os_info);
+	assert(tr == 0);
+	assert(memcmp(os_info.sysname, "Linux", 5) == 0);
+	int major, minor;	
+	char *end_num;
+	major = strtol(os_info.release, &end_num, 10);
+	assert(major > 0);
+	++end_num;
+	minor = strtol(end_num, NULL, 10);
+	bool can_invalidate_dentries = g_conf->client_try_dentry_invalidate &&
+	  (major < 3 ||
+	   (major == 3 && minor < 18));
+	tr = client->test_dentry_handling(can_invalidate_dentries);
+	if (tr != 0) {
+	  cerr << "ceph-fuse[" << getpid()
+	       << "]: fuse failed dentry invalidate/remount test with error "
+	       << cpp_strerror(tr) << ", stopping" << std::endl;
+
+	  char buf[5050];
+	  string mountpoint = cfuse->get_mount_point();
+	  snprintf(buf, 5049, "fusermount -u -z %s", mountpoint.c_str());
+	  system(buf);
+	}
+	return reinterpret_cast<void*>(tr);
+      }
+    } tester;
+
+
     // get monmap
     Messenger *messenger = NULL;
     Client *client;
     CephFuse *cfuse;
 
-    MonClient mc(g_ceph_context);
-    int r = mc.build_initial_monmap();
+    MonClient *mc = new MonClient(g_ceph_context);
+    int r = mc->build_initial_monmap();
     if (r == -EINVAL)
       usage();
     if (r < 0)
@@ -127,7 +170,7 @@ int main(int argc, const char **argv, const char *envp[]) {
     messenger->set_policy(entity_name_t::TYPE_MDS,
 			  Messenger::Policy::lossless_client(0, 0));
 
-    client = new Client(messenger, &mc);
+    client = new Client(messenger, mc);
     if (filer_flags) {
       client->set_filer_flags(filer_flags);
     }
@@ -169,17 +212,24 @@ int main(int argc, const char **argv, const char *envp[]) {
       cerr << "ceph-fuse[" << getpid() << "]: fuse failed to start" << std::endl;
       goto out_client_unmount;
     }
+
     cerr << "ceph-fuse[" << getpid() << "]: starting fuse" << std::endl;
+    tester.init(cfuse, client);
+    tester.create();
     r = cfuse->loop();
-    cerr << "ceph-fuse[" << getpid() << "]: fuse finished with error " << r << std::endl;
-
+    tester.join(&tester_rp);
+    tester_r = static_cast<int>(reinterpret_cast<uint64_t>(tester_rp));
+    cerr << "ceph-fuse[" << getpid() << "]: fuse finished with error " << r
+	 << " and tester_r " << tester_r <<std::endl;
+
+
   out_client_unmount:
     client->unmount();
     //cout << "unmounted" << std::endl;
-
+    
     cfuse->finalize();
     delete cfuse;
-
+    
   out_shutdown:
     client->shutdown();
   out_init_failed:
@@ -189,17 +239,19 @@ int main(int argc, const char **argv, const char *envp[]) {
   out_messenger_start_failed:
     delete client;
   out_mc_start_failed:
-
+    
     if (g_conf->daemonize) {
       //cout << "child signalling parent with " << r << std::endl;
       static int foo = 0;
       foo += ::write(fd[1], &r, sizeof(r));
     }
-
+    
     delete messenger;
     g_ceph_context->put();
     free(newargv);
-
+
+    delete mc;
+
     //cout << "child done" << std::endl;
     return r;
   } else {

src/client/Client.cc

@@ -164,6 +164,8 @@ Client::Client(Messenger *m, MonClient *mc)
     ino_invalidate_cb(NULL),
     dentry_invalidate_cb(NULL),
     getgroups_cb(NULL),
+    can_invalidate_dentries(false),
+    require_remount(false),
     async_ino_invalidator(m->cct),
     async_dentry_invalidator(m->cct),
     interrupt_finisher(m->cct),
@@ -3445,16 +3447,33 @@ class C_Client_Remount : public Context  {
 public:
   C_Client_Remount(Client *c) : client(c) {}
   void finish(int r) {
-    client->remount_cb(client->callback_handle);
+    assert (r == 0);
+    r = client->remount_cb(client->callback_handle);
+    if (r != 0) {
+      client_t whoami = client->get_nodeid();
+      lderr(client->cct) << "tried to remount (to trim kernel dentries) and got error "
+			 << r << dendl;
+      if (client->require_remount) {
+	assert(0 == "failed to remount for kernel dentry trimming");
+      }
+    }
   }
 };
 
 void Client::_invalidate_kernel_dcache()
 {
-  // Hacky:
-  // when remounting a file system, linux kernel trims all unused dentries in the file system
-  if (remount_cb)
+  if (can_invalidate_dentries && dentry_invalidate_cb && root->dir) {
+    for (ceph::unordered_map<string, Dentry*>::iterator p = root->dir->dentries.begin();
+	 p != root->dir->dentries.end();
+	 ++p) {
+      if (p->second->inode)
+	_schedule_invalidate_dentry_callback(p->second, false);
+    }
+  } else if (remount_cb) {
+    // Hacky:
+    // when remounting a file system, linux kernel trims all unused dentries in the fs
     remount_finisher.queue(new C_Client_Remount(this));
+  }
 }
 
 void Client::trim_caps(MetaSession *s, int max)
@@ -3486,6 +3505,13 @@ void Client::trim_caps(MetaSession *s, int max)
       while (q != in->dn_set.end()) {
 	Dentry *dn = *q++;
 	if (dn->lru_is_expireable()) {
+	  if (can_invalidate_dentries &&
+	      dn->dir->parent_inode->ino == MDS_INO_ROOT) {
+	    // Only issue one of these per DN for inodes in root: handle
+	    // others more efficiently by calling for root-child DNs at
+	    // the end of this function.
+	    _schedule_invalidate_dentry_callback(dn, true);
+	  }
 	  trim_dentry(dn);
         } else {
           ldout(cct, 20) << "  not expirable: " << dn->name << dendl;
@@ -7987,6 +8013,34 @@ void Client::ll_register_callbacks(struct client_callback_args *args)
   getgroups_cb = args->getgroups_cb;
 }
 
+int Client::test_dentry_handling(bool can_invalidate)
+{
+  int r = 0;
+
+  can_invalidate_dentries = can_invalidate;
+
+  if (can_invalidate_dentries) {
+    assert(dentry_invalidate_cb);
+    ldout(cct, 1) << "using dentry_invalidate_cb" << dendl;
+  } else if (remount_cb) {
+    ldout(cct, 1) << "using remount_cb" << dendl;
+    int s = remount_cb(callback_handle);
+    if (s) {
+      lderr(cct) << "Failed to invoke remount, needed to ensure kernel dcache consistency"
+		 << dendl;
+    }
+    if (cct->_conf->client_die_on_failed_remount) {
+      require_remount = true;
+      r = s;
+    }
+  } else {
+    lderr(cct) << "no method to invalidate kernel dentry cache; expect issues!" << dendl;
+    if (cct->_conf->client_die_on_failed_remount)
+      assert(0);
+  }
+  return r;
+}
+
 int Client::_sync_fs()
 {
   ldout(cct, 10) << "_sync_fs" << dendl;

src/client/Client.h

@@ -136,7 +136,7 @@ typedef void (*client_ino_callback_t)(void *handle, vinodeno_t ino, int64_t off,
 
 typedef void (*client_dentry_callback_t)(void *handle, vinodeno_t dirino,
 					 vinodeno_t ino, string& name);
-typedef void (*client_remount_callback_t)(void *handle);
+typedef int (*client_remount_callback_t)(void *handle);
 
 typedef int (*client_getgroups_callback_t)(void *handle, uid_t uid, gid_t **sgids);
 typedef void(*client_switch_interrupt_callback_t)(void *req, void *data);
@@ -242,6 +242,8 @@ class Client : public Dispatcher {
   client_ino_callback_t ino_invalidate_cb;
   client_dentry_callback_t dentry_invalidate_cb;
   client_getgroups_callback_t getgroups_cb;
+  bool can_invalidate_dentries;
+  bool require_remount;
 
   Finisher async_ino_invalidator;
   Finisher async_dentry_invalidator;
@@ -955,6 +957,7 @@ class Client : public Dispatcher {
   int ll_osdaddr(int osd, char* buf, size_t size);
 
   void ll_register_callbacks(struct client_callback_args *args);
+  int test_dentry_handling(bool can_invalidate);
 };
 
 #endif

src/client/fuse_ll.cc

@@ -757,14 +757,19 @@ static void dentry_invalidate_cb(void *handle, vinodeno_t dirino,
 #endif
 }
 
-static void remount_cb(void *handle)
+static int remount_cb(void *handle)
 {
   // used for trimming kernel dcache. when remounting a file system, linux kernel
   // trims all unused dentries in the file system
   char cmd[1024];
   CephFuse::Handle *cfuse = (CephFuse::Handle *)handle;
   snprintf(cmd, sizeof(cmd), "mount -i -o remount %s", cfuse->mountpoint);
-  system(cmd);
+  int r = system(cmd);
+  if (r != 0 && r != -1) {
+    r = WEXITSTATUS(r);
+  }
+
+  return r;
 }
 
 static void do_init(void *data, fuse_conn_info *bar)
@@ -864,8 +869,6 @@ CephFuse::Handle::~Handle()
 
 void CephFuse::Handle::finalize()
 {
-  client->ll_register_callbacks(NULL);
-
   if (se)
     fuse_remove_signal_handlers(se);
   if (ch)

src/common/config_opts.h

@@ -326,6 +326,8 @@ OPTION(fuse_big_writes, OPT_BOOL, true)
 OPTION(fuse_atomic_o_trunc, OPT_BOOL, true)
 OPTION(fuse_debug, OPT_BOOL, false)
 OPTION(fuse_multithreaded, OPT_BOOL, true)
+OPTION(client_try_dentry_invalidate, OPT_BOOL, true) // the client should try to use dentry invaldation instead of remounting, on kernels it believes that will work for
+OPTION(client_die_on_failed_remount, OPT_BOOL, true)
 
 OPTION(crush_location, OPT_STR, "")       // whitespace-separated list of key=value pairs describing crush location