Merge pull request ceph#3867 from wonzhq/fiemap

FileStore: fiemap implementation using SEEK_HOLE/SEEK_DATA Reviewed-by: Haomai Wang <[email protected]> Reviewed-by: Sage Weil <[email protected]>
mathslinux · Apr 6, 2015 · 9c7ea3b · 9c7ea3b
2 parents cee47b3 + afb23f4
commit 9c7ea3b
Show file tree

Hide file tree

Showing 5 changed files with 169 additions and 62 deletions.
diff --git a/src/common/config_opts.h b/src/common/config_opts.h
@@ -822,6 +822,7 @@ OPTION(filestore_btrfs_clone_range, OPT_BOOL, true)
 OPTION(filestore_zfs_snap, OPT_BOOL, false) // zfsonlinux is still unstable
 OPTION(filestore_fsync_flushes_journal_data, OPT_BOOL, false)
 OPTION(filestore_fiemap, OPT_BOOL, false)     // (try to) use fiemap
+OPTION(filestore_seek_data_hole, OPT_BOOL, false)     // (try to) use seek_data/hole
 OPTION(filestore_fadvise, OPT_BOOL, true)
 
 // (try to) use extsize for alloc hint NOTE: extsize seems to trigger

diff --git a/src/os/FileStore.cc b/src/os/FileStore.cc
@@ -2883,74 +2883,141 @@ int FileStore::read(
   }
 }
 
-int FileStore::fiemap(coll_t cid, const ghobject_t& oid,
-                    uint64_t offset, size_t len,
-                    bufferlist& bl)
+int FileStore::_do_fiemap(int fd, uint64_t offset, size_t len,
+                          map<uint64_t, uint64_t> *m)
 {
-  tracepoint(objectstore, fiemap_enter, cid.c_str(), offset, len);
+  struct fiemap *fiemap = NULL;
+  uint64_t i;
+  struct fiemap_extent *extent = NULL;
+  int r = 0;
 
-  if (!backend->has_fiemap() || len <= (size_t)m_filestore_fiemap_threshold) {
-    map<uint64_t, uint64_t> m;
-    m[offset] = len;
-    ::encode(m, bl);
-    return 0;
+  r = backend->do_fiemap(fd, offset, len, &fiemap);
+  if (r < 0)
+    return r;
+
+  if (fiemap->fm_mapped_extents == 0) {
+    free(fiemap);
+    return r;
   }
 
+  extent = &fiemap->fm_extents[0];
 
-  struct fiemap *fiemap = NULL;
-  map<uint64_t, uint64_t> exomap;
+  /* start where we were asked to start */
+  if (extent->fe_logical < offset) {
+    extent->fe_length -= offset - extent->fe_logical;
+    extent->fe_logical = offset;
+  }
 
-  dout(15) << "fiemap " << cid << "/" << oid << " " << offset << "~" << len << dendl;
+  i = 0;
 
-  FDRef fd;
-  int r = lfn_open(cid, oid, false, &fd);
-  if (r < 0) {
-    dout(10) << "read couldn't open " << cid << "/" << oid << ": " << cpp_strerror(r) << dendl;
-  } else {
-    uint64_t i;
+  while (i < fiemap->fm_mapped_extents) {
+    struct fiemap_extent *next = extent + 1;
 
-    r = backend->do_fiemap(**fd, offset, len, &fiemap);
-    if (r < 0)
-      goto done;
+    dout(10) << "FileStore::fiemap() fm_mapped_extents=" << fiemap->fm_mapped_extents
+             << " fe_logical=" << extent->fe_logical << " fe_length=" << extent->fe_length << dendl;
 
-    if (fiemap->fm_mapped_extents == 0) {
-      free(fiemap);
-      goto done;
+    /* try to merge extents */
+    while ((i < fiemap->fm_mapped_extents - 1) &&
+           (extent->fe_logical + extent->fe_length == next->fe_logical)) {
+        next->fe_length += extent->fe_length;
+        next->fe_logical = extent->fe_logical;
+        extent = next;
+        next = extent + 1;
+        i++;
     }
 
-    struct fiemap_extent *extent = &fiemap->fm_extents[0];
-
-    /* start where we were asked to start */
-    if (extent->fe_logical < offset) {
-      extent->fe_length -= offset - extent->fe_logical;
-      extent->fe_logical = offset;
-    }
+    if (extent->fe_logical + extent->fe_length > offset + len)
+      extent->fe_length = offset + len - extent->fe_logical;
+    (*m)[extent->fe_logical] = extent->fe_length;
+    i++;
+    extent++;
+  }
+  free(fiemap);
 
-    i = 0;
+  return r;
+}
 
-    while (i < fiemap->fm_mapped_extents) {
-      struct fiemap_extent *next = extent + 1;
+int FileStore::_do_seek_hole_data(int fd, uint64_t offset, size_t len,
+                                  map<uint64_t, uint64_t> *m)
+{
+#if defined(__linux__) && defined(SEEK_HOLE) && defined(SEEK_DATA)
+  off_t hole_pos, data_pos;
+  int r = 0;
 
-      dout(10) << "FileStore::fiemap() fm_mapped_extents=" << fiemap->fm_mapped_extents
-	       << " fe_logical=" << extent->fe_logical << " fe_length=" << extent->fe_length << dendl;
+  // If lseek fails with errno setting to be ENXIO, this means the current
+  // file offset is beyond the end of the file.
+  off_t start = offset;
+  while(start < (off_t)(offset + len)) {
+    data_pos = lseek(fd, start, SEEK_DATA);
+    if (data_pos < 0) {
+      if (errno == ENXIO)
+        break;
+      else {
+        r = -errno;
+        dout(10) << "failed to lseek: " << cpp_strerror(r) << dendl;
+	return r;
+      }
+    } else if (data_pos > (off_t)(offset + len)) {
+      break;
+    }
 
-      /* try to merge extents */
-      while ((i < fiemap->fm_mapped_extents - 1) &&
-             (extent->fe_logical + extent->fe_length == next->fe_logical)) {
-          next->fe_length += extent->fe_length;
-          next->fe_logical = extent->fe_logical;
-          extent = next;
-          next = extent + 1;
-          i++;
+    hole_pos = lseek(fd, data_pos, SEEK_HOLE);
+    if (hole_pos < 0) {
+      if (errno == ENXIO) {
+        break;
+      } else {
+        r = -errno;
+        dout(10) << "failed to lseek: " << cpp_strerror(r) << dendl;
+	return r;
       }
+    }
 
-      if (extent->fe_logical + extent->fe_length > offset + len)
-        extent->fe_length = offset + len - extent->fe_logical;
-      exomap[extent->fe_logical] = extent->fe_length;
-      i++;
-      extent++;
+    if (hole_pos >= (off_t)(offset + len)) {
+      (*m)[data_pos] = offset + len - data_pos;
+      break;
     }
-    free(fiemap);
+    (*m)[data_pos] = hole_pos - data_pos;
+    start = hole_pos;
+  }
+
+  return r;
+#else
+  (*m)[offset] = len;
+  return 0;
+#endif
+}
+
+int FileStore::fiemap(coll_t cid, const ghobject_t& oid,
+                    uint64_t offset, size_t len,
+                    bufferlist& bl)
+{
+  tracepoint(objectstore, fiemap_enter, cid.c_str(), offset, len);
+
+  if ((!backend->has_seek_data_hole() && !backend->has_fiemap()) ||
+      len <= (size_t)m_filestore_fiemap_threshold) {
+    map<uint64_t, uint64_t> m;
+    m[offset] = len;
+    ::encode(m, bl);
+    return 0;
+  }
+
+  dout(15) << "fiemap " << cid << "/" << oid << " " << offset << "~" << len << dendl;
+
+  map<uint64_t, uint64_t> exomap;
+  FDRef fd;
+
+  int r = lfn_open(cid, oid, false, &fd);
+  if (r < 0) {
+    dout(10) << "read couldn't open " << cid << "/" << oid << ": " << cpp_strerror(r) << dendl;
+    goto done;
+  }
+
+  if (backend->has_seek_data_hole()) {
+    dout(15) << "seek_data/seek_hole " << cid << "/" << oid << " " << offset << "~" << len << dendl;
+    r = _do_seek_hole_data(**fd, offset, len, &exomap);
+  } else if (backend->has_fiemap()) {
+    dout(15) << "fiemap ioctl" << cid << "/" << oid << " " << offset << "~" << len << dendl;
+    r = _do_fiemap(**fd, offset, len, &exomap);
   }
 
 done:

diff --git a/src/os/FileStore.h b/src/os/FileStore.h
@@ -534,6 +534,10 @@ class FileStore : public JournalingObjectStore,
     bufferlist& bl,
     uint32_t op_flags = 0,
     bool allow_eio = false);
+  int _do_fiemap(int fd, uint64_t offset, size_t len,
+                 map<uint64_t, uint64_t> *m);
+  int _do_seek_hole_data(int fd, uint64_t offset, size_t len,
+                         map<uint64_t, uint64_t> *m);
   int fiemap(coll_t cid, const ghobject_t& oid, uint64_t offset, size_t len, bufferlist& bl);
 
   int _touch(coll_t cid, const ghobject_t& oid);
@@ -796,6 +800,7 @@ class FileStoreBackend {
   virtual int destroy_checkpoint(const string& name) = 0;
   virtual int syncfs() = 0;
   virtual bool has_fiemap() = 0;
+  virtual bool has_seek_data_hole() = 0;
   virtual int do_fiemap(int fd, off_t start, size_t len, struct fiemap **pfiemap) = 0;
   virtual int clone_range(int from, int to, uint64_t srcoff, uint64_t len, uint64_t dstoff) = 0;
   virtual int set_alloc_hint(int fd, uint64_t hint) = 0;

diff --git a/src/os/GenericFileStoreBackend.cc b/src/os/GenericFileStoreBackend.cc
@@ -57,7 +57,9 @@
 GenericFileStoreBackend::GenericFileStoreBackend(FileStore *fs):
   FileStoreBackend(fs),
   ioctl_fiemap(false),
+  seek_data_hole(false),
   m_filestore_fiemap(g_conf->filestore_fiemap),
+  m_filestore_seek_data_hole(g_conf->filestore_seek_data_hole),
   m_filestore_fsync_flushes_journal_data(g_conf->filestore_fsync_flushes_journal_data) {}
 
 int GenericFileStoreBackend::detect_features()
@@ -110,24 +112,53 @@ int GenericFileStoreBackend::detect_features()
   }
 
   // fiemap an extent inside that
-  struct fiemap *fiemap;
-  int r = do_fiemap(fd, 2430421, 59284, &fiemap);
-  if (r < 0) {
-    dout(0) << "detect_features: FIEMAP ioctl is NOT supported" << dendl;
+  if (!m_filestore_fiemap) {
+    dout(0) << "detect_features: FIEMAP ioctl is disabled via 'filestore fiemap' config option" << dendl;
     ioctl_fiemap = false;
   } else {
-    if (fiemap->fm_mapped_extents == 0) {
-      dout(0) << "detect_features: FIEMAP ioctl is supported, but buggy -- upgrade your kernel" << dendl;
+    struct fiemap *fiemap;
+    int r = do_fiemap(fd, 2430421, 59284, &fiemap);
+    if (r < 0) {
+      dout(0) << "detect_features: FIEMAP ioctl is NOT supported" << dendl;
       ioctl_fiemap = false;
     } else {
-      dout(0) << "detect_features: FIEMAP ioctl is supported and appears to work" << dendl;
-      ioctl_fiemap = true;
+      if (fiemap->fm_mapped_extents == 0) {
+        dout(0) << "detect_features: FIEMAP ioctl is supported, but buggy -- upgrade your kernel" << dendl;
+        ioctl_fiemap = false;
+      } else {
+        dout(0) << "detect_features: FIEMAP ioctl is supported and appears to work" << dendl;
+        ioctl_fiemap = true;
+      }
+      free(fiemap);
     }
-    free(fiemap);
   }
-  if (!m_filestore_fiemap) {
-    dout(0) << "detect_features: FIEMAP ioctl is disabled via 'filestore fiemap' config option" << dendl;
-    ioctl_fiemap = false;
+
+  // SEEK_DATA/SEEK_HOLE detection
+  if (!m_filestore_seek_data_hole) {
+    dout(0) << "detect_features: SEEK_DATA/SEEK_HOLE is disabled via 'filestore seek data hole' config option" << dendl;
+    seek_data_hole = false;
+  } else {
+#if defined(__linux__) && defined(SEEK_HOLE) && defined(SEEK_DATA)
+    // If compiled on an OS with SEEK_HOLE/SEEK_DATA support, but running
+    // on an OS that doesn't support SEEK_HOLE/SEEK_DATA, EINVAL is returned.
+    // Fall back to use fiemap.
+    off_t hole_pos;
+
+    hole_pos = lseek(fd, 0, SEEK_HOLE);
+    if (hole_pos < 0) {
+      if (errno == EINVAL) {
+        dout(0) << "detect_features: lseek SEEK_DATA/SEEK_HOLE is NOT supported" << dendl;
+        seek_data_hole = false;
+      } else {
+        derr << "detect_features: failed to lseek " << fn << ": " << cpp_strerror(-errno) << dendl;
+        VOID_TEMP_FAILURE_RETRY(::close(fd));
+        return -errno;
+      }
+    } else {
+      dout(0) << "detect_features: lseek SEEK_DATA/SEEK_HOLE is supported" << dendl;
+      seek_data_hole = true;
+    }
+#endif
   }
 
   ::unlink(fn);

diff --git a/src/os/GenericFileStoreBackend.h b/src/os/GenericFileStoreBackend.h
@@ -22,7 +22,9 @@ class SloppyCRCMap;
 class GenericFileStoreBackend : public FileStoreBackend {
 private:
   bool ioctl_fiemap;
+  bool seek_data_hole;
   bool m_filestore_fiemap;
+  bool m_filestore_seek_data_hole;
   bool m_filestore_fsync_flushes_journal_data;
 public:
   GenericFileStoreBackend(FileStore *fs);
@@ -41,6 +43,7 @@ class GenericFileStoreBackend : public FileStoreBackend {
   virtual int destroy_checkpoint(const string& name) { return -EOPNOTSUPP; }
   virtual int syncfs();
   virtual bool has_fiemap() { return ioctl_fiemap; }
+  virtual bool has_seek_data_hole() { return seek_data_hole; }
   virtual int do_fiemap(int fd, off_t start, size_t len, struct fiemap **pfiemap);
   virtual int clone_range(int from, int to, uint64_t srcoff, uint64_t len, uint64_t dstoff) {
     return _copy_range(from, to, srcoff, len, dstoff);