tools,common,msg: Use O_BINARY for win32 compatibility

Unless O_BINARY is passed when opening files, Windows will translate newlines and handle CTRL+z (Alt+026) as EOF, which can abruptly end file reads. We'll define O_BINARY as 0 on other platforms and explicitly use O_BINARY when expecting binary files. Signed-off-by: Lucian Petrut <[email protected]>
amincheloh · Oct 22, 2020 · 0c57edc · 0c57edc
1 parent 8e10578
commit 0c57edc
Show file tree

Hide file tree

Showing 17 changed files with 40 additions and 32 deletions.
diff --git a/src/common/buffer.cc b/src/common/buffer.cc
@@ -1685,7 +1685,7 @@ void buffer::list::decode_base64(buffer::list& e)
 
 ssize_t buffer::list::pread_file(const char *fn, uint64_t off, uint64_t len, std::string *error)
 {
-  int fd = TEMP_FAILURE_RETRY(::open(fn, O_RDONLY|O_CLOEXEC));
+  int fd = TEMP_FAILURE_RETRY(::open(fn, O_RDONLY|O_CLOEXEC|O_BINARY));
   if (fd < 0) {
     int err = errno;
     std::ostringstream oss;
@@ -1745,7 +1745,7 @@ ssize_t buffer::list::pread_file(const char *fn, uint64_t off, uint64_t len, std
 
 int buffer::list::read_file(const char *fn, std::string *error)
 {
-  int fd = TEMP_FAILURE_RETRY(::open(fn, O_RDONLY|O_CLOEXEC));
+  int fd = TEMP_FAILURE_RETRY(::open(fn, O_RDONLY|O_CLOEXEC|O_BINARY));
   if (fd < 0) {
     int err = errno;
     std::ostringstream oss;
@@ -1812,7 +1812,7 @@ ssize_t buffer::list::recv_fd(int fd, size_t len)
 
 int buffer::list::write_file(const char *fn, int mode)
 {
-  int fd = TEMP_FAILURE_RETRY(::open(fn, O_WRONLY|O_CREAT|O_TRUNC|O_CLOEXEC, mode));
+  int fd = TEMP_FAILURE_RETRY(::open(fn, O_WRONLY|O_CREAT|O_TRUNC|O_CLOEXEC|O_BINARY, mode));
   if (fd < 0) {
     int err = errno;
     cerr << "bufferlist::write_file(" << fn << "): failed to open file: "

diff --git a/src/common/safe_io.c b/src/common/safe_io.c
@@ -237,7 +237,7 @@ int safe_write_file(const char *base, const char *file,
 
   snprintf(fn, sizeof(fn), "%s/%s", base, file);
   snprintf(tmp, sizeof(tmp), "%s/%s.tmp", base, file);
-  fd = open(tmp, O_WRONLY|O_CREAT|O_TRUNC, mode);
+  fd = open(tmp, O_WRONLY|O_CREAT|O_TRUNC|O_BINARY, mode);
   if (fd < 0) {
     ret = errno;
     return -ret;
@@ -262,7 +262,7 @@ int safe_write_file(const char *base, const char *file,
     return ret;
   }
 
-  fd = open(base, O_RDONLY);
+  fd = open(base, O_RDONLY|O_BINARY);
   if (fd < 0) {
     ret = -errno;
     return ret;
@@ -281,7 +281,7 @@ int safe_read_file(const char *base, const char *file,
   int fd, len;
 
   snprintf(fn, sizeof(fn), "%s/%s", base, file);
-  fd = open(fn, O_RDONLY);
+  fd = open(fn, O_RDONLY|O_BINARY);
   if (fd < 0) {
     return -errno;
   }

diff --git a/src/include/compat.h b/src/include/compat.h
@@ -355,4 +355,11 @@ static inline int ceph_sock_errno() {
 #endif
 }
 
+// Needed on Windows when handling binary files. Without it, line
+// endings will be replaced and certain characters can be treated as
+// EOF.
+#ifndef O_BINARY
+#define O_BINARY 0
+#endif
+
 #endif /* !CEPH_COMPAT_H */
diff --git a/src/include/denc.h b/src/include/denc.h
@@ -118,7 +118,7 @@ class DencDumper {
     ::snprintf(fn, sizeof(fn),
 	       ENCODE_STRINGIFY(ENCODE_DUMP_PATH) "/%s__%d.%x", name,
 	       getpid(), i++);
-    int fd = ::open(fn, O_WRONLY|O_TRUNC|O_CREAT|O_CLOEXEC, 0644);
+    int fd = ::open(fn, O_WRONLY|O_TRUNC|O_CREAT|O_CLOEXEC|O_BINARY, 0644);
     if (fd < 0) {
       return;
     }

diff --git a/src/include/encoding.h b/src/include/encoding.h
@@ -171,7 +171,7 @@ WRITE_FLTTYPE_ENCODER(double, uint64_t, le64)
       break;								\
     char fn[PATH_MAX];							\
     snprintf(fn, sizeof(fn), ENCODE_STRINGIFY(ENCODE_DUMP_PATH) "/%s__%d.%x", #cl, getpid(), i++); \
-    int fd = ::open(fn, O_WRONLY|O_TRUNC|O_CREAT|O_CLOEXEC, 0644);		\
+    int fd = ::open(fn, O_WRONLY|O_TRUNC|O_CREAT|O_CLOEXEC|O_BINARY, 0644);		\
     if (fd >= 0) {							\
       ::ceph::bufferlist sub;						\
       sub.substr_of(bl, pre_off, bl.length() - pre_off);		\

diff --git a/src/msg/Message.cc b/src/msg/Message.cc
@@ -283,7 +283,7 @@ void Message::encode(uint64_t features, int crcflags, bool skip_header_crc)
       snprintf(fn, sizeof(fn), ENCODE_STRINGIFY(ENCODE_DUMP) "/%s__%d.%x",
 	       abi::__cxa_demangle(typeid(*this).name(), 0, 0, &status),
 	       getpid(), i++);
-      int fd = ::open(fn, O_WRONLY|O_TRUNC|O_CREAT|O_CLOEXEC, 0644);
+      int fd = ::open(fn, O_WRONLY|O_TRUNC|O_CREAT|O_CLOEXEC|O_BINARY, 0644);
       if (fd >= 0) {
 	bl.write_fd(fd);
 	::close(fd);

diff --git a/src/rbd_replay/Replayer.cc b/src/rbd_replay/Replayer.cc
@@ -214,7 +214,7 @@ void Replayer::run(const std::string& replay_file) {
       m_rbd = new librbd::RBD();
       map<thread_id_t, Worker*> workers;
 
-      int fd = open(replay_file.c_str(), O_RDONLY);
+      int fd = open(replay_file.c_str(), O_RDONLY|O_BINARY);
       if (fd < 0) {
         std::cerr << "Failed to open " << replay_file << ": "
                   << cpp_strerror(errno) << std::endl;

diff --git a/src/rbd_replay/rbd-replay-prep.cc b/src/rbd_replay/rbd-replay-prep.cc
@@ -15,6 +15,7 @@
 // This code assumes that IO IDs and timestamps are related monotonically.
 // In other words, (a.id < b.id) == (a.timestamp < b.timestamp) for all IOs a and b.
 
+#include "include/compat.h"
 #include "common/errno.h"
 #include "rbd_replay/ActionTypes.h"
 #include <babeltrace/babeltrace.h>
@@ -198,7 +199,7 @@ class Processor {
 
     struct bt_iter *bt_itr = bt_ctf_get_iter(itr);
 
-    int fd = open(output_file_name.c_str(), O_WRONLY | O_CREAT | O_EXCL, 0644);
+    int fd = open(output_file_name.c_str(), O_WRONLY | O_CREAT | O_EXCL | O_BINARY, 0644);
     ASSERT_EXIT(fd >= 0, "Error opening output file " << output_file_name <<
                          ": " << cpp_strerror(errno));
     BOOST_SCOPE_EXIT( (fd) ) {

diff --git a/src/tools/ceph-dencoder/ceph_dencoder.cc b/src/tools/ceph-dencoder/ceph_dencoder.cc
@@ -185,7 +185,7 @@ int main(int argc, const char **argv)
 	cerr << "expecting filename" << std::endl;
 	exit(1);
       }
-      int fd = ::open(*i, O_WRONLY|O_CREAT|O_TRUNC, 0644);
+      int fd = ::open(*i, O_WRONLY|O_CREAT|O_TRUNC|O_BINARY, 0644);
       if (fd < 0) {
 	cerr << "error opening " << *i << " for write: " << cpp_strerror(errno) << std::endl;
 	exit(1);

diff --git a/src/tools/ceph_monstore_tool.cc b/src/tools/ceph_monstore_tool.cc
@@ -43,7 +43,7 @@ class TraceIter {
   MonitorDBStore::TransactionRef t;
 public:
   explicit TraceIter(string fname) : fd(-1), idx(-1) {
-    fd = ::open(fname.c_str(), O_RDONLY);
+    fd = ::open(fname.c_str(), O_RDONLY|O_BINARY);
     t.reset(new MonitorDBStore::Transaction);
   }
   bool valid() {
@@ -879,7 +879,7 @@ int main(int argc, char **argv) {
 
     int fd = STDOUT_FILENO;
     if (!outpath.empty()){
-      fd = ::open(outpath.c_str(), O_WRONLY|O_CREAT|O_TRUNC, 0666);
+      fd = ::open(outpath.c_str(), O_WRONLY|O_CREAT|O_TRUNC|O_BINARY, 0666);
       if (fd < 0) {
         std::cerr << "error opening output file: "
           << cpp_strerror(errno) << std::endl;

diff --git a/src/tools/cephfs/Dumper.cc b/src/tools/cephfs/Dumper.cc
@@ -104,7 +104,7 @@ int Dumper::dump(const char *dump_file)
 
   cout << "journal is " << start << "~" << len << std::endl;
 
-  int fd = ::open(dump_file, O_WRONLY|O_CREAT|O_TRUNC, 0644);
+  int fd = ::open(dump_file, O_WRONLY|O_CREAT|O_TRUNC|O_BINARY, 0644);
   if (fd >= 0) {
     // include an informative header
     uuid_d fsid = monc->get_fsid();
@@ -213,7 +213,7 @@ int Dumper::undump(const char *dump_file, bool force)
     derr << "recover_journal failed, try to get header from dump file " << dendl;
   }
 
-  int fd = ::open(dump_file, O_RDONLY);
+  int fd = ::open(dump_file, O_RDONLY|O_BINARY);
   if (fd < 0) {
     r = errno;
     derr << "couldn't open " << dump_file << ": " << cpp_strerror(r) << dendl;

diff --git a/src/tools/rados/rados.cc b/src/tools/rados/rados.cc
@@ -445,7 +445,7 @@ static int dump_data(std::string const &filename, bufferlist const &data)
   if (filename == "-") {
     fd = STDOUT_FILENO;
   } else {
-    fd = TEMP_FAILURE_RETRY(::open(filename.c_str(), O_WRONLY|O_CREAT|O_TRUNC, 0644));
+    fd = TEMP_FAILURE_RETRY(::open(filename.c_str(), O_WRONLY|O_CREAT|O_TRUNC|O_BINARY, 0644));
     if (fd < 0) {
       int err = errno;
       cerr << "failed to open file: " << cpp_strerror(err) << std::endl;
@@ -471,7 +471,7 @@ static int do_get(IoCtx& io_ctx, const char *objname, const char *outfile, unsig
   if (strcmp(outfile, "-") == 0) {
     fd = STDOUT_FILENO;
   } else {
-    fd = TEMP_FAILURE_RETRY(::open(outfile, O_WRONLY|O_CREAT|O_TRUNC, 0644));
+    fd = TEMP_FAILURE_RETRY(::open(outfile, O_WRONLY|O_CREAT|O_TRUNC|O_BINARY, 0644));
     if (fd < 0) {
       int err = errno;
       cerr << "failed to open file: " << cpp_strerror(err) << std::endl;
@@ -568,7 +568,7 @@ static int do_put(IoCtx& io_ctx,
   int ret = 0;
   int fd = STDIN_FILENO;
   if (!stdio)
-    fd = open(infile, O_RDONLY);
+    fd = open(infile, O_RDONLY|O_BINARY);
   if (fd < 0) {
     cerr << "error reading input file " << infile << ": " << cpp_strerror(errno) << std::endl;
     return 1;
@@ -629,7 +629,7 @@ static int do_append(IoCtx& io_ctx,
   int ret = 0;
   int fd = STDIN_FILENO;
   if (!stdio)
-    fd = open(infile, O_RDONLY);
+    fd = open(infile, O_RDONLY|O_BINARY);
   if (fd < 0) {
     cerr << "error reading input file " << infile << ": " << cpp_strerror(errno) << std::endl;
     return 1;
@@ -3855,7 +3855,7 @@ static int rados_tool_common(const std::map < std::string, std::string > &opts,
     if (nargs.size() < 2 || std::string(nargs[1]) == "-") {
       file_fd = STDOUT_FILENO;
     } else {
-      file_fd = open(nargs[1], O_WRONLY|O_CREAT|O_TRUNC, 0666);
+      file_fd = open(nargs[1], O_WRONLY|O_CREAT|O_TRUNC|O_BINARY, 0666);
       if (file_fd < 0) {
         cerr << "Error opening '" << nargs[1] << "': "
           << cpp_strerror(file_fd) << std::endl;
@@ -3904,7 +3904,7 @@ static int rados_tool_common(const std::map < std::string, std::string > &opts,
     if (filename == "-") {
       file_fd = STDIN_FILENO;
     } else {
-      file_fd = open(filename.c_str(), O_RDONLY);
+      file_fd = open(filename.c_str(), O_RDONLY|O_BINARY);
       if (file_fd < 0) {
         cerr << "Error opening '" << filename << "': "
           << cpp_strerror(file_fd) << std::endl;

diff --git a/src/tools/rbd/action/Export.cc b/src/tools/rbd/action/Export.cc
@@ -229,7 +229,7 @@ int do_export_diff(librbd::Image& image, const char *fromsnapname,
   if (strcmp(path, "-") == 0)
     fd = STDOUT_FILENO;
   else
-    fd = open(path, O_WRONLY | O_CREAT | O_EXCL, 0644);
+    fd = open(path, O_WRONLY | O_CREAT | O_EXCL | O_BINARY, 0644);
   if (fd < 0)
     return -errno;
 
@@ -565,7 +565,7 @@ static int do_export(librbd::Image& image, const char *path, bool no_progress,
   if (to_stdout) {
     fd = STDOUT_FILENO;
   } else {
-    fd = open(path, O_WRONLY | O_CREAT | O_EXCL, 0644);
+    fd = open(path, O_WRONLY | O_CREAT | O_EXCL | O_BINARY, 0644);
     if (fd < 0) {
       return -errno;
     }

diff --git a/src/tools/rbd/action/Import.cc b/src/tools/rbd/action/Import.cc
@@ -449,7 +449,7 @@ int do_import_diff(librados::Rados &rados, librbd::Image &image,
   if (strcmp(path, "-") == 0) {
     fd = STDIN_FILENO;
   } else {
-    fd = open(path, O_RDONLY);
+    fd = open(path, O_RDONLY|O_BINARY);
     if (fd < 0) {
       r = -errno;
       std::cerr << "rbd: error opening " << path << std::endl;
@@ -843,7 +843,7 @@ static int do_import(librados::Rados &rados, librbd::RBD &rbd,
     fd = STDIN_FILENO;
     size = 1ULL << order;
   } else {
-    if ((fd = open(path, O_RDONLY)) < 0) {
+    if ((fd = open(path, O_RDONLY|O_BINARY)) < 0) {
       r = -errno;
       std::cerr << "rbd: error opening " << path << std::endl;
       goto done2;

diff --git a/src/tools/rbd/action/Journal.cc b/src/tools/rbd/action/Journal.cc
@@ -709,7 +709,7 @@ static int do_export_journal(librados::IoCtx& io_ctx,
   if (to_stdout) {
     fd = STDOUT_FILENO;
   } else {
-    fd = open(path.c_str(), O_WRONLY | O_CREAT | O_EXCL, 0644);
+    fd = open(path.c_str(), O_WRONLY | O_CREAT | O_EXCL | O_BINARY, 0644);
     if (fd < 0) {
       r = -errno;
       std::cerr << "rbd: error creating " << path << std::endl;
@@ -920,7 +920,7 @@ static int do_import_journal(librados::IoCtx& io_ctx,
   if (from_stdin) {
     fd = STDIN_FILENO;
   } else {
-    if ((fd = open(path.c_str(), O_RDONLY)) < 0) {
+    if ((fd = open(path.c_str(), O_RDONLY|O_BINARY)) < 0) {
       r = -errno;
       std::cerr << "rbd: error opening " << path << std::endl;
       return r;

diff --git a/src/tools/rbd/action/MergeDiff.cc b/src/tools/rbd/action/MergeDiff.cc
@@ -173,15 +173,15 @@ static int do_merge_diff(const char *first, const char *second,
   if (first_stdin) {
     fd = STDIN_FILENO;
   } else {
-    fd = open(first, O_RDONLY);
+    fd = open(first, O_RDONLY|O_BINARY);
     if (fd < 0) {
       r = -errno;
       std::cerr << "rbd: error opening " << first << std::endl;
       goto done;
     }
   }
 
-  sd = open(second, O_RDONLY);
+  sd = open(second, O_RDONLY|O_BINARY);
   if (sd < 0) {
     r = -errno;
     std::cerr << "rbd: error opening " << second << std::endl;
@@ -191,7 +191,7 @@ static int do_merge_diff(const char *first, const char *second,
   if (strcmp(path, "-") == 0) {
     pd = 1;
   } else {
-    pd = open(path, O_WRONLY | O_CREAT | O_EXCL, 0644);
+    pd = open(path, O_WRONLY | O_CREAT | O_EXCL | O_BINARY, 0644);
     if (pd < 0) {
       r = -errno;
       std::cerr << "rbd: error create " << path << std::endl;

diff --git a/src/tools/rbd/action/MirrorPool.cc b/src/tools/rbd/action/MirrorPool.cc
@@ -911,7 +911,7 @@ int execute_peer_bootstrap_import(
 
   int fd = STDIN_FILENO;
   if (token_path != "-") {
-    fd = open(token_path.c_str(), O_RDONLY);
+    fd = open(token_path.c_str(), O_RDONLY|O_BINARY);
     if (fd < 0) {
       r = -errno;
       std::cerr << "rbd: error opening " << token_path << std::endl;