fuse: whole-file write throughput test

src/fuse/ustack/CMakeLists.txt

@@ -9,7 +9,8 @@ project(fuse-ustack CXX)
 
 include_directories(${Nanomsg_INCLUDE_DIRS} ${TBB_INCLUDE_DIRS} ${Flatbuffers_INCLUDE_DIRS} ${CMAKE_CURRENT_BINARY_DIR}/src)
 
-add_definitions(${GCC_COVERAGE_COMPILE_FLAGS} -DCONFIG_DEBUG -D_FILE_OFFSET_BITS=64)
+add_definitions(${GCC_COVERAGE_COMPILE_FLAGS} -D_FILE_OFFSET_BITS=64)
+#add_definitions(${GCC_COVERAGE_COMPILE_FLAGS} -DCONFIG_DEBUG -D_FILE_OFFSET_BITS=64)
 
 # if not having this, src won't be in the build dir
 configure_file(

src/fuse/ustack/src/kv_ustack.cpp

@@ -275,7 +275,7 @@ static int kvfs_ustack_read(const char *path, char *buf, size_t size, off_t offs
     return -1;
   }
 
-  PINF("[%s]: read value %s from path %s ", __func__, buf, path);
+  PLOG("[%s]: read value %s from path %s ", __func__, buf, path);
 
   return size;
 }
@@ -293,7 +293,7 @@ int (kvfs_ustack_write) (const char *path, const char *buf, size_t size, off_t o
 
   uint64_t id;
 
-  PINF("[%s]: write content %s to path %s", __func__,buf, path);
+  PLOG("[%s]: write content %s to path %s", __func__,buf, path);
 
   KV_ustack_info *info = reinterpret_cast<KV_ustack_info *>(fuse_get_context()->private_data);
 

src/fuse/ustack/src/ustack.cpp

@@ -293,7 +293,7 @@ void Ustack::uipc_channel_thread_entry(Core::UIPC::Channel * channel, pid_t clie
       break;
     if(!msg) continue;
 
-    PMAJOR("recv'ed UIPC msg:status=%d, type=%d (%s)",s, msg->type, msg->data);
+    PLOG("recv'ed UIPC msg:status=%d, type=%d (%s)",s, msg->type, msg->data);
 
     assert(msg);
     switch(msg->type){

src/fuse/ustack/src/ustack.h

@@ -53,6 +53,7 @@ class Ustack : public Core::IPC_server {
     //get the virtual address and issue the io
     void *buf; //the mapped io mem
 
+    // TODO: need to control offset 
     auto iomem_list = _iomem_map[client_id] ;
     if(iomem_list.empty()){
       PERR("iomem for pid %d is empty", client_id);

src/fuse/ustack/unit_test/test_preload.cpp

@@ -4,30 +4,82 @@
 #include <sys/types.h>  //open
 #include <unistd.h>     // close
 #include <cstdlib>
+#include <string>
+#include <chrono> /* milliseconds */
+#include <common/utils.h>
 /** Preload the ustack libray and those operations will be intecepted
  * 1. malloc free
- * 2. open/read/write*/
-static const char* filepath = "foobar.dat";
-int                main()
+ * 2. open/read/write
+ * Without preload: ./src/fuse/ustack/unit_test/test-preload /tmp/mymount/
+ * With preload:  LD_PRELOAD=./src/fuse/ustack/libustack_client.so ./src/fuse/ustack/unit_test/test-preload /tmp/mymount */
+
+static size_t file_size=KB(4);
+static constexpr unsigned nr_buffer_copies=3; // make sure each time using different buffer
+static constexpr unsigned ITERATIONS = 1000;
+
+int                main(int argc, char * argv[])
 {
-  int fd = open(filepath, O_WRONLY | O_CREAT, S_IRWXU);
+
+  std::chrono::high_resolution_clock::time_point _start_time, _end_time;
+
+  if(argc != 3){
+    PERR("command format: (LD_PRELOAD=pathto-libfuseclient.so)./test_preload mount_path iosize(KB)")
+    return -1;
+  }
+  file_size= KB(atoi(argv[2]));
+  int open_flags = O_WRONLY | O_CREAT | O_DIRECT |O_SYNC;
+  // int open_flags = O_WRONLY | O_CREAT  |O_SYNC;
+
+  int use_preload = (getenv("LD_PRELOAD"))?1:0;
+  std::string dir_name(argv[1]);
+  std::string method_str = "preload"+ std::to_string(use_preload) + "-sz" + std::to_string(file_size) + "-f" + std::to_string(open_flags) + "-iter"+ std::to_string(ITERATIONS);
+
+  std::string filepath = dir_name + "/foobar-" + method_str +".dat";
+
+  PMAJOR("Using file (%s), size %lu)", filepath.c_str(), file_size);
+
+  int fd = open(filepath.c_str(), open_flags, S_IRWXU);
   if (fd == -1) {
-    PERR("Open failed, delete file (%s)first", filepath);
+    PERR("Open failed, delete file (%s)first", filepath.c_str());
     return -1;
   }
-  void* ptr = malloc(4096);
-  memset(ptr, 'a', 4096);
+  void * buffer;
+  // buffer = malloc(file_size*nr_buffer_copies);
+  if(posix_memalign(&buffer, 4096, file_size*nr_buffer_copies)){PERR("aligned alloc files"); return -1;}
+
+  for(unsigned i = 0, offset=0; i < nr_buffer_copies; i+= 1){
+    memset((char *)buffer+offset, 'a'+i, file_size);
+    offset += file_size;
+  }
 
 #if 0
   int fd = open("foobar.dat", O_SYNC | O_CREAT | O_TRUNC, O_WRONLY);
   assert(fd != -1);
 
 #endif
-  for (unsigned i = 0; i < 1; i++) {
-    ssize_t res = write(fd, ptr, 4096);
+  _start_time = std::chrono::high_resolution_clock::now();
+  for (unsigned i = 0; i < ITERATIONS; i++) {
+    char* ptr = (char*)buffer  + file_size*(i % nr_buffer_copies);
+    ssize_t res = write(fd, ptr, file_size);
+    if(res > file_size || res < 0){
+      PWRN("write return %lu in iteration %u, ptr = %p", res, i, ptr);
+    }
+    fsync(fd);
+    lseek(fd, 0, SEEK_SET);
   }
+  _end_time = std::chrono::high_resolution_clock::now();
+
+  double secs = std::chrono::duration_cast<std::chrono::milliseconds>(
+                      _end_time - _start_time)
+                      .count() /
+                  1000.0;
+  double iops = ((double) ITERATIONS) / secs;
+  double throughput_in_mb = iops*file_size/(MB(1));
+
+  PMAJOR("iops(%.1f), throughput(%.3f MB/s), iterations(%u)", iops, throughput_in_mb, ITERATIONS);
 
   close(fd);
+  free(buffer);
   PLOG("done!");
   return 0;
 }