rdma client use a zero-length sge to get IMM ACK

src/infinistore.cpp

@@ -271,8 +271,10 @@ void Client::cq_poll_handle(uv_poll_t *handle, int status, int events) {
             }
             else if (wc.opcode ==
                      IBV_WC_RECV_RDMA_WITH_IMM) {  // write cache: we alreay have all data now.
-                INFO("write cache completed successfully, #keys {} saved", wc.imm_data);
-                INFO("ready for next request");
+
+                // client should not use WRITE_WITH_IMM to notify.
+                // it should use COMMIT message to notify.
+                WARN("WRITE_WITH_IMM is not supported in server side");
                 if (prepare_recv_rdma_request(wc.wr_id) < 0) {
                     ERROR("Failed to prepare recv rdma request");
                     return;

src/libinfinistore.cpp

@@ -953,10 +953,16 @@ int r_rdma(connection_t *conn, std::vector<block_t> &blocks, int block_size, voi
     conn->rdma_inflight_count++;
 
     // recv ACK for whole batch.
+    struct ibv_sge recv_sge = {
+        .addr = (uintptr_t)conn->recv_buffer,
+        .length = 0,
+        .lkey = conn->recv_mr->lkey,
+    };
     struct ibv_recv_wr recv_wr = {
-        .wr_id = (uintptr_t)mr,
-        .sg_list = NULL,
-        .num_sge = 0,
+        .wr_id = 0,
+        .next = NULL,
+        .sg_list = &recv_sge,
+        .num_sge = 1,
     };
 
     struct ibv_recv_wr *bad_wr_recv = NULL;