From 7e3eae583c20869efebb1e89c8b1e4c407e0a797 Mon Sep 17 00:00:00 2001 From: zhangjiao Date: Fri, 8 Nov 2019 21:09:31 +0800 Subject: [PATCH 1/3] pybind/rados: add Ioctx::writesame() and test Ioctx::writesame() Signed-off-by: Zhang Jiao --- src/pybind/rados/rados.pyx | 34 ++++++++++++++++++++++++++++++++++ src/test/pybind/test_rados.py | 4 ++++ 2 files changed, 38 insertions(+) diff --git a/src/pybind/rados/rados.pyx b/src/pybind/rados/rados.pyx index 4ad8152d1ba73..3093bb265f0ed 100644 --- a/src/pybind/rados/rados.pyx +++ b/src/pybind/rados/rados.pyx @@ -214,6 +214,7 @@ cdef extern from "rados/librados.h" nogil: int rados_stat(rados_ioctx_t io, const char *o, uint64_t *psize, time_t *pmtime) int rados_write(rados_ioctx_t io, const char *oid, const char *buf, size_t len, uint64_t off) int rados_write_full(rados_ioctx_t io, const char *oid, const char *buf, size_t len) + int rados_writesame(rados_ioctx_t io, const char *oid, const char *buf, size_t data_len, size_t write_len, uint64_t off) int rados_append(rados_ioctx_t io, const char *oid, const char *buf, size_t len) int rados_read(rados_ioctx_t io, const char *oid, char *buf, size_t len, uint64_t off) int rados_remove(rados_ioctx_t io, const char *oid) @@ -2795,6 +2796,39 @@ returned %d, but should return zero on success." % (self.name, ret)) raise LogicError("Ioctx.write_full(%s): rados_write_full \ returned %d, but should return zero on success." % (self.name, ret)) + @requires(('key', str_type), ('data', bytes), ('write_len', int), ('offset', int)) + def writesame(self, key, data, write_len, offset=0): + """ + Write the same buffer multiple times + :param key: name of the object + :type key: str + :param data: data to write + :type data: bytes + :param write_len: total number of bytes to write + :type write_len: int + :param offset: byte offset in the object to begin writing at + :type offset: int + + :raises: :class:`TypeError` + :raises: :class:`LogicError` + """ + self.require_ioctx_open() + + key = cstr(key, 'key') + cdef: + char *_key = key + char *_data = data + size_t _data_len = len(data) + size_t _write_len = write_len + uint64_t _offset = offset + + with nogil: + ret = rados_writesame(self.io, _key, _data, _data_len, _write_len, _offset) + if ret < 0: + raise make_ex(ret, "Ioctx.writesame(%s): failed to write %s" + % (self.name, key)) + assert(ret == 0) + @requires(('key', str_type), ('data', bytes)) def append(self, key, data): """ diff --git a/src/test/pybind/test_rados.py b/src/test/pybind/test_rados.py index 285d0a1a900ad..8542aea877085 100644 --- a/src/test/pybind/test_rados.py +++ b/src/test/pybind/test_rados.py @@ -316,6 +316,10 @@ def test_write_full(self): self.ioctx.write_full('abc', b'd') eq(self.ioctx.read('abc'), b'd') + def test_writesame(self): + self.ioctx.writesame('ob', b'rzx', 9) + eq(self.ioctx.read('ob'), b'rzxrzxrzx') + def test_append(self): self.ioctx.write('abc', b'a') self.ioctx.append('abc', b'b') From 8d2e311f470d23294dfd9e1c3da7c90743058368 Mon Sep 17 00:00:00 2001 From: zhangjiao Date: Fri, 8 Nov 2019 23:32:54 +0800 Subject: [PATCH 2/3] pybind/rados: add Ioctx::aio_writesame() and test Ioctx::aio_writesame() Signed-off-by: Zhang Jiao --- src/pybind/rados/rados.pyx | 44 +++++++++++++++++++++++++++++++++++ src/test/pybind/test_rados.py | 17 ++++++++++++++ 2 files changed, 61 insertions(+) diff --git a/src/pybind/rados/rados.pyx b/src/pybind/rados/rados.pyx index 3093bb265f0ed..64611b61ca665 100644 --- a/src/pybind/rados/rados.pyx +++ b/src/pybind/rados/rados.pyx @@ -275,6 +275,7 @@ cdef extern from "rados/librados.h" nogil: int rados_aio_write(rados_ioctx_t io, const char * oid, rados_completion_t completion, const char * buf, size_t len, uint64_t off) int rados_aio_append(rados_ioctx_t io, const char * oid, rados_completion_t completion, const char * buf, size_t len) int rados_aio_write_full(rados_ioctx_t io, const char * oid, rados_completion_t completion, const char * buf, size_t len) + int rados_aio_writesame(rados_ioctx_t io, const char *oid, rados_completion_t completion, const char *buf, size_t data_len, size_t write_len, uint64_t off) int rados_aio_remove(rados_ioctx_t io, const char * oid, rados_completion_t completion) int rados_aio_read(rados_ioctx_t io, const char * oid, rados_completion_t completion, char * buf, size_t len, uint64_t off) int rados_aio_flush(rados_ioctx_t io) @@ -2409,6 +2410,49 @@ cdef class Ioctx(object): raise make_ex(ret, "error writing object %s" % object_name) return completion + @requires(('object_name', str_type), ('to_write', bytes), ('write_len', int), + ('offset', int), ('oncomplete', opt(Callable))) + def aio_writesame(self, object_name, to_write, write_len, offset=0, + oncomplete=None): + """ + Asynchronously write the same buffer multiple times + + :param object_name: name of the object + :type object_name: str + :param to_write: data to write + :type to_write: bytes + :param write_len: total number of bytes to write + :type write_len: int + :param offset: byte offset in the object to begin writing at + :type offset: int + :param oncomplete: what to do when the writesame is safe and + complete in memory on all replicas + :type oncomplete: completion + :raises: :class:`Error` + :returns: completion object + """ + + object_name = cstr(object_name, 'object_name') + + cdef: + Completion completion + char* _object_name = object_name + char* _to_write = to_write + size_t _data_len = len(to_write) + size_t _write_len = write_len + uint64_t _offset = offset + + completion = self.__get_completion(oncomplete, None) + self.__track_completion(completion) + with nogil: + ret = rados_aio_writesame(self.io, _object_name, completion.rados_comp, + _to_write, _data_len, _write_len, _offset) + + if ret < 0: + completion._cleanup() + raise make_ex(ret, "error writing object %s" % object_name) + return completion + @requires(('object_name', str_type), ('to_append', bytes), ('oncomplete', opt(Callable)), ('onsafe', opt(Callable))) def aio_append(self, object_name, to_append, oncomplete=None, onsafe=None): diff --git a/src/test/pybind/test_rados.py b/src/test/pybind/test_rados.py index 8542aea877085..705629ce19ab4 100644 --- a/src/test/pybind/test_rados.py +++ b/src/test/pybind/test_rados.py @@ -699,6 +699,23 @@ def cb(blah): eq(contents, b"bar") [i.remove() for i in self.ioctx.list_objects()] + def test_aio_writesame(self): + lock = threading.Condition() + count = [0] + def cb(blah): + with lock: + count[0] += 1 + lock.notify() + return 0 + comp = self.ioctx.aio_writesame("abc", b"rzx", 9, 0, cb) + comp.wait_for_complete() + with lock: + while count[0] < 1: + lock.wait() + eq(comp.get_return_value(), 0) + eq(self.ioctx.read("abc"), b"rzxrzxrzx") + [i.remove() for i in self.ioctx.list_objects()] + def test_aio_stat(self): lock = threading.Condition() count = [0] From 65cbfc850845bc736b9d1ee6f9f4aaf7df3df2ab Mon Sep 17 00:00:00 2001 From: zhangjiao Date: Tue, 3 Dec 2019 12:21:57 +0800 Subject: [PATCH 3/3] pybind/rados: add WriteOp::writesame() and test WriteOp::writesame() Signed-off-by: Zhang Jiao --- src/pybind/rados/rados.pyx | 20 ++++++++++++++++++++ src/test/pybind/test_rados.py | 6 ++++++ 2 files changed, 26 insertions(+) diff --git a/src/pybind/rados/rados.pyx b/src/pybind/rados/rados.pyx index 64611b61ca665..3a69700a5f4bb 100644 --- a/src/pybind/rados/rados.pyx +++ b/src/pybind/rados/rados.pyx @@ -306,6 +306,7 @@ cdef extern from "rados/librados.h" nogil: void rados_write_op_truncate(rados_write_op_t write_op, uint64_t offset) void rados_write_op_zero(rados_write_op_t write_op, uint64_t offset, uint64_t len) void rados_write_op_exec(rados_write_op_t write_op, const char *cls, const char *method, const char *in_buf, size_t in_len, int *prval) + void rados_write_op_writesame(rados_write_op_t write_op, const char *buffer, size_t data_len, size_t write_len, uint64_t offset) void rados_read_op_omap_get_vals2(rados_read_op_t read_op, const char * start_after, const char * filter_prefix, uint64_t max_return, rados_omap_iter_t * iter, unsigned char *pmore, int * prval) void rados_read_op_omap_get_keys2(rados_read_op_t read_op, const char * start_after, uint64_t max_return, rados_omap_iter_t * iter, unsigned char *pmore, int * prval) void rados_read_op_omap_get_vals_by_keys(rados_read_op_t read_op, const char * const* keys, size_t keys_len, rados_omap_iter_t * iter, int * prval) @@ -2164,6 +2165,25 @@ cdef class WriteOp(object): with nogil: rados_write_op_exec(self.write_op, _cls, _method, _data, _data_len, NULL) + @requires(('to_write', bytes), ('write_len', int), ('offset', int)) + def writesame(self, to_write, write_len, offset=0): + """ + Write the same buffer multiple times + :param to_write: data to write + :type to_write: bytes + :param write_len: total number of bytes to write + :type len: int + :param offset: byte offset in the object to begin writing at + :type offset: int + """ + cdef: + char *_to_write = to_write + size_t _data_len = len(to_write) + size_t _write_len = write_len + uint64_t _offset = offset + with nogil: + rados_write_op_writesame(self.write_op, _to_write, _data_len, _write_len, _offset) + class WriteOpCtx(WriteOp, OpCtx): """write operation context manager""" diff --git a/src/test/pybind/test_rados.py b/src/test/pybind/test_rados.py index 705629ce19ab4..3d32a379c94fb 100644 --- a/src/test/pybind/test_rados.py +++ b/src/test/pybind/test_rados.py @@ -538,6 +538,12 @@ def test_execute_op(self): self.ioctx.operate_write_op(write_op, "object") eq(self.ioctx.read('object'), b"Hello, ebs!") + def test_writesame_op(self): + with WriteOpCtx() as write_op: + write_op.writesame(b'rzx', 9) + self.ioctx.operate_write_op(write_op, 'abc') + eq(self.ioctx.read('abc'), b'rzxrzxrzx') + def test_get_omap_vals_by_keys(self): keys = ("1", "2", "3", "4") values = (b"aaa", b"bbb", b"ccc", b"\x04\x04\x04\x04")