Skip to content

Commit

Permalink
Merge pull request ceph#19536 from dillaman/wip-rbd-mirror-trash
Browse files Browse the repository at this point in the history
rbd-mirror: support deferred deletions of mirrored images

Reviewed-by: Mykola Golub <[email protected]>
  • Loading branch information
trociny authored Dec 24, 2017
2 parents c23574b + 2a8ab2d commit 8249ffd
Show file tree
Hide file tree
Showing 52 changed files with 4,284 additions and 1,207 deletions.
15 changes: 12 additions & 3 deletions src/cls/rbd/cls_rbd_types.h
Original file line number Diff line number Diff line change
Expand Up @@ -354,13 +354,22 @@ struct TrashImageSpec {

TrashImageSpec() {}
TrashImageSpec(TrashImageSource source, const std::string &name,
utime_t deletion_time, utime_t deferment_end_time) :
source(source), name(name), deletion_time(deletion_time),
deferment_end_time(deferment_end_time) {}
const utime_t& deletion_time,
const utime_t& deferment_end_time)
: source(source), name(name), deletion_time(deletion_time),
deferment_end_time(deferment_end_time) {
}

void encode(bufferlist &bl) const;
void decode(bufferlist::iterator& it);
void dump(Formatter *f) const;

inline bool operator==(const TrashImageSpec& rhs) const {
return (source == rhs.source &&
name == rhs.name &&
deletion_time == rhs.deletion_time &&
deferment_end_time == rhs.deferment_end_time);
}
};
WRITE_CLASS_ENCODER(TrashImageSpec);

Expand Down
4 changes: 4 additions & 0 deletions src/common/options.cc
Original file line number Diff line number Diff line change
Expand Up @@ -5873,6 +5873,10 @@ static std::vector<Option> get_rbd_options() {
.set_default(false)
.set_description("automatically start image resync after mirroring is disconnected due to being laggy"),

Option("rbd_mirroring_delete_delay", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
.set_default(0)
.set_description("time-delay in seconds for rbd-mirror delete propagation"),

Option("rbd_mirroring_replay_delay", Option::TYPE_INT, Option::LEVEL_ADVANCED)
.set_default(0)
.set_description("time-delay in seconds for rbd-mirror asynchronous replication"),
Expand Down
4 changes: 4 additions & 0 deletions src/librbd/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
add_library(rbd_types STATIC
journal/Types.cc
mirroring_watcher/Types.cc
trash_watcher/Types.cc
watcher/Types.cc
WatchNotifyTypes.cc)

Expand All @@ -20,6 +21,7 @@ set(librbd_internal_srcs
MirroringWatcher.cc
ObjectMap.cc
Operations.cc
TrashWatcher.cc
Utils.cc
Watcher.cc
api/DiffIterate.cc
Expand Down Expand Up @@ -63,6 +65,7 @@ set(librbd_internal_srcs
journal/PromoteRequest.cc
journal/RemoveRequest.cc
journal/Replay.cc
journal/ResetRequest.cc
journal/StandardPolicy.cc
journal/Utils.cc
managed_lock/AcquireRequest.cc
Expand Down Expand Up @@ -107,6 +110,7 @@ set(librbd_internal_srcs
operation/SnapshotUnprotectRequest.cc
operation/SnapshotLimitRequest.cc
operation/TrimRequest.cc
trash/MoveRequest.cc
watcher/Notifier.cc
watcher/RewatchRequest.cc
${CMAKE_SOURCE_DIR}/src/common/ContextCompletion.cc)
Expand Down
2 changes: 2 additions & 0 deletions src/librbd/ImageCtx.cc
Original file line number Diff line number Diff line change
Expand Up @@ -1020,6 +1020,7 @@ struct C_InvalidateCache : public Context {
"rbd_journal_max_payload_bytes", false)(
"rbd_journal_max_concurrent_object_sets", false)(
"rbd_mirroring_resync_after_disconnect", false)(
"rbd_mirroring_delete_delay", false)(
"rbd_mirroring_replay_delay", false)(
"rbd_skip_partial_discard", false)(
"rbd_qos_iops_limit", false);
Expand Down Expand Up @@ -1080,6 +1081,7 @@ struct C_InvalidateCache : public Context {
ASSIGN_OPTION(journal_max_payload_bytes, uint64_t);
ASSIGN_OPTION(journal_max_concurrent_object_sets, int64_t);
ASSIGN_OPTION(mirroring_resync_after_disconnect, bool);
ASSIGN_OPTION(mirroring_delete_delay, uint64_t);
ASSIGN_OPTION(mirroring_replay_delay, int64_t);
ASSIGN_OPTION(skip_partial_discard, bool);
ASSIGN_OPTION(blkin_trace_all, bool);
Expand Down
1 change: 1 addition & 0 deletions src/librbd/ImageCtx.h
Original file line number Diff line number Diff line change
Expand Up @@ -196,6 +196,7 @@ namespace librbd {
uint32_t journal_max_payload_bytes;
int journal_max_concurrent_object_sets;
bool mirroring_resync_after_disconnect;
uint64_t mirroring_delete_delay;
int mirroring_replay_delay;
bool skip_partial_discard;
bool blkin_trace_all;
Expand Down
54 changes: 9 additions & 45 deletions src/librbd/Journal.cc
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
#include "librbd/journal/DemoteRequest.h"
#include "librbd/journal/OpenRequest.h"
#include "librbd/journal/RemoveRequest.h"
#include "librbd/journal/ResetRequest.h"
#include "librbd/journal/Replay.h"
#include "librbd/journal/PromoteRequest.h"

Expand Down Expand Up @@ -405,54 +406,17 @@ int Journal<I>::reset(librados::IoCtx &io_ctx, const std::string &image_id) {
CephContext *cct = reinterpret_cast<CephContext *>(io_ctx.cct());
ldout(cct, 5) << __func__ << ": image=" << image_id << dendl;

Journaler journaler(io_ctx, image_id, IMAGE_CLIENT_ID, {});
ThreadPool *thread_pool;
ContextWQ *op_work_queue;
ImageCtx::get_thread_pool_instance(cct, &thread_pool, &op_work_queue);

C_SaferCond cond;
journaler.init(&cond);
BOOST_SCOPE_EXIT_ALL(&journaler) {
journaler.shut_down();
};

int r = cond.wait();
if (r == -ENOENT) {
return 0;
} else if (r < 0) {
lderr(cct) << __func__ << ": "
<< "failed to initialize journal: " << cpp_strerror(r) << dendl;
return r;
}

uint8_t order, splay_width;
int64_t pool_id;
journaler.get_metadata(&order, &splay_width, &pool_id);

std::string pool_name;
if (pool_id != -1) {
librados::Rados rados(io_ctx);
r = rados.pool_reverse_lookup(pool_id, &pool_name);
if (r < 0) {
lderr(cct) << __func__ << ": "
<< "failed to lookup data pool: " << cpp_strerror(r) << dendl;
return r;
}
}

C_SaferCond ctx1;
journaler.remove(true, &ctx1);
r = ctx1.wait();
if (r < 0) {
lderr(cct) << __func__ << ": "
<< "failed to reset journal: " << cpp_strerror(r) << dendl;
return r;
}
auto req = journal::ResetRequest<I>::create(io_ctx, image_id, IMAGE_CLIENT_ID,
Journal<>::LOCAL_MIRROR_UUID,
op_work_queue, &cond);
req->send();

r = create(io_ctx, image_id, order, splay_width, pool_name);
if (r < 0) {
lderr(cct) << __func__ << ": "
<< "failed to create journal: " << cpp_strerror(r) << dendl;
return r;
}
return 0;
return cond.wait();
}

template <typename I>
Expand Down
115 changes: 115 additions & 0 deletions src/librbd/TrashWatcher.cc
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
// vim: ts=8 sw=2 smarttab

#include "librbd/TrashWatcher.h"
#include "include/rbd_types.h"
#include "include/rados/librados.hpp"
#include "common/errno.h"
#include "librbd/Utils.h"
#include "librbd/watcher/Utils.h"

#define dout_subsys ceph_subsys_rbd
#undef dout_prefix
#define dout_prefix *_dout << "librbd::TrashWatcher: " << __func__ << ": "

namespace librbd {

using namespace trash_watcher;
using namespace watcher;

using librbd::util::create_rados_callback;

namespace {

static const uint64_t NOTIFY_TIMEOUT_MS = 5000;

} // anonymous namespace

template <typename I>
TrashWatcher<I>::TrashWatcher(librados::IoCtx &io_ctx, ContextWQ *work_queue)
: Watcher(io_ctx, work_queue, RBD_TRASH) {
}

template <typename I>
void TrashWatcher<I>::notify_image_added(
librados::IoCtx &io_ctx, const std::string& image_id,
const cls::rbd::TrashImageSpec& trash_image_spec, Context *on_finish) {
CephContext *cct = reinterpret_cast<CephContext*>(io_ctx.cct());
ldout(cct, 20) << dendl;

bufferlist bl;
::encode(NotifyMessage{ImageAddedPayload{image_id, trash_image_spec}}, bl);

librados::AioCompletion *comp = create_rados_callback(on_finish);
int r = io_ctx.aio_notify(RBD_TRASH, comp, bl, NOTIFY_TIMEOUT_MS, nullptr);
assert(r == 0);
comp->release();
}

template <typename I>
void TrashWatcher<I>::notify_image_removed(librados::IoCtx &io_ctx,
const std::string& image_id,
Context *on_finish) {
CephContext *cct = reinterpret_cast<CephContext*>(io_ctx.cct());
ldout(cct, 20) << dendl;

bufferlist bl;
::encode(NotifyMessage{ImageRemovedPayload{image_id}}, bl);

librados::AioCompletion *comp = create_rados_callback(on_finish);
int r = io_ctx.aio_notify(RBD_TRASH, comp, bl, NOTIFY_TIMEOUT_MS, nullptr);
assert(r == 0);
comp->release();
}

template <typename I>
void TrashWatcher<I>::handle_notify(uint64_t notify_id, uint64_t handle,
uint64_t notifier_id, bufferlist &bl) {
CephContext *cct = this->m_cct;
ldout(cct, 15) << "notify_id=" << notify_id << ", "
<< "handle=" << handle << dendl;


NotifyMessage notify_message;
try {
bufferlist::iterator iter = bl.begin();
::decode(notify_message, iter);
} catch (const buffer::error &err) {
lderr(cct) << "error decoding image notification: " << err.what()
<< dendl;
Context *ctx = new C_NotifyAck(this, notify_id, handle);
ctx->complete(0);
return;
}

apply_visitor(watcher::util::HandlePayloadVisitor<TrashWatcher<I>>(
this, notify_id, handle), notify_message.payload);
}

template <typename I>
bool TrashWatcher<I>::handle_payload(const ImageAddedPayload &payload,
Context *on_notify_ack) {
CephContext *cct = this->m_cct;
ldout(cct, 20) << dendl;
handle_image_added(payload.image_id, payload.trash_image_spec);
return true;
}

template <typename I>
bool TrashWatcher<I>::handle_payload(const ImageRemovedPayload &payload,
Context *on_notify_ack) {
CephContext *cct = this->m_cct;
ldout(cct, 20) << dendl;
handle_image_removed(payload.image_id);
return true;
}

template <typename I>
bool TrashWatcher<I>::handle_payload(const UnknownPayload &payload,
Context *on_notify_ack) {
return true;
}

} // namespace librbd

template class librbd::TrashWatcher<librbd::ImageCtx>;
58 changes: 58 additions & 0 deletions src/librbd/TrashWatcher.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
// vim: ts=8 sw=2 smarttab

#ifndef CEPH_LIBRBD_TRASH_WATCHER_H
#define CEPH_LIBRBD_TRASH_WATCHER_H

#include "include/int_types.h"
#include "cls/rbd/cls_rbd_types.h"
#include "librbd/ImageCtx.h"
#include "librbd/Watcher.h"
#include "librbd/trash_watcher/Types.h"

namespace librados { class IoCtx; }

namespace librbd {

namespace watcher {
namespace util {
template <typename> struct HandlePayloadVisitor;
} // namespace util
} // namespace watcher

template <typename ImageCtxT = librbd::ImageCtx>
class TrashWatcher : public Watcher {
friend struct watcher::util::HandlePayloadVisitor<TrashWatcher<ImageCtxT>>;
public:
TrashWatcher(librados::IoCtx &io_ctx, ContextWQ *work_queue);

static void notify_image_added(librados::IoCtx &io_ctx,
const std::string& image_id,
const cls::rbd::TrashImageSpec& spec,
Context *on_finish);
static void notify_image_removed(librados::IoCtx &io_ctx,
const std::string& image_id,
Context *on_finish);

protected:
virtual void handle_image_added(const std::string &image_id,
const cls::rbd::TrashImageSpec& spec) = 0;
virtual void handle_image_removed(const std::string &image_id) = 0;

private:
void handle_notify(uint64_t notify_id, uint64_t handle,
uint64_t notifier_id, bufferlist &bl) override;

bool handle_payload(const trash_watcher::ImageAddedPayload &payload,
Context *on_notify_ack);
bool handle_payload(const trash_watcher::ImageRemovedPayload &payload,
Context *on_notify_ack);
bool handle_payload(const trash_watcher::UnknownPayload &payload,
Context *on_notify_ack);
};

} // namespace librbd

extern template class librbd::TrashWatcher<librbd::ImageCtx>;

#endif // CEPH_LIBRBD_TRASH_WATCHER_H
Loading

0 comments on commit 8249ffd

Please sign in to comment.