Skip to content

Commit

Permalink
Merge pull request ceph#9360 from dillaman/wip-16045
Browse files Browse the repository at this point in the history 
rbd-mirror: support multiple replicated pools

Reviewed-by: Mykola Golub <[email protected]>
  • Loading branch information
@trociny
trociny committed May 28, 2016
2 parents 60588e8 + f4339ee commit 14a150b
Show file tree
Hide file tree
Showing 14 changed files with 375 additions and 557 deletions.
15 changes: 12 additions & 3 deletions qa/workunits/rbd/rbd_mirror.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@
CLUSTER1=cluster1
CLUSTER2=cluster2
POOL=mirror
PARENT_POOL=mirror_parent
SRC_DIR=$(readlink -f $(dirname $0)/../../../src)
TEMPDIR=

Expand Down Expand Up @@ -134,12 +135,18 @@ setup()

ceph --cluster ${CLUSTER1} osd pool create ${POOL} 64 64
ceph --cluster ${CLUSTER2} osd pool create ${POOL} 64 64
ceph --cluster ${CLUSTER1} osd pool create ${PARENT_POOL} 64 64
ceph --cluster ${CLUSTER2} osd pool create ${PARENT_POOL} 64 64

rbd --cluster ${CLUSTER1} mirror pool enable ${POOL} pool
rbd --cluster ${CLUSTER2} mirror pool enable ${POOL} pool
rbd --cluster ${CLUSTER1} mirror pool enable ${PARENT_POOL} image
rbd --cluster ${CLUSTER2} mirror pool enable ${PARENT_POOL} image

rbd --cluster ${CLUSTER1} mirror pool peer add ${POOL} ${CLUSTER2}
rbd --cluster ${CLUSTER2} mirror pool peer add ${POOL} ${CLUSTER1}
rbd --cluster ${CLUSTER1} mirror pool peer add ${PARENT_POOL} ${CLUSTER2}
rbd --cluster ${CLUSTER2} mirror pool peer add ${PARENT_POOL} ${CLUSTER1}
}

cleanup()
Expand All @@ -158,6 +165,8 @@ cleanup()
else
ceph --cluster ${CLUSTER1} osd pool rm ${POOL} ${POOL} --yes-i-really-really-mean-it
ceph --cluster ${CLUSTER2} osd pool rm ${POOL} ${POOL} --yes-i-really-really-mean-it
ceph --cluster ${CLUSTER1} osd pool rm ${PARENT_POOL} ${PARENT_POOL} --yes-i-really-really-mean-it
ceph --cluster ${CLUSTER2} osd pool rm ${PARENT_POOL} ${PARENT_POOL} --yes-i-really-really-mean-it
fi
rm -Rf ${TEMPDIR}
}
Expand Down Expand Up @@ -590,7 +599,7 @@ admin_daemon ${CLUSTER1} rbd mirror restart
wait_for_image_replay_started ${CLUSTER1} ${image}
wait_for_image_replay_started ${CLUSTER1} ${image1}

admin_daemon ${CLUSTER1} rbd mirror stop ${CLUSTER2}
admin_daemon ${CLUSTER1} rbd mirror stop ${POOL} ${CLUSTER2}
wait_for_image_replay_stopped ${CLUSTER1} ${image}
wait_for_image_replay_stopped ${CLUSTER1} ${image1}

Expand All @@ -600,12 +609,12 @@ wait_for_image_replay_started ${CLUSTER1} ${image}
admin_daemon ${CLUSTER1} rbd mirror start
wait_for_image_replay_started ${CLUSTER1} ${image1}

admin_daemon ${CLUSTER1} rbd mirror start ${CLUSTER2}
admin_daemon ${CLUSTER1} rbd mirror start ${POOL} ${CLUSTER2}

admin_daemon ${CLUSTER1} rbd mirror restart ${POOL}/${image}
wait_for_image_replay_started ${CLUSTER1} ${image}

admin_daemon ${CLUSTER1} rbd mirror restart ${CLUSTER2}
admin_daemon ${CLUSTER1} rbd mirror restart ${POOL} ${CLUSTER2}
wait_for_image_replay_started ${CLUSTER1} ${image}
wait_for_image_replay_started ${CLUSTER1} ${image1}

Expand Down
14 changes: 7 additions & 7 deletions src/test/rbd_mirror/test_ClusterWatcher.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ class TestClusterWatcher : public ::testing::Test {
uuid != nullptr ? uuid : &gen_uuid,
peer.cluster_name,
peer.client_name));
m_peer_configs[peer].insert(pool_id);
m_pool_peers[pool_id].insert(peer);
m_mirrored_pools.insert(pool_name);
}
if (name != nullptr) {
Expand All @@ -71,11 +71,11 @@ class TestClusterWatcher : public ::testing::Test {
void delete_pool(const string &name, const peer_t &peer) {
int64_t pool_id = m_cluster->pool_lookup(name.c_str());
ASSERT_GE(pool_id, 0);
if (m_peer_configs.find(peer) != m_peer_configs.end()) {
m_peer_configs[peer].erase(pool_id);
if (m_pool_peers.find(pool_id) != m_pool_peers.end()) {
m_pool_peers[pool_id].erase(peer);
m_mirrored_pools.erase(name);
if (m_peer_configs[peer].empty()) {
m_peer_configs.erase(peer);
if (m_pool_peers[pool_id].empty()) {
m_pool_peers.erase(pool_id);
}
}
m_pools.erase(name);
Expand Down Expand Up @@ -121,7 +121,7 @@ class TestClusterWatcher : public ::testing::Test {
void check_peers() {
m_cluster_watcher->refresh_pools();
Mutex::Locker l(m_lock);
ASSERT_EQ(m_peer_configs, m_cluster_watcher->get_peer_configs());
ASSERT_EQ(m_pool_peers, m_cluster_watcher->get_pool_peers());
ASSERT_EQ(m_mirrored_pools, m_cluster_watcher->get_pool_names());
}

Expand All @@ -131,7 +131,7 @@ class TestClusterWatcher : public ::testing::Test {

set<string> m_pools;
set<string> m_mirrored_pools;
map<peer_t, set<int64_t> > m_peer_configs;
ClusterWatcher::PoolPeers m_pool_peers;
};

TEST_F(TestClusterWatcher, NoPools) {
Expand Down
112 changes: 12 additions & 100 deletions src/test/rbd_mirror/test_PoolWatcher.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,6 @@ TestPoolWatcher() : m_lock("TestPoolWatcherLock"),
{
m_cluster = std::make_shared<librados::Rados>();
EXPECT_EQ("", connect_cluster_pp(*m_cluster));
m_pool_watcher.reset(new PoolWatcher(m_cluster, 30, m_lock, m_cond));
}

~TestPoolWatcher() {
Expand All @@ -59,9 +58,12 @@ TestPoolWatcher() : m_lock("TestPoolWatcherLock"),
int64_t pool_id = m_cluster->pool_lookup(pool_name.c_str());
ASSERT_GE(pool_id, 0);
m_pools.insert(pool_name);

librados::IoCtx ioctx;
ASSERT_EQ(0, m_cluster->ioctx_create2(pool_id, ioctx));

m_pool_watcher.reset(new PoolWatcher(ioctx, 30, m_lock, m_cond));
if (enable_mirroring) {
librados::IoCtx ioctx;
ASSERT_EQ(0, m_cluster->ioctx_create2(pool_id, ioctx));
ASSERT_EQ(0, librbd::mirror_mode_set(ioctx, RBD_MIRROR_MODE_POOL));
std::string uuid;
ASSERT_EQ(0, librbd::mirror_peer_add(ioctx, &uuid,
Expand All @@ -73,50 +75,6 @@ TestPoolWatcher() : m_lock("TestPoolWatcherLock"),
}
}

void delete_pool(const string &name, const peer_t &peer) {
int64_t pool_id = m_cluster->pool_lookup(name.c_str());
ASSERT_GE(pool_id, 0);
m_pools.erase(name);
ASSERT_EQ(0, m_cluster->pool_delete(name.c_str()));
m_mirrored_images.erase(pool_id);
}

void create_cache_pool(const string &base_pool, string *cache_pool_name) {
bufferlist inbl;
*cache_pool_name = get_temp_pool_name("test-rbd-mirror-");
ASSERT_EQ(0, m_cluster->pool_create(cache_pool_name->c_str()));

ASSERT_EQ(0, m_cluster->mon_command(
"{\"prefix\": \"osd tier add\", \"pool\": \"" + base_pool +
"\", \"tierpool\": \"" + *cache_pool_name +
"\", \"force_nonempty\": \"--force-nonempty\" }",
inbl, NULL, NULL));
ASSERT_EQ(0, m_cluster->mon_command(
"{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + base_pool +
"\", \"overlaypool\": \"" + *cache_pool_name + "\"}",
inbl, NULL, NULL));
ASSERT_EQ(0, m_cluster->mon_command(
"{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + *cache_pool_name +
"\", \"mode\": \"writeback\"}",
inbl, NULL, NULL));
m_cluster->wait_for_latest_osdmap();
}

void remove_cache_pool(const string &base_pool, const string &cache_pool) {
bufferlist inbl;
// tear down tiers
ASSERT_EQ(0, m_cluster->mon_command(
"{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + base_pool +
"\"}",
inbl, NULL, NULL));
ASSERT_EQ(0, m_cluster->mon_command(
"{\"prefix\": \"osd tier remove\", \"pool\": \"" + base_pool +
"\", \"tierpool\": \"" + cache_pool + "\"}",
inbl, NULL, NULL));
m_cluster->wait_for_latest_osdmap();
m_cluster->pool_delete(cache_pool.c_str());
}

string get_image_id(librados::IoCtx *ioctx, const string &image_name) {
string obj = librbd::util::id_obj_name(image_name);
string id;
Expand Down Expand Up @@ -148,7 +106,7 @@ TestPoolWatcher() : m_lock("TestPoolWatcherLock"),
sizeof(mirror_image_info)));
image.close();

m_mirrored_images[ioctx.get_id()].insert(PoolWatcher::ImageIds(
m_mirrored_images.insert(PoolWatcher::ImageId(
get_image_id(&ioctx, name), name, mirror_image_info.global_id));
}
if (image_name != nullptr)
Expand Down Expand Up @@ -193,7 +151,7 @@ TestPoolWatcher() : m_lock("TestPoolWatcherLock"),
sizeof(mirror_image_info)));
image.close();

m_mirrored_images[cioctx.get_id()].insert(PoolWatcher::ImageIds(
m_mirrored_images.insert(PoolWatcher::ImageId(
get_image_id(&cioctx, name), name, mirror_image_info.global_id));
}
if (image_name != nullptr)
Expand All @@ -212,23 +170,23 @@ TestPoolWatcher() : m_lock("TestPoolWatcherLock"),
unique_ptr<PoolWatcher> m_pool_watcher;

set<string> m_pools;
PoolWatcher::PoolImageIds m_mirrored_images;
PoolWatcher::ImageIds m_mirrored_images;

uint64_t m_image_number;
uint64_t m_snap_number;
};

TEST_F(TestPoolWatcher, NoPools) {
TEST_F(TestPoolWatcher, EmptyPool) {
string uuid1 = "00000000-0000-0000-0000-000000000001";
peer_t site1(uuid1, "site1", "mirror1");
create_pool(true, site1);
check_images();
}

TEST_F(TestPoolWatcher, ReplicatedPools) {
string uuid1 = "00000000-0000-0000-0000-000000000001";
string uuid2 = "20000000-2222-2222-2222-000000000002";
peer_t site1(uuid1, "site1", "mirror1");
peer_t site2(uuid2, "site2", "mirror2");
string first_pool, local_pool, last_pool;
check_images();
create_pool(true, site1, &first_pool);
check_images();
create_image(first_pool);
Expand All @@ -242,50 +200,4 @@ TEST_F(TestPoolWatcher, ReplicatedPools) {
check_images();
create_image(first_pool, false);
check_images();

create_pool(false, peer_t(), &local_pool);
check_images();
create_image(local_pool, false);
check_images();
clone_image(first_pool, parent_image2, local_pool, false);
check_images();
create_pool(true, site2);
check_images();

create_pool(true, site2, &last_pool);
check_images();
clone_image(first_pool, parent_image2, last_pool);
check_images();
create_image(last_pool);
check_images();
delete_pool(last_pool, site2);
check_images();
delete_pool(first_pool, site1);
check_images();
}

TEST_F(TestPoolWatcher, CachePools) {
peer_t site1("11111111-1111-1111-1111-111111111111", "site1", "mirror1");
string base1, base2, cache1, cache2;
create_pool(true, site1, &base1);
check_images();

create_cache_pool(base1, &cache1);
BOOST_SCOPE_EXIT( base1, cache1, this_ ) {
this_->remove_cache_pool(base1, cache1);
} BOOST_SCOPE_EXIT_END;
check_images();
create_image(base1);
check_images();
create_image(base1, false);
check_images();

create_pool(false, peer_t(), &base2);
create_cache_pool(base2, &cache2);
BOOST_SCOPE_EXIT( base2, cache2, this_ ) {
this_->remove_cache_pool(base2, cache2);
} BOOST_SCOPE_EXIT_END;
check_images();
create_image(base2, false);
check_images();
}
29 changes: 13 additions & 16 deletions src/tools/rbd_mirror/ClusterWatcher.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,8 @@

#define dout_subsys ceph_subsys_rbd_mirror
#undef dout_prefix
#define dout_prefix *_dout << "rbd-mirror: ClusterWatcher::" << __func__ << ": "
#define dout_prefix *_dout << "rbd::mirror::ClusterWatcher:" << this << " " \
<< __func__ << ": "

using std::list;
using std::map;
Expand All @@ -31,13 +32,13 @@ ClusterWatcher::ClusterWatcher(RadosRef cluster, Mutex &lock) :
{
}

const map<peer_t, set<int64_t> >& ClusterWatcher::get_peer_configs() const
const ClusterWatcher::PoolPeers& ClusterWatcher::get_pool_peers() const
{
assert(m_lock.is_locked());
return m_peer_configs;
return m_pool_peers;
}

const std::set<std::string>& ClusterWatcher::get_pool_names() const
const ClusterWatcher::PoolNames& ClusterWatcher::get_pool_names() const
{
assert(m_lock.is_locked());
return m_pool_names;
Expand All @@ -46,19 +47,20 @@ const std::set<std::string>& ClusterWatcher::get_pool_names() const
void ClusterWatcher::refresh_pools()
{
dout(20) << "enter" << dendl;
map<peer_t, set<int64_t> > peer_configs;
set<string> pool_names;
read_configs(&peer_configs, &pool_names);

PoolPeers pool_peers;
PoolNames pool_names;
read_pool_peers(&pool_peers, &pool_names);

Mutex::Locker l(m_lock);
m_peer_configs = peer_configs;
m_pool_peers = pool_peers;
m_pool_names = pool_names;
// TODO: perhaps use a workqueue instead, once we get notifications
// about config changes for existing pools
}

void ClusterWatcher::read_configs(map<peer_t, set<int64_t> > *peer_configs,
set<string> *pool_names)
void ClusterWatcher::read_pool_peers(PoolPeers *pool_peers,
PoolNames *pool_names)
{
list<pair<int64_t, string> > pools;
int r = m_cluster->pool_list2(pools);
Expand Down Expand Up @@ -114,12 +116,7 @@ void ClusterWatcher::read_configs(map<peer_t, set<int64_t> > *peer_configs,
continue;
}

for (peer_t peer : configs) {
dout(20) << "pool " << pool_name << " has mirroring enabled for peer "
<< peer << dendl;
(*peer_configs)[peer].insert(pool_id);
}

pool_peers->insert({pool_id, Peers{configs.begin(), configs.end()}});
pool_names->insert(pool_name);
}
}
Expand Down
18 changes: 11 additions & 7 deletions src/tools/rbd_mirror/ClusterWatcher.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,23 +23,27 @@ namespace mirror {
*/
class ClusterWatcher {
public:
typedef std::set<peer_t> Peers;
typedef std::map<int64_t, Peers> PoolPeers;
typedef std::set<std::string> PoolNames;

ClusterWatcher(RadosRef cluster, Mutex &lock);
~ClusterWatcher() = default;
ClusterWatcher(const ClusterWatcher&) = delete;
ClusterWatcher& operator=(const ClusterWatcher&) = delete;

// Caller controls frequency of calls
void refresh_pools();
const std::map<peer_t, std::set<int64_t> >& get_peer_configs() const;
const std::set<std::string>& get_pool_names() const;
const PoolPeers& get_pool_peers() const;
const PoolNames& get_pool_names() const;

private:
void read_configs(std::map<peer_t, std::set<int64_t> > *peer_configs,
std::set<std::string> *pool_names);

Mutex &m_lock;
RadosRef m_cluster;
std::map<peer_t, std::set<int64_t> > m_peer_configs;
std::set<std::string> m_pool_names;
PoolPeers m_pool_peers;
PoolNames m_pool_names;

void read_pool_peers(PoolPeers *pool_peers, PoolNames *pool_names);
};

} // namespace mirror
Expand Down
3 changes: 2 additions & 1 deletion src/tools/rbd_mirror/ImageDeleter.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,8 @@

#define dout_subsys ceph_subsys_rbd_mirror
#undef dout_prefix
#define dout_prefix *_dout << "rbd-mirror: ImageDeleter::" << __func__ << ": "
#define dout_prefix *_dout << "rbd::mirror::ImageDeleter: " << this << " " \
<< __func__ << ": "

using std::string;
using std::map;
Expand Down
Loading

0 comments on commit 14a150b

Please sign in to comment.