Skip to content

Commit

Permalink
heartbeat集成测试:mds端
Browse files Browse the repository at this point in the history 
Change-Id: I7c913be83f7bd828b23658d7a3373c98d5dfcd3b
  • Loading branch information
lixiaocui1 committed Nov 6, 2019
1 parent 3347679 commit fa5ed68
Show file tree
Hide file tree
Showing 6 changed files with 1,324 additions and 34 deletions.
9 changes: 5 additions & 4 deletions test/integration/cluster_common/cluster.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ void CurveCluster::StopCluster() {

for (auto it = etcdPidMap_.begin(); it != etcdPidMap_.end();) {
LOG(INFO) << "begin to stop etcd" << it->first << " " << it->second;
int res = kill(it->second, SIGTERM);
int res = kill(it->second, SIGKILL);
waitpid(it->second, &waitStatus, 0);
ASSERT_EQ(0, res);
ASSERT_EQ(0, ProbePort(etcdClientIpPort_[it->first], 10000, false));
Expand Down Expand Up @@ -223,14 +223,15 @@ void CurveCluster::StopEtcd(int id) {
LOG(INFO) << "stop etcd " << etcdClientIpPort_[id] << " begin...";

if (etcdPidMap_.find(id) != etcdPidMap_.end()) {
int res = kill(etcdPidMap_[id], SIGTERM);
int res = kill(etcdPidMap_[id], SIGKILL);
int waitStatus;
waitpid(etcdPidMap_[id], &waitStatus, 0);
ASSERT_EQ(0, res);
ASSERT_EQ(0, ProbePort(etcdClientIpPort_[id], 10000, false));
ASSERT_EQ(0, ProbePort(etcdPeersIpPort_[id], 10000, false));
LOG(INFO) << "stop etcd " << etcdClientIpPort_[id]
<< ", " << etcdPidMap_[id] << " success";
etcdPidMap_.erase(id);
LOG(INFO) << "stop etcd " << etcdClientIpPort_[id] << " success";
} else {
LOG(INFO) << "etcd " << id << " not exist";
}
Expand All @@ -244,7 +245,7 @@ void CurveCluster::StopAllEtcd() {
int waitStatus;
for (auto it = etcdPidMap_.begin(); it != etcdPidMap_.end();) {
LOG(INFO) << "begin to stop etcd" << it->first << " " << it->second;
int res = kill(it->second, SIGTERM);
int res = kill(it->second, SIGKILL);
waitpid(it->second, &waitStatus, 0);
ASSERT_EQ(0, res);
ASSERT_EQ(0, ProbePort(etcdClientIpPort_[it->first], 10000, false));
Expand Down
78 changes: 59 additions & 19 deletions test/integration/cluster_common/cluster_basic_test.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,10 +67,6 @@ class ClusterBasicTest : public ::testing::Test {

// 清理DB数据和文件
curveCluster_->mdsRepo_->dropDataBase();
system("rm -fr basic*");
system("rm -fr test_multi_etcd_node*");
system("rm -fr test_start_stop_module2.etcd");
system("rm -r ClusterBasicTest.test_start_stop_module2.etcd.log");
}

void TearDown() {
Expand Down Expand Up @@ -114,21 +110,28 @@ TEST_F(ClusterBasicTest, DISABLED_test_start_stop_module1) {
}

TEST_F(ClusterBasicTest, test_start_stop_module2) {
std::string commonDir = "./runlog/ClusterBasicTest.test_start_stop_module2";
ASSERT_EQ(0, system("rm -fr test_start_stop_module2.etcd"));
ASSERT_EQ(0, system("rm -fr basic*"));
ASSERT_EQ(0, system((std::string("mkdir ") + commonDir).c_str()));

// 起etcd
std::string etcdDir = commonDir + "/etcd.log";
curveCluster_->StarSingleEtcd(1, "127.0.0.1:2221", "127.0.0.1:2222",
std::vector<std::string>{
" --name test_start_stop_module2",
" --log-level 'debug'",
" > ClusterBasicTest.test_start_stop_module2.etcd.log 2>&1"});
std::vector<std::string>{" --name test_start_stop_module2"});
ASSERT_TRUE(curveCluster_->WaitForEtcdClusterAvalible());

// 起mds
curveCluster_->StartSingleMDS(1, "127.0.0.1:3333", mdsConf, true);
auto mdsConfbak = mdsConf;
auto mdsDir = commonDir + "/mds";
ASSERT_EQ(0, system((std::string("mkdir ") + mdsDir).c_str()));
mdsConfbak.emplace_back(" -log_dir=" + mdsDir);
curveCluster_->StartSingleMDS(1, "127.0.0.1:3333", mdsConfbak, true);
// 初始化mdsclient
MetaServerOption_t op;
op.rpcTimeoutMs = 2000;
op.rpcTimeoutMs = 500;
op.rpcRetryTimes = 3;
op.synchronizeRPCTimeoutMS = 2000;
op.synchronizeRPCTimeoutMS = 500;
op.metaaddrvec = std::vector<std::string>{"127.0.0.1:3333"};
curveCluster_->InitMdsClient(op);

Expand All @@ -138,15 +141,24 @@ TEST_F(ClusterBasicTest, test_start_stop_module2) {

// 创建chunkserver
auto copy1 = chunkserverConf1;
std::string chunkserver1Dir = commonDir + "/chunkserver1";
ASSERT_EQ(0, system((std::string("mkdir ") + chunkserver1Dir).c_str()));
copy1.push_back(" -mdsListenAddr=127.0.0.1:3333");
copy1.push_back(" -log_dir=" + chunkserver1Dir);
curveCluster_->StartSingleChunkServer(1, "127.0.0.1:2002", copy1);

auto copy2 = chunkserverConf2;
std::string chunkserver2Dir = commonDir + "/chunkserver2";
ASSERT_EQ(0, system((std::string("mkdir ") + chunkserver2Dir).c_str()));
copy2.push_back(" -mdsListenAddr=127.0.0.1:3333");
copy2.push_back(" -log_dir=" + chunkserver2Dir);
curveCluster_->StartSingleChunkServer(2, "127.0.0.1:2003", copy2);

auto copy3 = chunkserverConf3;
std::string chunkserver3Dir = commonDir + "/chunkserver3";
ASSERT_EQ(0, system((std::string("mkdir ") + chunkserver3Dir).c_str()));
copy3.push_back(" -mdsListenAddr=127.0.0.1:3333");
copy3.push_back(" -log_dir=" + chunkserver3Dir);
curveCluster_->StartSingleChunkServer(3, "127.0.0.1:2004", copy3);

// 创建逻辑池和copyset
Expand Down Expand Up @@ -186,28 +198,56 @@ TEST_F(ClusterBasicTest, test_start_stop_module2) {
}

TEST_F(ClusterBasicTest, test_multi_mds_and_etcd) {
std::string commonDir = "./runlog/ClusterBasicTest.test_multi_mds_and_etcd";
ASSERT_EQ(0, system("rm -fr test_multi_etcd_node*.etcd"));
ASSERT_EQ(0, system((std::string("mkdir ") + commonDir).c_str()));

// 起三个etcd
std::string etcdDir = commonDir + "/etcd";
ASSERT_EQ(0, system((std::string("mkdir ") + etcdDir).c_str()));
std::string etcdcluster = std::string(" --initial-cluster ")
+ std::string("'test_multi_etcd_node1=http://127.0.0.1:2302,")
+ std::string("test_multi_etcd_node2=http://127.0.0.1:2304,")
+ std::string("test_multi_etcd_node3=http://127.0.0.1:2306'");
curveCluster_->StarSingleEtcd(1, "127.0.0.1:2301", "127.0.0.1:2302",
std::vector<std::string>{" --name test_multi_etcd_node1", etcdcluster});
std::vector<std::string>{
" --name test_multi_etcd_node1",
etcdcluster});
ASSERT_FALSE(curveCluster_->WaitForEtcdClusterAvalible(3));
curveCluster_->StarSingleEtcd(2, "127.0.0.1:2303", "127.0.0.1:2304",
std::vector<std::string>{" --name test_multi_etcd_node2", etcdcluster});
std::vector<std::string>{
" --name test_multi_etcd_node2",
etcdcluster});
curveCluster_->StarSingleEtcd(3, "127.0.0.1:2305", "127.0.0.1:2306",
std::vector<std::string>{" --name test_multi_etcd_node3", etcdcluster});
std::vector<std::string>{
" --name test_multi_etcd_node3",
etcdcluster});
ASSERT_TRUE(curveCluster_->WaitForEtcdClusterAvalible());

// 起三mds
std::string mds1Dir = commonDir + "/mds1";
std::string mds2Dir = commonDir + "/mds2";
std::string mds3Dir = commonDir + "/mds3";
ASSERT_EQ(0, system((std::string("mkdir ") + mds1Dir).c_str()));
ASSERT_EQ(0, system((std::string("mkdir ") + mds2Dir).c_str()));
ASSERT_EQ(0, system((std::string("mkdir ") + mds3Dir).c_str()));
std::string etcdClinetAddrs("127.0.0.1:2301,127.0.0.1:2303,127.0.0.1:2305");
auto copy = mdsConf;
copy.emplace_back(" --etcdAddr=" + etcdClinetAddrs);
curveCluster_->StartSingleMDS(1, "127.0.0.1:2310", copy, true);

auto copy1 = mdsConf;
copy1.emplace_back(" --etcdAddr=" + etcdClinetAddrs);
copy1.emplace_back(" -log_dir=" + mds1Dir);
curveCluster_->StartSingleMDS(1, "127.0.0.1:2310", copy1, true);
std::this_thread::sleep_for(std::chrono::seconds(1));
curveCluster_->StartSingleMDS(2, "127.0.0.1:2311", copy, false);
curveCluster_->StartSingleMDS(3, "127.0.0.1:2312", copy, false);

auto copy2 = mdsConf;
copy1.emplace_back(" --etcdAddr=" + etcdClinetAddrs);
copy2.emplace_back(" -log_dir=" + mds2Dir);
curveCluster_->StartSingleMDS(2, "127.0.0.1:2311", copy2, false);

auto copy3 = mdsConf;
copy1.emplace_back(" --etcdAddr=" + etcdClinetAddrs);
copy3.emplace_back(" -log_dir=" + mds3Dir);
curveCluster_->StartSingleMDS(3, "127.0.0.1:2312", copy3, false);

// 获取当前正在服务的mds
int curMds;
Expand Down
13 changes: 3 additions & 10 deletions test/integration/cluster_common/mds.basic.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,19 +23,12 @@ mds.segment.alloc.periodic.persistInterMs=1000
# 出错情况下的重试间隔,单位ms
mds.segment.alloc.retryInterMs=1000

#
# leader election相关配置
#
# leader竞选成功后，会进行observe操作。observe的行为是对该leadername进行watch,
# watch会从etcd中不断获取key-value。在服务端未挂掉的情况下，竞选的超时时间是sessionInterSec,
# 在服务端挂掉或者网络分区的情况下，observeTimeoutMs后mds会退出进程，防止出现双主.
# 单位是ms
mds.leader.observeTimeoutMs=10000

# leader竞选时会创建session, 单位是秒, 因为go端代码的接口这个值得单位就是s
mds.leader.sessionInterSec=10
mds.leader.sessionInterSec=5
# leader竞选的超时时间，如果为0竞选不成功会一直block, 如果大于0，在electionTimeoutMs时间
# 内未当选leader会返回错误。这里设置10分钟超时，超时后mds会继续竞选
mds.leader.electionTimeoutMs=600000
mds.leader.electionTimeoutMs=0

#
# scheduler相关配置
Expand Down
14 changes: 14 additions & 0 deletions test/integration/heartbeat/BUILD
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
cc_test(
name = "mds_heartbeat_basic_integration",
srcs = glob(["heartbeat_basic_test.cpp"]),
deps = [
"//src/mds/heartbeat:heartbeat",
"//src/mds/nameserver2:nameserver2",
"//src/common:curve_common",
"//src/mds/topology:topology",
"//src/mds/schedule",
"@com_google_googletest//:gtest_main",
"@com_google_googletest//:gtest",
"//external:brpc"
]
)
Loading

0 comments on commit fa5ed68

Please sign in to comment.