Skip to content

Commit

Permalink
introduce chunkserver IO test
Browse files Browse the repository at this point in the history
Change-Id: Ia08b775895713bbfc517129a723feeadc13c3f8e
  • Loading branch information
qinyi committed Mar 31, 2020
1 parent 28e4120 commit db7ea94
Show file tree
Hide file tree
Showing 5 changed files with 462 additions and 0 deletions.
25 changes: 25 additions & 0 deletions test/integration/chunkserver/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -22,3 +22,28 @@ cc_test(
"@com_google_googletest//:gtest_main",
],
)

cc_test(
name = "chunkserver-basic-test",
srcs = glob([
"peer_cluster.h",
"peer_cluster.cpp",
"chunkserver_basic_test.cpp",
]),
copts = ["-std=c++14"],
deps = [
"//external:braft",
"//external:brpc",
"//external:bthread",
"//external:butil",
"//external:gflags",
"//external:glog",
"//external:leveldb",
"//proto:chunkserver-cc-protos",
"//src/chunkserver:chunkserver-test-lib",
"//src/common:curve_common",
"//test/chunkserver/datastore:chunkfilepool_helper",
"//test/integration/common:integration-test-common",
"@com_google_googletest//:gtest_main",
],
)
3 changes: 3 additions & 0 deletions test/integration/common/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ cc_library(
"config_generator.h",
"peer_cluster.h",
"peer_cluster.cpp",
"chunkservice_op.h",
"chunkservice_op.cpp",
]),
copts = ["-std=c++14"],
visibility = ["//visibility:public"],
Expand All @@ -22,3 +24,4 @@ cc_library(
"@com_google_googletest//:gtest",
],
)

286 changes: 286 additions & 0 deletions test/integration/common/chunkservice_op.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,286 @@
/*
* Project: curve
* Created Date: 2020-03-10
* Author: qinyi
* Copyright (c) 2020 netease
*/

#include "test/integration/common/chunkservice_op.h"
#include "proto/chunk.pb.h"

namespace curve {
namespace chunkserver {

const PageSizeType kPageSize = kOpRequestAlignSize;

int ChunkServiceOp::WriteChunk(struct ChunkServiceOpConf *opConf,
ChunkID chunkId, SequenceNum sn, off_t offset,
size_t len, const char *data) {
PeerId leaderId(opConf->leaderPeer->address());
brpc::Channel channel;
channel.Init(leaderId.addr, NULL);
ChunkService_Stub stub(&channel);
brpc::Controller cntl;
cntl.set_timeout_ms(opConf->rpcTimeout);

ChunkRequest request;
ChunkResponse response;
request.set_optype(CHUNK_OP_TYPE::CHUNK_OP_WRITE);
request.set_logicpoolid(opConf->logicPoolId);
request.set_copysetid(opConf->copysetId);
request.set_chunkid(chunkId);
request.set_sn(sn);
request.set_offset(offset);
request.set_size(len);
cntl.request_attachment().append(data, len);
stub.WriteChunk(&cntl, &request, &response, nullptr);

if (cntl.Failed()) {
LOG(ERROR) << "write failed: " << cntl.ErrorText();
return -1;
}

CHUNK_OP_STATUS status = response.status();
LOG_IF(ERROR, status) << "write failed: " << CHUNK_OP_STATUS_Name(status);
return status;
}

int ChunkServiceOp::ReadChunk(struct ChunkServiceOpConf *opConf,
ChunkID chunkId, SequenceNum sn, off_t offset,
size_t len, std::string *data) {
PeerId leaderId(opConf->leaderPeer->address());
brpc::Channel channel;
channel.Init(leaderId.addr, NULL);
ChunkService_Stub stub(&channel);
brpc::Controller cntl;
cntl.set_timeout_ms(opConf->rpcTimeout);

ChunkRequest request;
ChunkResponse response;
request.set_optype(CHUNK_OP_TYPE::CHUNK_OP_READ);
request.set_logicpoolid(opConf->logicPoolId);
request.set_copysetid(opConf->copysetId);
request.set_chunkid(chunkId);
request.set_sn(sn);
request.set_offset(offset);
request.set_size(len);
request.set_appliedindex(1);

stub.ReadChunk(&cntl, &request, &response, nullptr);
if (cntl.Failed()) {
LOG(ERROR) << "read failed: " << cntl.ErrorText();
return -1;
}

CHUNK_OP_STATUS status = response.status();
LOG_IF(ERROR, status) << "read failed: " << CHUNK_OP_STATUS_Name(status);

// 读成功,复制内容到data
if (status == CHUNK_OP_STATUS_SUCCESS && data != nullptr) {
cntl.response_attachment().copy_to(data,
cntl.response_attachment().size());
}

return status;
}

int ChunkServiceOp::DeleteChunk(struct ChunkServiceOpConf *opConf,
ChunkID chunkId, SequenceNum sn) {
PeerId leaderId(opConf->leaderPeer->address());
brpc::Channel channel;
channel.Init(leaderId.addr, NULL);
ChunkService_Stub stub(&channel);
brpc::Controller cntl;
cntl.set_timeout_ms(opConf->rpcTimeout);

ChunkRequest request;
ChunkResponse response;
request.set_optype(CHUNK_OP_TYPE::CHUNK_OP_DELETE);
request.set_logicpoolid(opConf->logicPoolId);
request.set_copysetid(opConf->copysetId);
request.set_chunkid(chunkId);
request.set_sn(sn);
stub.DeleteChunk(&cntl, &request, &response, nullptr);

if (cntl.Failed()) {
LOG(ERROR) << "delete failed: " << cntl.ErrorText();
return -1;
}

CHUNK_OP_STATUS status = response.status();
LOG_IF(ERROR, status) << "delete failed: " << CHUNK_OP_STATUS_Name(status);

return status;
}

int ChunkServiceOp::GetChunkInfo(struct ChunkServiceOpConf *opConf,
ChunkID chunkId, SequenceNum *curSn,
SequenceNum *snapSn,
std::string *redirectedLeader) {
PeerId leaderId(opConf->leaderPeer->address());
brpc::Channel channel;
channel.Init(leaderId.addr, NULL);
ChunkService_Stub stub(&channel);
brpc::Controller cntl;
cntl.set_timeout_ms(opConf->rpcTimeout);

GetChunkInfoRequest request;
GetChunkInfoResponse response;
request.set_logicpoolid(opConf->logicPoolId);
request.set_copysetid(opConf->copysetId);
request.set_chunkid(chunkId);
stub.GetChunkInfo(&cntl, &request, &response, nullptr);
if (cntl.Failed()) {
LOG(ERROR) << "GetChunkInfo failed: " << cntl.ErrorText();
return -1;
}

CHUNK_OP_STATUS status = response.status();
if (status == CHUNK_OP_STATUS_SUCCESS) {
switch (response.chunksn().size()) {
case 2:
*snapSn = response.chunksn(1);
case 1:
*curSn = response.chunksn(0);
break;
case 0:
break;
default:
LOG(ERROR) << "GetChunkInfo failed, invalid chunkSn size: "
<< response.chunksn().size();
return -1;
}
}

if (status == CHUNK_OP_STATUS_REDIRECTED) {
*redirectedLeader = response.redirect();
}

LOG_IF(ERROR, status) << "GetChunkInfo failed: "
<< CHUNK_OP_STATUS_Name(status);
return status;
}

int ChunkServiceVerify::VerifyWriteChunk(ChunkID chunkId, SequenceNum sn,
off_t offset, size_t len,
const char *data, string *chunkData) {
int ret =
ChunkServiceOp::WriteChunk(opConf_, chunkId, sn, offset, len, data);

LOG(INFO) << "Write Chunk " << chunkId << ", sn=" << sn
<< ", offset=" << offset << ", len=" << len << ", ret=" << ret;
// chunk写成功,同步更新chunkData内容和existChunks_
if (ret == CHUNK_OP_STATUS_SUCCESS && chunkData != nullptr)
chunkData->replace(offset, len, data);
existChunks_.insert(chunkId);

return ret;
}

int ChunkServiceVerify::VerifyReadChunk(ChunkID chunkId, SequenceNum sn,
off_t offset, size_t len,
string *chunkData) {
std::string data(len, 0);
bool chunk_existed = existChunks_.find(chunkId) != std::end(existChunks_);

int ret =
ChunkServiceOp::ReadChunk(opConf_, chunkId, sn, offset, len, &data);
LOG(INFO) << "Read Chunk " << chunkId << ", sn=" << sn
<< ", offset=" << offset << ", len=" << len << ", ret=" << ret;

if (ret != CHUNK_OP_STATUS_SUCCESS &&
ret != CHUNK_OP_STATUS_CHUNK_NOTEXIST) {
return -1;
} else if (ret == CHUNK_OP_STATUS_SUCCESS && !chunk_existed) {
LOG(ERROR) << "Unexpected read success, chunk " << chunkId
<< " should not existed";
return -1;
} else if (ret == CHUNK_OP_STATUS_CHUNK_NOTEXIST && chunk_existed) {
LOG(ERROR) << "Unexpected read missing, chunk " << chunkId
<< " must be existed";
return -1;
}

// 读成功,则判断内容是否与chunkData吻合
if (ret == CHUNK_OP_STATUS_SUCCESS && chunkData != nullptr) {
// 查找数据有差异的位置
uint32_t i = 0;
while (i < len && data[i] == (*chunkData)[offset + i]) ++i;
// 读取数据与预期相符,返回0
if (i == len)
return 0;

LOG(ERROR) << "read data missmatch for chunk " << chunkId
<< ", from offset " << offset + i << ", read "
<< static_cast<int>(data[i]) << ", expected "
<< static_cast<int>((*chunkData)[offset + i]);
// 打印每个page的第一个字节
uint32_t j = i / kPageSize * kPageSize;
for (; j < len; j += kPageSize) {
LOG(ERROR) << "chunk offset " << offset + j << ": read "
<< static_cast<int>(data[j]) << ", expected "
<< static_cast<int>((*chunkData)[offset + j]);
}
return -1;
}

return 0;
}

int ChunkServiceVerify::VerifyDeleteChunk(ChunkID chunkId, SequenceNum sn) {
int ret = ChunkServiceOp::DeleteChunk(opConf_, chunkId, sn);
LOG(INFO) << "Delete Chunk " << chunkId << ", sn " << sn << ", ret=" << ret;

if (ret == CHUNK_OP_STATUS_SUCCESS)
existChunks_.erase(chunkId);
return ret;
}

int ChunkServiceVerify::VerifyGetChunkInfo(ChunkID chunkId,
SequenceNum expCurSn,
SequenceNum expSnapSn,
string expLeader) {
SequenceNum curSn = NULL_SN;
SequenceNum snapSn = NULL_SN;
string redirectedLeader = "";
int ret = ChunkServiceOp::GetChunkInfo(opConf_, chunkId, &curSn, &snapSn,
&redirectedLeader);
LOG(INFO) << "GetChunkInfo for chunk " << chunkId << ", curSn=" << curSn
<< ", snapSn=" << snapSn << ", redirectedLeader={"
<< redirectedLeader << "}, ret=" << ret;

bool chunk_existed = existChunks_.find(chunkId) != std::end(existChunks_);
switch (ret) {
case CHUNK_OP_STATUS_SUCCESS:
// 如果curSn或snapSn与预期不符,则返回-1
LOG_IF(ERROR, (curSn != expCurSn || snapSn != expSnapSn))
<< "GetChunkInfo for " << chunkId << " failed, curSn=" << curSn
<< ", expected " << expCurSn << "; snapSn=" << snapSn
<< ", expected " << expSnapSn;
return (curSn != expCurSn || snapSn != expSnapSn) ? -1 : 0;

case CHUNK_OP_STATUS_CHUNK_NOTEXIST:
// 如果chunk预期存在,则返回-1
LOG_IF(ERROR, chunk_existed)
<< "Unexpected GetChunkInfo NOTEXIST, chunk " << chunkId
<< " must be existed";
return chunk_existed ? -1 : 0;

case CHUNK_OP_STATUS_REDIRECTED:
// 如果返回的redirectedLeader与给定的不符,则返回-1
LOG_IF(ERROR, expLeader != redirectedLeader)
<< "GetChunkInfo failed, redirected to " << redirectedLeader
<< ", expected " << expLeader;
return (expLeader != redirectedLeader) ? -1 : 0;

default:
LOG(ERROR) << "GetChunkInfo for " << chunkId << "failed, ret=" << ret;
return -1;
}

LOG(ERROR) << "GetChunkInfo for " << chunkId << "failed, Illgal branch";
return -1;
}

} // namespace chunkserver
} // namespace curve
Loading

0 comments on commit db7ea94

Please sign in to comment.