Skip to content

Commit

Permalink
fix delete force segment can't delete
Browse files Browse the repository at this point in the history 
Change-Id: I9da87d32137a53775963325c3f648f8aef8e153d
  • Loading branch information
@hzchenwei7
hzchenwei7 committed Jul 17, 2019
1 parent c7a7e6a commit 1a89396
Show file tree
Hide file tree
Showing 8 changed files with 427 additions and 32 deletions.
15 changes: 10 additions & 5 deletions src/mds/nameserver2/clean_core.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,18 +16,20 @@ StatusCode CleanCore::CleanSnapShotFile(const FileInfo & fileInfo,
return StatusCode::KInternalError;
}
uint32_t segmentNum = fileInfo.length() / fileInfo.segmentsize();
uint64_t segmentSize = fileInfo.segmentsize();
for (uint32_t i = 0; i < segmentNum; i++) {
// load segment
PageFileSegment segment;
StoreStatus storeRet = storage_->GetSegment(fileInfo.parentid(),
i * fileInfo.segmentsize(),
i * segmentSize,
&segment);
if (storeRet == StoreStatus::KeyNotExist) {
continue;
} else if (storeRet != StoreStatus::OK) {
LOG(ERROR) << "cleanSnapShot File Error: "
<< "GetSegment Error, inodeid = " << fileInfo.id()
<< ", filename = " << fileInfo.filename()
<< ", offset = " << i * segmentSize
<< ", sequenceNum = " << fileInfo.seqnum();
progress->SetStatus(TaskStatus::FAILED);
return StatusCode::kSnapshotFileDeleteError;
Expand Down Expand Up @@ -85,17 +87,19 @@ StatusCode CleanCore::CleanFile(const FileInfo & commonFile,
}

int segmentNum = commonFile.length() / commonFile.segmentsize();
uint64_t segmentSize = commonFile.segmentsize();
for (int i = 0; i != segmentNum; i++) {
// load segment
PageFileSegment segment;
StoreStatus storeRet = storage_->GetSegment(commonFile.id(),
i * commonFile.segmentsize(), &segment);
i * segmentSize, &segment);
if (storeRet == StoreStatus::KeyNotExist) {
continue;
} else if (storeRet != StoreStatus::OK) {
LOG(ERROR) << "Clean common File Error: "
<< "GetSegment Error, inodeid = " << commonFile.id()
<< ", filename = " << commonFile.filename();
<< "GetSegment Error, inodeid = " << commonFile.id()
<< ", filename = " << commonFile.filename()
<< ", offset = " << i * segmentSize;
progress->SetStatus(TaskStatus::FAILED);
return StatusCode::kCommonFileDeleteError;
}
Expand All @@ -121,11 +125,12 @@ StatusCode CleanCore::CleanFile(const FileInfo & commonFile,

// delete segment
storeRet = storage_->DeleteSegment(commonFile.id(),
i * commonFile.segmentsize());
i * segmentSize);
if (storeRet != StoreStatus::OK) {
LOG(ERROR) << "Clean common File Error: "
<< "DeleteSegment Error, inodeid = " << commonFile.id()
<< ", filename = " << commonFile.filename()
<< ", offset = " << i * segmentSize
<< ", sequenceNum = " << commonFile.seqnum();
progress->SetStatus(TaskStatus::FAILED);
return StatusCode::kCommonFileDeleteError;
Expand Down
8 changes: 5 additions & 3 deletions src/mds/nameserver2/curvefs.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -397,8 +397,8 @@ StatusCode CurveFS::DeleteFile(const std::string & filename, uint64_t fileId,

// 查看任务是否已经在
if ( cleanManager_->GetTask(fileInfo.id()) != nullptr ) {
LOG(WARNING) << "filename = " << filename
<< ", deleteFile task already submited";
LOG(WARNING) << "filename = " << filename << ", inode = "
<< fileInfo.id() << ", deleteFile task already submited";
return StatusCode::kOK;
}

Expand All @@ -413,12 +413,14 @@ StatusCode CurveFS::DeleteFile(const std::string & filename, uint64_t fileId,
// 提交一个删除文件的任务
if (!cleanManager_->SubmitDeleteCommonFileJob(fileInfo)) {
LOG(ERROR) << "fileName = " << filename
<< ", inode = " << fileInfo.id()
<< ", submit delete file job fail.";
return StatusCode::KInternalError;
}

LOG(INFO) << "delete file task submitted, file is pagefile"
<< ", filename = " << filename;
<< ", inode = " << fileInfo.id()
<< ", filename = " << filename;
return StatusCode::kOK;
}
} else {
Expand Down
2 changes: 1 addition & 1 deletion test/mds/chunkserverclient/test_chunkserver_client.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
#include "proto/chunk.pb.h"
#include "src/mds/chunkserverclient/chunkserver_client.h"
#include "test/mds/mock/mock_topology.h"
#include "test/mds/chunkserverclient/mock_chunkserver.h"
#include "test/mds/mock/mock_chunkserver.h"


using ::curve::mds::topology::READWRITE;
Expand Down
2 changes: 1 addition & 1 deletion test/mds/chunkserverclient/test_copyset_client.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
#include "proto/chunk.pb.h"
#include "src/mds/chunkserverclient/copyset_client.h"
#include "test/mds/mock/mock_topology.h"
#include "test/mds/chunkserverclient/mock_chunkserver.h"
#include "test/mds/mock/mock_chunkserver.h"
#include "test/mds/chunkserverclient/mock_chunkserverclient.h"


Expand Down
3 changes: 3 additions & 0 deletions test/mds/mock/BUILD
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,4 +9,7 @@ cc_library(
srcs = glob(["*.cpp", "*.h"]),
copts = GCC_TEST_FLAGS,
visibility = ["//visibility:public"],
deps = [
"//proto:chunkserver-cc-protos",
],
)
6 changes: 3 additions & 3 deletions .../mds/chunkserverclient/mock_chunkserver.h → test/mds/mock/mock_chunkserver.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@
* Copyright (c) 2018 netease
*/

#ifndef TEST_MDS_CHUNKSERVERCLIENT_MOCK_CHUNKSERVER_H_
#define TEST_MDS_CHUNKSERVERCLIENT_MOCK_CHUNKSERVER_H_
#ifndef TEST_MDS_MOCK_MOCK_CHUNKSERVER_H_
#define TEST_MDS_MOCK_MOCK_CHUNKSERVER_H_

#include "proto/cli2.pb.h"
#include "proto/chunk.pb.h"
Expand Down Expand Up @@ -46,4 +46,4 @@ class MockCliService : public CliService2 {
} // namespace curve


#endif // TEST_MDS_CHUNKSERVERCLIENT_MOCK_CHUNKSERVER_H_
#endif // TEST_MDS_MOCK_MOCK_CHUNKSERVER_H_
100 changes: 84 additions & 16 deletions test/mds/nameserver2/clean_core_test.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,22 @@ TEST(CleanCore, testcleansnapshotfile) {
auto cleanCore = new CleanCore(storage, topology);

{
// delete ok (no, segment)
EXPECT_CALL(*storage, GetSegment(_, _, _))
.WillRepeatedly(Return(StoreStatus::KeyNotExist));
// segment size = 0
FileInfo cleanFile;
cleanFile.set_length(kMiniFileLength);
cleanFile.set_segmentsize(0);
TaskProgress progress;
ASSERT_EQ(cleanCore->CleanSnapShotFile(cleanFile, &progress),
StatusCode::KInternalError);
}

{
// delete ok (no, segment)
uint32_t segmentNum = kMiniFileLength / DefaultSegmentSize;
for (uint32_t i = 0; i < segmentNum; i++) {
EXPECT_CALL(*storage, GetSegment(_, i * DefaultSegmentSize, _))
.WillOnce(Return(StoreStatus::KeyNotExist));
}

EXPECT_CALL(*storage, DeleteSnapshotFile(_, _))
.Times(1)
Expand All @@ -46,9 +58,11 @@ TEST(CleanCore, testcleansnapshotfile) {
}
{
// all ok , but do DeleteFile namespace meta error
EXPECT_CALL(*storage, GetSegment(_, _, _))
.WillRepeatedly(Return(StoreStatus::KeyNotExist));

uint32_t segmentNum = kMiniFileLength / DefaultSegmentSize;
for (uint32_t i = 0; i < segmentNum; i++) {
EXPECT_CALL(*storage, GetSegment(_, i * DefaultSegmentSize, _))
.WillOnce(Return(StoreStatus::KeyNotExist));
}

EXPECT_CALL(*storage, DeleteSnapshotFile(_, _))
.WillOnce(Return(StoreStatus::InternalError));
Expand All @@ -63,7 +77,7 @@ TEST(CleanCore, testcleansnapshotfile) {

{
// get segment error
EXPECT_CALL(*storage, GetSegment(_, _, _))
EXPECT_CALL(*storage, GetSegment(_, 0, _))
.Times(1)
.WillOnce(Return(StoreStatus::InternalError));

Expand All @@ -77,9 +91,13 @@ TEST(CleanCore, testcleansnapshotfile) {
{
// 联调Bug修复：快照文件共享源文件的segment，所以在查询segment的时候需要使用
// ParentID 进行查找
uint32_t segmentNum = kMiniFileLength / DefaultSegmentSize;
uint64_t expectParentID = 101;
EXPECT_CALL(*storage, GetSegment(expectParentID, _, _))
.WillRepeatedly(Return(StoreStatus::KeyNotExist));
for (uint32_t i = 0; i < segmentNum; i++) {
EXPECT_CALL(*storage,
GetSegment(expectParentID, i * DefaultSegmentSize, _))
.WillOnce(Return(StoreStatus::KeyNotExist));
}

EXPECT_CALL(*storage, DeleteSnapshotFile(_, _))
.Times(1)
Expand All @@ -99,6 +117,29 @@ TEST(CleanCore, testcleansnapshotfile) {
{
// get segment ok, DeleteSnapShotChunk Error
}

{
// get segment ok, DeleteSnapShotChunk OK
uint32_t segmentNum = kMiniFileLength / DefaultSegmentSize;
for (uint32_t i = 0; i < segmentNum; i++) {
EXPECT_CALL(*storage, GetSegment(_, i * DefaultSegmentSize, _))
.WillOnce(Return(StoreStatus::OK));
}

EXPECT_CALL(*storage, DeleteSnapshotFile(_, _))
.Times(1)
.WillOnce(Return(StoreStatus::OK));

FileInfo cleanFile;
cleanFile.set_length(kMiniFileLength);
cleanFile.set_segmentsize(DefaultSegmentSize);
TaskProgress progress;
ASSERT_EQ(cleanCore->CleanSnapShotFile(cleanFile, &progress),
StatusCode::kOK);

ASSERT_EQ(progress.GetStatus(), TaskStatus::SUCCESS);
ASSERT_EQ(progress.GetProgress(), 100);
}
delete storage;
}

Expand All @@ -108,10 +149,22 @@ TEST(CleanCore, testcleanfile) {
auto cleanCore = new CleanCore(storage, topology);

{
// delete ok (no, segment)
EXPECT_CALL(*storage, GetSegment(_, _, _))
.WillRepeatedly(Return(StoreStatus::KeyNotExist));
// segmentsize = 0
FileInfo cleanFile;
cleanFile.set_length(kMiniFileLength);
cleanFile.set_segmentsize(0);
TaskProgress progress;
ASSERT_EQ(cleanCore->CleanFile(cleanFile, &progress),
StatusCode::KInternalError);
}

{
// delete ok (no, segment)
uint32_t segmentNum = kMiniFileLength / DefaultSegmentSize;
for (uint32_t i = 0; i < segmentNum; i++) {
EXPECT_CALL(*storage, GetSegment(_, i * DefaultSegmentSize, _))
.WillOnce(Return(StoreStatus::KeyNotExist));
}

EXPECT_CALL(*storage, DeleteFile(_, _))
.Times(1)
Expand All @@ -130,9 +183,11 @@ TEST(CleanCore, testcleanfile) {

{
// all ok , but do DeleteFile namespace meta error
EXPECT_CALL(*storage, GetSegment(_, _, _))
.WillRepeatedly(Return(StoreStatus::KeyNotExist));

uint32_t segmentNum = kMiniFileLength / DefaultSegmentSize;
for (uint32_t i = 0; i < segmentNum; i++) {
EXPECT_CALL(*storage, GetSegment(_, i * DefaultSegmentSize, _))
.WillOnce(Return(StoreStatus::KeyNotExist));
}

EXPECT_CALL(*storage, DeleteFile(_, _))
.WillOnce(Return(StoreStatus::InternalError));
Expand All @@ -148,7 +203,7 @@ TEST(CleanCore, testcleanfile) {

{
// get segment error
EXPECT_CALL(*storage, GetSegment(_, _, _))
EXPECT_CALL(*storage, GetSegment(_, 0, _))
.Times(1)
.WillOnce(Return(StoreStatus::InternalError));

Expand All @@ -165,6 +220,19 @@ TEST(CleanCore, testcleanfile) {
}
{
// get segment ok, DeleteSnapShotChunk ok, DeleteSegment error
EXPECT_CALL(*storage, GetSegment(_, 0, _))
.WillOnce(Return(StoreStatus::OK));

EXPECT_CALL(*storage, DeleteSegment(_, _))
.WillOnce(Return(StoreStatus::InternalError));

FileInfo cleanFile;
cleanFile.set_length(kMiniFileLength);
cleanFile.set_segmentsize(DefaultSegmentSize);
TaskProgress progress;
ASSERT_EQ(cleanCore->CleanFile(cleanFile, &progress),
StatusCode::kCommonFileDeleteError);
ASSERT_EQ(progress.GetStatus(), TaskStatus::FAILED);
}
delete storage;
}
Expand Down
Loading

0 comments on commit 1a89396

Please sign in to comment.